Repository: kislyuk/domovoi Branch: master Commit: 7d68f312b6f9 Files: 39 Total size: 120.1 KB Directory structure: gitextract_670wysb1/ ├── .gitignore ├── .travis.yml ├── Changes.rst ├── LICENSE ├── MANIFEST.in ├── Makefile ├── README.rst ├── common.mk ├── docs/ │ ├── Makefile │ ├── conf.py │ └── index.rst ├── domovoi/ │ ├── __init__.py │ ├── app.py │ ├── default_iam_policy.json │ ├── examples/ │ │ ├── alb-event.json │ │ ├── alexa-event.json │ │ ├── apigateway-event.json │ │ ├── cloudformation-event.json │ │ ├── cloudfront-event.json │ │ ├── cloudtrail-event.json │ │ ├── cloudwatch-event.json │ │ ├── codecommit-event.json │ │ ├── cognito-event.json │ │ ├── config-event.json │ │ ├── dynamodb-event.json │ │ ├── firehose-event.json │ │ ├── kinesis-event.json │ │ ├── logs-event.json │ │ ├── s3-event.json │ │ ├── ses-event.json │ │ ├── sns-event.json │ │ ├── sqs-event.json │ │ ├── state_machine_app.py │ │ └── state_machine_threadpool_app.py │ └── utils.py ├── scripts/ │ └── domovoi ├── setup.cfg ├── setup.py └── test/ └── test.py ================================================ FILE CONTENTS ================================================ ================================================ FILE: .gitignore ================================================ # Reminder: # - A leading slash means the pattern is anchored at the root. # - No leading slash means the pattern matches at any depth. # Python files *.pyc __pycache__/ .tox/ *.egg-info/ /build/ /dist/ /.eggs/ # Sphinx documentation /docs/_build/ # IDE project files /.pydevproject # vim python-mode plugin /.ropeproject # IntelliJ IDEA / PyCharm project files /.idea /*.iml # JS/node/npm/web dev files node_modules npm-debug.log # OS X metadata files .DS_Store ================================================ FILE: .travis.yml ================================================ language: python sudo: required dist: bionic cache: pip python: - 2.7 - 3.6 - 3.7 - 3.8 env: global: - AWS_DEFAULT_REGION=us-east-1 before_install: - pip install --quiet coverage flake8 pyyaml install: - make install script: - make test after_success: - bash <(curl -s https://codecov.io/bash) sudo: false ================================================ FILE: Changes.rst ================================================ Changes for v2.0.2 (2019-08-26) =============================== - Record step function ARN in deployed values. Fixes #22 - Return enclosed function in ALB decorator - Minor documentation improvements Changes for v2.0.1 (2019-04-15) =============================== - Manage route53 records for ALB Changes for v2.0.0 (2019-04-15) =============================== - Initial support for ALB Changes for v1.9.0 (2019-01-10) =============================== - Condense SNS/SQS names for brevity Changes for v1.8.3 (2018-10-11) =============================== - Add app.state_machine.start_named_execution(name, \*args) Changes for v1.8.2 (2018-09-14) =============================== Retain S3-SQS mux ability Changes for v1.8.1 (2018-09-14) =============================== Build SNS-SQS bridge to mux lambdas onto S3 event types Changes for v1.8.0 (2018-08-29) =============================== Fix essential logging, take 4 Changes for v1.7.10 (2018-08-29) ================================ Fix essential logging, take 3 Changes for v1.7.9 (2018-08-29) =============================== Fix essential logging, take 2 Changes for v1.7.8 (2018-08-29) =============================== Fix essential logging Changes for v1.7.7 (2018-08-29) =============================== - Enable refresh deployment in packager - Use Python logging library instead of Lambda context.log (#11, thanks to @irgeek) Changes for v1.7.6 (2018-08-28) =============================== - Remove debug statement Changes for v1.7.5 (2018-08-14) =============================== - Grant SQS permissions to invoke lambda Changes for v1.7.4 (2018-07-10) =============================== - Allow queue attributes to be set in SQS Changes for v1.7.3 (2018-07-06) =============================== - SQS S3 event envelope support Changes for v1.7.2 (2018-07-05) =============================== - Support state machine introspection and SQS-SFN springboard - Update docs for SQS Changes for v1.7.1 (2018-07-05) =============================== - Don’t assume eventSource exists Changes for v1.7.0 (2018-07-05) =============================== - Add support for SQS event sources Changes for v1.6.1 (2018-06-06) =============================== - Set function description from “description” config key - Fix version reporting and provide it in published Lambda tag Changes for v1.6.0 (2018-06-05) =============================== - Domovoi is now compatible with Chalice 1.2+. This should be a backwards compatible change. However, Chalice underwent a complete deployment system rewrite in version 1.2, so deployment state or other aspects of your app’s deployment may be affected. You may need to clear the deployment state of your app. Changes for v1.5.8 (2018-05-04) =============================== - Allow customizable rule name for @scheduled_function (fixes #9) Changes for v1.5.7 (2018-05-03) =============================== - Ensure domovoi errors when there is no handler Changes for v1.5.6 (2018-04-09) =============================== - Allow configurable concurrency reservation Changes for v1.5.5 (2018-03-16) =============================== - Add support for new-project and update docs Changes for v1.5.4 (2018-03-03) =============================== - Skip updating event source mapping when diff is null Changes for v1.5.3 (2018-02-21) =============================== - Avoid triggering Lambda API rate limits when managing state aliases Changes for v1.5.2 (2018-02-05) =============================== - Ensure SNS topic names can represent all DNS-compliant S3 bucket names. Fixes #5 Changes for v1.5.1 (2018-02-01) =============================== - Fix routing of domovoi dynamodb handlers Changes for v1.5.0 (2018-02-01) =============================== - Add DynamoDB streams support - Bypass prompt when writing IAM policy for the first time Changes for v1.4.5 (2017-12-12) =============================== - Only call step\_function\_task if the state has a Resource field that's callable Changes for v1.4.4 (2017-12-11) =============================== - Allow state machine registration; pass state name in context - Deconflict concurrent S3 notification config operations Changes for v1.4.3 (2017-11-29) =============================== - Improve SM updates: use update\_state\_machine Changes for v1.4.2 (2017-11-14) =============================== Accommodate eventual consistency in SM update loop Changes for v1.4.1 (2017-11-14) =============================== - Add statement to debug SM deploy loop crash Changes for v1.4.0 (2017-11-09) =============================== - Add support for CloudWatch Logs subscription filter events - Expand docs for step function / state machine examples Changes for v1.3.2 (2017-11-07) =============================== - Support nested states Changes for v1.3.1 (2017-10-30) =============================== - Key state machine tasks by state name, not function name - Parameterize sfn trust statement by region Changes for v1.3.0 (2017-10-26) =============================== - Add step functions support Changes for v1.2.6 (2017-08-26) =============================== - Monkey-patch chalice to avoid dependency wheel management bug - Use more intuitive errors when handler not found Changes for v1.2.5 (2017-08-17) =============================== Avoid running privileged op on update Changes for v1.2.4 (2017-08-17) =============================== - Chalice 1.0 compat, part 3 Changes for v1.2.3 (2017-08-17) =============================== - Chalice 1.0 compat, part 2 Changes for v1.2.2 (2017-08-17) =============================== Chalice 1.0 compatibility fixes Changes for v1.2.1 (2017-07-14) =============================== - Simplify DLQ handling; add docs for DLQ Changes for v1.2.0 (2017-07-14) =============================== - Support DLQ lambda config Changes for v1.1.1 (2017-07-05) =============================== - Parameterize stage name, part 2 Changes for v1.1.0 (2017-07-05) =============================== - Parameterize stage name Changes for v1.0.9 (2017-06-24) =============================== - Forward S3 notifications through SNS by default Changes for v1.0.8 (2017-06-24) =============================== - Don't clobber existing S3 bucket notifications Changes for v1.0.7 (2017-06-22) =============================== - Pass through configure\_logs - Test improvements Changes for v1.0.6 (2017-06-15) =============================== Fix error in release Changes for v1.0.5 (2017-06-15) =============================== Enable idempotent Lambda permission grants Changes for v1.0.4 (2017-06-09) =============================== - Hardcode no autogen policy Changes for v1.0.3 (2017-06-08) =============================== - Ensure S3 bucket notifications work without filters specified Changes for v1.0.2 (2017-06-01) =============================== - Fix dispatching of S3 events - Fixes to deploy procedure Changes for v1.0.1 (2017-06-01) =============================== - Fix event subscriptions Changes for v1.0.0 (2017-05-28) =============================== - Update to be compatible with Chalice 0.8 and Python 3.6 Changes for v0.0.3 (2016-12-19) =============================== - Autogenerate IAM policy - Release automation Version 0.0.1 (2016-12-14) -------------------------- - Initial release. ================================================ FILE: LICENSE ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "{}" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright {yyyy} {name of copyright owner} Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: MANIFEST.in ================================================ include *.rst include test/* include domovoi/*.json include domovoi/examples/*.json ================================================ FILE: Makefile ================================================ SHELL=/bin/bash lint: ./setup.py flake8 flake8 scripts/* test: lint -rm -rf testproject testproject2 testproject-sfn python ./test/test.py -v init_docs: cd docs; sphinx-quickstart docs: $(MAKE) -C docs html install: -rm -rf dist python setup.py bdist_wheel pip install --upgrade dist/*.whl .PHONY: test release docs include common.mk ================================================ FILE: README.rst ================================================ Domovoi: AWS Lambda event handler manager ========================================= *Domovoi* is an extension to `AWS Chalice `_ to handle `AWS Lambda `_ `event sources `_ other than HTTP requests through API Gateway. Domovoi lets you easily configure and deploy a Lambda function to serve HTTP requests through `ALB `_, on a schedule, or in response to a variety of events like an `SNS `_ or `SQS `_ message, S3 event, or custom `state machine `_ transition: .. code-block:: python import json, boto3, domovoi app = domovoi.Domovoi() # Compared to API Gateway, ALB increases the response timeout from 30s to 900s, but reduces the payload # limit from 10MB to 1MB. It also does not try to negotiate on the Accept/Content-Type headers. @app.alb_target() def serve(event, context): return dict(statusCode=200, statusDescription="200 OK", isBase64Encoded=False, headers={"Content-Type": "application/json"}, body=json.dumps({"hello": "world"})) @app.scheduled_function("cron(0 18 ? * MON-FRI *)") def foo(event, context): context.log("foo invoked at 06:00pm (UTC) every Mon-Fri") return dict(result=True) @app.scheduled_function("rate(1 minute)") def bar(event, context): context.log("bar invoked once a minute") boto3.resource("sns").create_topic(Name="bartender").publish(Message=json.dumps({"beer": 1})) return dict(result="Work work work") @app.sns_topic_subscriber("bartender") def tend(event, context): message = json.loads(event["Records"][0]["Sns"]["Message"]) context.log(dict(beer="Quadrupel", quantity=message["beer"])) # SQS messages are deleted upon successful exit, requeued otherwise. # See https://docs.aws.amazon.com/lambda/latest/dg/with-sqs.html @app.sqs_queue_subscriber("my_queue", batch_size=64) def process_queue_messages(event, context): message = json.loads(event["Records"][0]["body"]) message_attributes = event["Records"][0]["messageAttributes"] # You can colocate a state machine definition with an SQS handler to launch a SFN driven lambda from SQS. return app.state_machine.start_execution(**message)["executionArn"] @app.cloudwatch_event_handler(source=["aws.ecs"]) def monitor_ecs_events(event, context): message = json.loads(event["Records"][0]["Sns"]["Message"]) context.log("Got an event from ECS: {}".format(message)) @app.s3_event_handler(bucket="myS3bucket", events=["s3:ObjectCreated:*"], prefix="foo", suffix=".bar") def monitor_s3(event, context): context.log("Got an event from S3: {}".format(event)) # Set use_sns=False, use_sqs=False to subscribe your Lambda directly to S3 events without forwarding them through an SNS-SQS bridge. # That approach has fewer moving parts, but you can only subscribe one Lambda function to events in a given S3 bucket. @app.s3_event_handler(bucket="myS3bucket", events=["s3:ObjectCreated:*"], prefix="foo", suffix=".bar", use_sns=False, use_sqs=False) def monitor_s3(event, context): context.log("Got an event from S3: {}".format(event)) # DynamoDB event format: https://docs.aws.amazon.com/lambda/latest/dg/with-ddb.html @app.dynamodb_stream_handler(table_name="MyDynamoTable", batch_size=200) def handle_dynamodb_stream(event, context): context.log("Got {} events from DynamoDB".format(len(event["Records"]))) context.log("First event: {}".format(event["Records"][0]["dynamodb"])) # Use the following command to log a CloudWatch Logs message that will trigger this handler: # python -c'import watchtower as w, logging as l; L=l.getLogger(); L.addHandler(w.CloudWatchLogHandler()); L.error(dict(x=8))' # See http://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/FilterAndPatternSyntax.html for the filter pattern syntax @app.cloudwatch_logs_sub_filter_handler(log_group_name="watchtower", filter_pattern="{$.x = 8}") def monitor_cloudwatch_logs(event, context): print("Got a CWL subscription filter event:", event) # See http://docs.aws.amazon.com/step-functions/latest/dg/concepts-amazon-states-language.html # See the "AWS Step Functions state machines" section below for a complete example of setting up a state machine. @app.step_function_task(state_name="Worker", state_machine_definition=state_machine) def worker(event, context): return {"result": event["input"] + 1, "my_state": context.stepfunctions_task_name} Installation ------------ :: pip install domovoi Usage ----- First-time setup:: domovoi new-project * Edit the Domovoi app entry point in ``app.py`` using examples above. * Edit the IAM policy for your Lambda function in ``my_project/.chalice/policy-dev.json`` to add any permissions it needs. * Deploy the event handlers:: domovoi deploy To stage files into the deployment package, use a ``domovoilib`` directory in your project where you would use ``chalicelib`` in Chalice. For example, ``my_project/domovoilib/rds_cert.pem`` becomes ``/var/task/domovoilib/rds_cert.pem`` with your function executing in ``/var/task/app.py`` with ``/var/task`` as the working directory. See the `Chalice docs `_ for more information on how to set up Chalice configuration. Supported event types ~~~~~~~~~~~~~~~~~~~~~ See `Supported Event Sources `_ for an overview of event sources that can be used to trigger Lambda functions. Domovoi supports the following event sources: * `ALB HTTPS requests `_ * `SNS subscriptions `_ * `SQS queues `_ * CloudWatch Events rule targets, including `CloudWatch Scheduled Events `_ (see `CloudWatch Events Event Examples `_ for a list of event types supported by CloudWatch Events) * `S3 events `_ * AWS Step Functions state machine tasks * `CloudWatch Logs filter subscriptions `_ * `DynamoDB stream events `_ Possible future event sources to support: * Kinesis stream events * SES (email) events AWS Step Functions state machines ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Domovoi supports AWS Lambda integration with `AWS Step Functions `_. Step Functions state machines can be started using the `StartExecution `_ method or the `API Gateway Step Functions integration `_. See the `domovoi/examples `_ directory for examples of Domovoi ``app.py`` apps using a state machine, including a loop that restarts the Lambda when it's about to hit its execution time limit, and a threadpool pattern that divides work between multiple Lambdas. When creating a Step Functions State Machine driven Domovoi daemon Lambda, the State Machine assumes the same IAM role as the Lambda itself. To allow the State Machine to invoke the Lambda, edit the IAM policy (under your app directory, in ``.chalice/policy-dev.json``) to include a statement allowing the "lambda:InvokeFunction" action on all resources, or on the ARN of the Lambda itself. Configuration ~~~~~~~~~~~~~ ALB ^^^ To use your Lambda as an ALB target with the ``@alb_target(prefix="...")`` decorator, you should pre-configure the following resources in your AWS account: * A Route 53 hosted DNS zone such as ``example.com.``, with a domain (``example.com``) pointing to it * An active (verified/issued) ACM certificate for a DNS name within your DNS zone, such as ``domovoi.example.com`` After configuring these, set the ``alb_acm_cert_dns_name`` configuration key in the file ``.chalice/config.json`` to your DNS name. For example:: { "app_name": "my_app", ... "alb_acm_cert_dns_name": "domovoi.example.com" } Domovoi will automatically create, manage, and link the ALB and DNS record in your Route 53 zone. Dead Letter Queues ^^^^^^^^^^^^^^^^^^ To enable your Lambda function to forward failed invocation notifications to `dead letter queues `_, set the configuration key ``dead_letter_queue_target_arn`` in the file ``.chalice/config.json`` to the target DLQ ARN. For example:: { "app_name": "my_app", ... "dead_letter_queue_target_arn": "arn:aws:sns:us-east-1:123456789012:my-dlq" } You may need to update your Lambda IAM policy (``.chalice/policy-dev.json``) to give your Lambda access to SNS or SQS. Concurrency Reservations ^^^^^^^^^^^^^^^^^^^^^^^^ For high volume Lambda invocations in accounts with multiple Lambdas, you may need to set `per-function concurrency limits `_ to partition the overall concurrency quota and prevent one set of Lambdas from overloading another. In Domovoi, you can do so by setting the configuration key ``reserved_concurrent_executions`` in the file ``.chalice/config.json`` to the desired concurrency reservation. For example:: { "app_name": "my_app", ... "reserved_concurrent_executions": 500 } Links ----- * `Project home page (GitHub) `_ * `Documentation (Read the Docs) `_ * `Package distribution (PyPI) `_ * `Change log `_ Bugs ~~~~ Please report bugs, issues, feature requests, etc. on `GitHub `_. License ------- Licensed under the terms of the `Apache License, Version 2.0 `_. .. image:: https://travis-ci.org/kislyuk/domovoi.png :target: https://travis-ci.org/kislyuk/domovoi .. image:: https://codecov.io/github/kislyuk/domovoi/coverage.svg?branch=master :target: https://codecov.io/github/kislyuk/domovoi?branch=master .. image:: https://img.shields.io/pypi/v/domovoi.svg :target: https://pypi.python.org/pypi/domovoi .. image:: https://img.shields.io/pypi/l/domovoi.svg :target: https://pypi.python.org/pypi/domovoi .. image:: https://readthedocs.org/projects/domovoi/badge/?version=latest :target: https://domovoi.readthedocs.org/ ================================================ FILE: common.mk ================================================ SHELL=/bin/bash -eo pipefail release_major: $(eval export TAG=$(shell git describe --tags --match 'v*.*.*' | perl -ne '/^v(\d+)\.(\d+)\.(\d+)/; print "v@{[$$1+1]}.0.0"')) $(MAKE) release release_minor: $(eval export TAG=$(shell git describe --tags --match 'v*.*.*' | perl -ne '/^v(\d+)\.(\d+)\.(\d+)/; print "v$$1.@{[$$2+1]}.0"')) $(MAKE) release release_patch: $(eval export TAG=$(shell git describe --tags --match 'v*.*.*' | perl -ne '/^v(\d+)\.(\d+)\.(\d+)/; print "v$$1.$$2.@{[$$3+1]}"')) $(MAKE) release release: @if [[ -z $$TAG ]]; then echo "Use release_{major,minor,patch}"; exit 1; fi @if ! type -P pandoc; then echo "Please install pandoc"; exit 1; fi @if ! type -P sponge; then echo "Please install moreutils"; exit 1; fi @if ! type -P http; then echo "Please install httpie"; exit 1; fi @if ! type -P twine; then echo "Please install twine"; exit 1; fi $(eval REMOTE=$(shell git remote get-url origin | perl -ne '/([^\/\:]+\/.+?)(\.git)?$$/; print $$1')) $(eval GIT_USER=$(shell git config --get user.email)) $(eval GH_AUTH=$(shell if grep -q '@github.com' ~/.git-credentials; then echo $$(grep '@github.com' ~/.git-credentials | python3 -c 'import sys, urllib.parse as p; print(p.urlparse(sys.stdin.read()).netloc.split("@")[0])'); else echo $(GIT_USER); fi)) $(eval RELEASES_API=https://api.github.com/repos/${REMOTE}/releases) $(eval UPLOADS_API=https://uploads.github.com/repos/${REMOTE}/releases) git pull git clean -x --force $$(python setup.py --name) sed -i -e "s/version=\([\'\"]\)[0-9]*\.[0-9]*\.[0-9]*/version=\1$${TAG:1}/" setup.py git add setup.py TAG_MSG=$$(mktemp); \ echo "# Changes for ${TAG} ($$(date +%Y-%m-%d))" > $$TAG_MSG; \ git log --pretty=format:%s $$(git describe --abbrev=0)..HEAD >> $$TAG_MSG; \ $${EDITOR:-emacs} $$TAG_MSG; \ if [[ -f Changes.md ]]; then cat $$TAG_MSG <(echo) Changes.md | sponge Changes.md; git add Changes.md; fi; \ if [[ -f Changes.rst ]]; then cat <(pandoc --from markdown --to rst $$TAG_MSG) <(echo) Changes.rst | sponge Changes.rst; git add Changes.rst; fi; \ git commit -m ${TAG}; \ git tag --sign --annotate --file $$TAG_MSG ${TAG} git push --follow-tags http --auth ${GH_AUTH} ${RELEASES_API} tag_name=${TAG} name=${TAG} \ body="$$(git tag --list ${TAG} -n99 | perl -pe 's/^\S+\s*// if $$. == 1' | sed 's/^\s\s\s\s//')" $(MAKE) install http --auth ${GH_AUTH} POST ${UPLOADS_API}/$$(http --auth ${GH_AUTH} ${RELEASES_API}/latest | jq .id)/assets \ name==$$(basename dist/*.whl) label=="Python Wheel" < dist/*.whl $(MAKE) pypi_release pypi_release: python setup.py sdist bdist_wheel twine upload dist/*.tar.gz dist/*.whl --sign --verbose .PHONY: release ================================================ FILE: docs/Makefile ================================================ # Makefile for Sphinx documentation # # You can set these variables from the command line. SPHINXOPTS = SPHINXBUILD = sphinx-build PAPER = BUILDDIR = _build # Internal variables. PAPEROPT_a4 = -D latex_paper_size=a4 PAPEROPT_letter = -D latex_paper_size=letter ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . # the i18n builder cannot share the environment and doctrees with the others I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . .PHONY: help help: @echo "Please use \`make ' where is one of" @echo " html to make standalone HTML files" @echo " dirhtml to make HTML files named index.html in directories" @echo " singlehtml to make a single large HTML file" @echo " pickle to make pickle files" @echo " json to make JSON files" @echo " htmlhelp to make HTML files and a HTML help project" @echo " qthelp to make HTML files and a qthelp project" @echo " applehelp to make an Apple Help Book" @echo " devhelp to make HTML files and a Devhelp project" @echo " epub to make an epub" @echo " epub3 to make an epub3" @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" @echo " latexpdf to make LaTeX files and run them through pdflatex" @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" @echo " text to make text files" @echo " man to make manual pages" @echo " texinfo to make Texinfo files" @echo " info to make Texinfo files and run them through makeinfo" @echo " gettext to make PO message catalogs" @echo " changes to make an overview of all changed/added/deprecated items" @echo " xml to make Docutils-native XML files" @echo " pseudoxml to make pseudoxml-XML files for display purposes" @echo " linkcheck to check all external links for integrity" @echo " doctest to run all doctests embedded in the documentation (if enabled)" @echo " coverage to run coverage check of the documentation (if enabled)" @echo " dummy to check syntax errors of document sources" .PHONY: clean clean: rm -rf $(BUILDDIR)/* .PHONY: html html: $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." .PHONY: dirhtml dirhtml: $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." .PHONY: singlehtml singlehtml: $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml @echo @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." .PHONY: pickle pickle: $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle @echo @echo "Build finished; now you can process the pickle files." .PHONY: json json: $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json @echo @echo "Build finished; now you can process the JSON files." .PHONY: htmlhelp htmlhelp: $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp @echo @echo "Build finished; now you can run HTML Help Workshop with the" \ ".hhp project file in $(BUILDDIR)/htmlhelp." .PHONY: qthelp qthelp: $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp @echo @echo "Build finished; now you can run "qcollectiongenerator" with the" \ ".qhcp project file in $(BUILDDIR)/qthelp, like this:" @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/Domovoi.qhcp" @echo "To view the help file:" @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/Domovoi.qhc" .PHONY: applehelp applehelp: $(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp @echo @echo "Build finished. The help book is in $(BUILDDIR)/applehelp." @echo "N.B. You won't be able to view it unless you put it in" \ "~/Library/Documentation/Help or install it in your application" \ "bundle." .PHONY: devhelp devhelp: $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp @echo @echo "Build finished." @echo "To view the help file:" @echo "# mkdir -p $$HOME/.local/share/devhelp/Domovoi" @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/Domovoi" @echo "# devhelp" .PHONY: epub epub: $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub @echo @echo "Build finished. The epub file is in $(BUILDDIR)/epub." .PHONY: epub3 epub3: $(SPHINXBUILD) -b epub3 $(ALLSPHINXOPTS) $(BUILDDIR)/epub3 @echo @echo "Build finished. The epub3 file is in $(BUILDDIR)/epub3." .PHONY: latex latex: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." @echo "Run \`make' in that directory to run these through (pdf)latex" \ "(use \`make latexpdf' here to do that automatically)." .PHONY: latexpdf latexpdf: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo "Running LaTeX files through pdflatex..." $(MAKE) -C $(BUILDDIR)/latex all-pdf @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." .PHONY: latexpdfja latexpdfja: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo "Running LaTeX files through platex and dvipdfmx..." $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." .PHONY: text text: $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text @echo @echo "Build finished. The text files are in $(BUILDDIR)/text." .PHONY: man man: $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man @echo @echo "Build finished. The manual pages are in $(BUILDDIR)/man." .PHONY: texinfo texinfo: $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo @echo @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." @echo "Run \`make' in that directory to run these through makeinfo" \ "(use \`make info' here to do that automatically)." .PHONY: info info: $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo @echo "Running Texinfo files through makeinfo..." make -C $(BUILDDIR)/texinfo info @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." .PHONY: gettext gettext: $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale @echo @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." .PHONY: changes changes: $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes @echo @echo "The overview file is in $(BUILDDIR)/changes." .PHONY: linkcheck linkcheck: $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck @echo @echo "Link check complete; look for any errors in the above output " \ "or in $(BUILDDIR)/linkcheck/output.txt." .PHONY: doctest doctest: $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest @echo "Testing of doctests in the sources finished, look at the " \ "results in $(BUILDDIR)/doctest/output.txt." .PHONY: coverage coverage: $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage @echo "Testing of coverage in the sources finished, look at the " \ "results in $(BUILDDIR)/coverage/python.txt." .PHONY: xml xml: $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml @echo @echo "Build finished. The XML files are in $(BUILDDIR)/xml." .PHONY: pseudoxml pseudoxml: $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml @echo @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." .PHONY: dummy dummy: $(SPHINXBUILD) -b dummy $(ALLSPHINXOPTS) $(BUILDDIR)/dummy @echo @echo "Build finished. Dummy builder generates no files." ================================================ FILE: docs/conf.py ================================================ # -*- coding: utf-8 -*- # # Domovoi documentation build configuration file, created by # sphinx-quickstart on Wed Dec 14 20:23:38 2016. # # This file is execfile()d with the current directory set to its # containing dir. # # Note that not all possible configuration values are present in this # autogenerated file. # # All configuration values have a default; values that are commented out # serve to show the default. # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. # # import os # import sys # sys.path.insert(0, os.path.abspath('.')) # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. # # needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ 'sphinx.ext.autodoc', 'sphinx.ext.viewcode', 'sphinx.ext.githubpages', ] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: # # source_suffix = ['.rst', '.md'] source_suffix = '.rst' # The encoding of source files. # # source_encoding = 'utf-8-sig' # The master toctree document. master_doc = 'index' # General information about the project. project = u'Domovoi' copyright = u'2016, Andrey Kislyuk' author = u'Andrey Kislyuk' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. version = u'0.0.1' # The full version, including alpha/beta/rc tags. release = u'0.0.1' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: # # today = '' # # Else, today_fmt is used as the format for a strftime call. # # today_fmt = '%B %d, %Y' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This patterns also effect to html_static_path and html_extra_path exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] # The reST default role (used for this markup: `text`) to use for all # documents. # # default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. # # add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). # # add_module_names = True # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. # # show_authors = False # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' # A list of ignored prefixes for module index sorting. # modindex_common_prefix = [] # If true, keep warnings as "system message" paragraphs in the built documents. # keep_warnings = False # If true, `todo` and `todoList` produce output, else they produce nothing. todo_include_todos = False # -- Options for HTML output ---------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # html_theme = 'default' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. # # html_theme_options = {} # Add any paths that contain custom themes here, relative to this directory. # html_theme_path = [] # The name for this set of Sphinx documents. # " v documentation" by default. # # html_title = u'Domovoi v0.0.1' # A shorter title for the navigation bar. Default is the same as html_title. # # html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. # # html_logo = None # The name of an image file (relative to this directory) to use as a favicon of # the docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. # # html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['_static'] # Add any extra paths that contain custom files (such as robots.txt or # .htaccess) here, relative to this directory. These files are copied # directly to the root of the documentation. # # html_extra_path = [] # If not None, a 'Last updated on:' timestamp is inserted at every page # bottom, using the given strftime format. # The empty string is equivalent to '%b %d, %Y'. # # html_last_updated_fmt = None # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. # # html_use_smartypants = True # Custom sidebar templates, maps document names to template names. # # html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. # # html_additional_pages = {} # If false, no module index is generated. # # html_domain_indices = True # If false, no index is generated. # # html_use_index = True # If true, the index is split into individual pages for each letter. # # html_split_index = False # If true, links to the reST sources are added to the pages. # # html_show_sourcelink = True # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. # # html_show_sphinx = True # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. # # html_show_copyright = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. # # html_use_opensearch = '' # This is the file name suffix for HTML files (e.g. ".xhtml"). # html_file_suffix = None # Language to be used for generating the HTML full-text search index. # Sphinx supports the following languages: # 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja' # 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr', 'zh' # # html_search_language = 'en' # A dictionary with options for the search language support, empty by default. # 'ja' uses this config value. # 'zh' user can custom change `jieba` dictionary path. # # html_search_options = {'type': 'default'} # The name of a javascript file (relative to the configuration directory) that # implements a search results scorer. If empty, the default will be used. # # html_search_scorer = 'scorer.js' # Output file base name for HTML help builder. htmlhelp_basename = 'Domovoidoc' # -- Options for LaTeX output --------------------------------------------- latex_elements = { # The paper size ('letterpaper' or 'a4paper'). # # 'papersize': 'letterpaper', # The font size ('10pt', '11pt' or '12pt'). # # 'pointsize': '10pt', # Additional stuff for the LaTeX preamble. # # 'preamble': '', # Latex figure (float) alignment # # 'figure_align': 'htbp', } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ (master_doc, 'Domovoi.tex', u'Domovoi Documentation', u'Andrey Kislyuk', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of # the title page. # # latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. # # latex_use_parts = False # If true, show page references after internal links. # # latex_show_pagerefs = False # If true, show URL addresses after external links. # # latex_show_urls = False # Documents to append as an appendix to all manuals. # # latex_appendices = [] # If false, no module index is generated. # # latex_domain_indices = True # -- Options for manual page output --------------------------------------- # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ (master_doc, 'domovoi', u'Domovoi Documentation', [author], 1) ] # If true, show URL addresses after external links. # # man_show_urls = False # -- Options for Texinfo output ------------------------------------------- # Grouping the document tree into Texinfo files. List of tuples # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ (master_doc, 'Domovoi', u'Domovoi Documentation', author, 'Domovoi', 'One line description of project.', 'Miscellaneous'), ] # Documents to append as an appendix to all manuals. # # texinfo_appendices = [] # If false, no module index is generated. # # texinfo_domain_indices = True # How to display URL addresses: 'footnote', 'no', or 'inline'. # # texinfo_show_urls = 'footnote' # If true, do not generate a @detailmenu in the "Top" node's menu. # # texinfo_no_detailmenu = False ================================================ FILE: docs/index.rst ================================================ .. include:: ../README.rst API documentation ================= .. automodule:: domovoi :members: Table of Contents ================= .. toctree:: :maxdepth: 5 index * :ref:`genindex` * :ref:`modindex` * :ref:`search` ================================================ FILE: domovoi/__init__.py ================================================ from domovoi.app import Domovoi ================================================ FILE: domovoi/app.py ================================================ from __future__ import absolute_import, division, print_function, unicode_literals import json, gzip, base64, logging from chalice.app import Chalice, LambdaFunction, DecoratorAPI as ChaliceDecoratorAPI class DomovoiException(Exception): pass class ARN: fields = "arn partition service region account_id resource".split() def __init__(self, arn="arn:aws::::", **kwargs): self.__dict__.update(dict(zip(self.fields, arn.split(":", 5)), **kwargs)) def __str__(self): return ":".join(getattr(self, field) for field in self.fields) class StateMachine: def __init__(self, app, client=None): self.app = app self._client = client @property def stepfunctions(self): if self._client is None: import boto3 self._client = boto3.client("stepfunctions") return self._client def start_execution(self, **input): return self.start_named_execution(None, **input) def start_named_execution(self, name, **input): lambda_arn = ARN(self.app.lambda_context.invoked_function_arn) lambda_name = lambda_arn.resource.split(":")[1] state_machine_arn = ARN(str(lambda_arn), service="states", resource="stateMachine:" + lambda_name) start_execution_args = dict(stateMachineArn=str(state_machine_arn), input=json.dumps(input)) if name is not None: start_execution_args.update(name=name) return self.stepfunctions.start_execution(**start_execution_args) class Domovoi(Chalice): cloudwatch_events_rules = {} sns_subscribers = {} sqs_subscribers = {} s3_subscribers = {} sfn_tasks = {} cwl_sub_filters = {} dynamodb_event_sources = {} alb_targets = {} sqs_default_queue_attributes = {"VisibilityTimeout": "320"} def unsupported_decorator(*args, **kwargs): raise NotImplementedError("Domovoi does not support this Chalice decorator") def __init__(self, app_name="Domovoi", configure_logs=True): Chalice.__init__(self, app_name=app_name, configure_logs=configure_logs) self.pure_lambda_functions = [LambdaFunction(self, name=app_name, handler_string="app.app")] for f in dir(ChaliceDecoratorAPI): if callable(getattr(ChaliceDecoratorAPI, f)) and not f.startswith("_"): setattr(self, f, Domovoi.unsupported_decorator) def _configure_log_level(self): if self._debug: level = logging.DEBUG else: level = logging.INFO self.log.setLevel(level) def alb_target(self, prefix=""): def register_alb_target(func): self.alb_targets[prefix] = dict(func=func, prefix=prefix) return func return register_alb_target def scheduled_function(self, schedule, rule_name=None): return self.cloudwatch_rule(schedule_expression=schedule, event_pattern=None, rule_name=rule_name) def sns_topic_subscriber(self, topic_name): def register_sns_subscriber(func): self.sns_subscribers[topic_name] = func return func return register_sns_subscriber def sqs_queue_subscriber(self, queue_name, batch_size=None, queue_attributes=None): def register_sqs_subscriber(func): self.sqs_subscribers[queue_name] = dict(func=func, batch_size=batch_size, queue_attributes=queue_attributes) return func return register_sqs_subscriber def dynamodb_stream_handler(self, table_name, batch_size=None): def register_dynamodb_event_source(func): self.dynamodb_event_sources[table_name] = dict(batch_size=batch_size, func=func) return func return register_dynamodb_event_source def kinesis_stream_handler(self, **kwargs): raise NotImplementedError() def email_receipt_handler(self): # http://boto3.readthedocs.io/en/latest/reference/services/ses.html#SES.Client.create_receipt_rule raise NotImplementedError() def cloudwatch_logs_sub_filter_handler(self, log_group_name, filter_pattern): def register_cwl_subscription_filter(func): self.cwl_sub_filters[log_group_name] = dict(log_group_name=log_group_name, filter_pattern=filter_pattern, func=func) return func return register_cwl_subscription_filter def cloudwatch_event_handler(self, **kwargs): return self.cloudwatch_rule(schedule_expression=None, event_pattern=kwargs) def s3_event_handler(self, bucket, events, prefix=None, suffix=None, use_sns=True, use_sqs=False, sqs_batch_size=1, sqs_queue_attributes=None): def register_s3_subscriber(func): self.s3_subscribers[bucket] = dict(events=events, prefix=prefix, suffix=suffix, func=func, use_sns=use_sns, use_sqs=use_sqs, sqs_batch_size=sqs_batch_size, sqs_queue_attributes=sqs_queue_attributes) return func return register_s3_subscriber def cloudwatch_rule(self, schedule_expression, event_pattern, rule_name=None): def register_rule(func): _rule_name = rule_name or func.__name__ if _rule_name in self.cloudwatch_events_rules: raise KeyError(func.__name__) rule = dict(schedule_expression=schedule_expression, event_pattern=event_pattern, func=func) self.cloudwatch_events_rules[_rule_name] = rule return func return register_rule def step_function_task(self, state_name, state_machine_definition): def register_sfn_task(func): if state_name in self.sfn_tasks: raise KeyError(state_name) self.sfn_tasks[state_name] = dict(state_name=state_name, state_machine_definition=state_machine_definition, func=func) return func return register_sfn_task def register_state_machine(self, state_machine_definition): for state_name, state_data in self.get_all_states(state_machine_definition).items(): if callable(state_data.get("Resource", None)): self.step_function_task(state_name, state_machine_definition)(state_data["Resource"]) @classmethod def get_all_states(cls, state_machine): states = dict(state_machine["States"]) for state_name, state_data in state_machine["States"].items(): for sub_sm in state_data.get("Branches", []): states.update(cls.get_all_states(sub_sm)) return states @property def state_machine(self): return StateMachine(app=self) def _find_forwarded_s3_event(self, s3_event_envelope, forwarding_service): assert forwarding_service in {"sns", "sqs"} if forwarding_service == "sns": assert s3_event_envelope['Records'][0]["Sns"]["Subject"] == "Amazon S3 Notification" s3_event = json.loads(s3_event_envelope['Records'][0]["Sns"]["Message"]) elif forwarding_service == "sqs": forwarded_event = json.loads(s3_event_envelope["Records"][0]["body"]) if forwarded_event.get("TopicArn") and forwarded_event.get("Subject") == "Amazon S3 Notification": s3_event = json.loads(forwarded_event["Message"]) else: s3_event = forwarded_event assert s3_event.get("Event") == "s3:TestEvent" or s3_event['Records'][0].get("eventSource") == "aws:s3" s3_bucket_name = s3_event.get("Bucket") or s3_event['Records'][0]["s3"]["bucket"]["name"] handler = self.s3_subscribers[s3_bucket_name]["func"] if s3_bucket_name in self.s3_subscribers else None return s3_event, handler def __call__(self, event, context): self.log.info("Domovoi dispatch of event %s", event) self.lambda_context = context invoked_function_arn = ARN(context.invoked_function_arn) handler = None if "requestContext" in event and "elb" in event["requestContext"]: target = None # TODO: use suffix tree to avoid O(N) scan of route table for prefix, alb_target in self.alb_targets.items(): if event["path"].startswith(prefix): if target is None or len(target["prefix"]) < len(alb_target["prefix"]): target = alb_target handler = target["func"] if "task_name" in event: if event["task_name"] not in self.cloudwatch_events_rules: raise DomovoiException("Received CloudWatch event for a task with no known handler") handler = self.cloudwatch_events_rules[event["task_name"]]["func"] event = event["event"] elif "Records" in event and "s3" in event["Records"][0]: s3_bucket_name = event["Records"][0]["s3"]["bucket"]["name"] if s3_bucket_name not in self.s3_subscribers: raise DomovoiException("Received S3 event for a bucket with no known handler") handler = self.s3_subscribers[s3_bucket_name]["func"] elif "Records" in event and "Sns" in event["Records"][0]: try: event, handler = self._find_forwarded_s3_event(event, forwarding_service="sns") except Exception: sns_topic = ARN(event["Records"][0]["Sns"]["TopicArn"]).resource if sns_topic not in self.sns_subscribers: raise DomovoiException("Received SNS or S3-SNS event with no known handler") handler = self.sns_subscribers[sns_topic] elif "Records" in event and event["Records"][0].get("eventSource") == "aws:sqs": try: event, handler = self._find_forwarded_s3_event(event, forwarding_service="sqs") except Exception: queue_name = ARN(event["Records"][0]["eventSourceARN"]).resource handler = self.sqs_subscribers[queue_name]["func"] elif "Records" in event and "dynamodb" in event["Records"][0]: event_source_arn = ARN(event["Records"][0]["eventSourceARN"]) table_name = event_source_arn.resource.split("/")[1] handler = self.dynamodb_event_sources[table_name]["func"] elif "awslogs" in event: event = json.loads(gzip.decompress(base64.b64decode(event["awslogs"]["data"]))) handler = self.cwl_sub_filters[event["logGroup"]]["func"] elif "domovoi-stepfunctions-task" in invoked_function_arn.resource: _, lambda_name, lambda_alias = invoked_function_arn.resource.split(":") assert lambda_alias.startswith("domovoi-stepfunctions-task-") task_name = lambda_alias[len("domovoi-stepfunctions-task-"):] context.stepfunctions_task_name = task_name handler = self.sfn_tasks[task_name]["func"] if handler is None: raise DomovoiException("No handler found for event {}".format(event)) result = handler(event, context) self.log.info("%s", result) return result ================================================ FILE: domovoi/default_iam_policy.json ================================================ { "Version": "2012-10-17", "Statement": [ { "Action": [ "events:*", "iam:ListAttachedRolePolicies", "iam:ListRolePolicies", "iam:ListRoles", "iam:PassRole" ], "Resource": "*", "Effect": "Allow" }, { "Action": [ "sns:CreateTopic", "sns:Publish" ], "Resource": "arn:aws:sns:*:*:*", "Effect": "Allow" }, { "Action": [ "sqs:ReceiveMessage", "sqs:DeleteMessage", "sqs:GetQueueAttributes" ], "Resource": [ "arn:aws:sqs:*:*:*" ], "Effect": "Allow" }, { "Action": [ "logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents", "logs:DescribeLogStreams" ], "Resource": [ "arn:aws:logs:*:*:*" ], "Effect": "Allow" } ] } ================================================ FILE: domovoi/examples/alb-event.json ================================================ { "requestContext": { "elb": { "targetGroupArn": "arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/lambda-279XGJDqGZ5rsrHC2Fjr/49e9d65c45c6791a" } }, "httpMethod": "GET", "path": "/lambda", "queryStringParameters": { "query": "1234ABCD" }, "headers": { "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8", "accept-encoding": "gzip", "accept-language": "en-US,en;q=0.9", "connection": "keep-alive", "host": "lambda-alb-123578498.us-east-2.elb.amazonaws.com", "upgrade-insecure-requests": "1", "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36", "x-amzn-trace-id": "Root=1-5c536348-3d683b8b04734faae651f476", "x-forwarded-for": "72.12.164.125", "x-forwarded-port": "80", "x-forwarded-proto": "http", "x-imforwards": "20" }, "body": "", "isBase64Encoded": false } ================================================ FILE: domovoi/examples/alexa-event.json ================================================ { "header": { "payloadVersion": "1", "namespace": "Control", "name": "SwitchOnOffRequest" }, "payload": { "switchControlAction": "TURN_ON", "appliance": { "additionalApplianceDetails": { "key2": "value2", "key1": "value1" }, "applianceId": "sampleId" }, "accessToken": "sampleAccessToken" } } ================================================ FILE: domovoi/examples/apigateway-event.json ================================================ { "path": "/test/hello", "headers": { "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", "Accept-Encoding": "gzip, deflate, lzma, sdch, br", "Accept-Language": "en-US,en;q=0.8", "CloudFront-Forwarded-Proto": "https", "CloudFront-Is-Desktop-Viewer": "true", "CloudFront-Is-Mobile-Viewer": "false", "CloudFront-Is-SmartTV-Viewer": "false", "CloudFront-Is-Tablet-Viewer": "false", "CloudFront-Viewer-Country": "US", "Host": "wt6mne2s9k.execute-api.us-west-2.amazonaws.com", "Upgrade-Insecure-Requests": "1", "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.82 Safari/537.36 OPR/39.0.2256.48", "Via": "1.1 fb7cca60f0ecd82ce07790c9c5eef16c.cloudfront.net (CloudFront)", "X-Amz-Cf-Id": "nBsWBOrSHMgnaROZJK1wGCZ9PcRcSpq_oSXZNQwQ10OTZL4cimZo3g==", "X-Forwarded-For": "192.168.100.1, 192.168.1.1", "X-Forwarded-Port": "443", "X-Forwarded-Proto": "https" }, "pathParameters": { "proxy": "hello" }, "requestContext": { "accountId": "123456789012", "resourceId": "us4z18", "stage": "test", "requestId": "41b45ea3-70b5-11e6-b7bd-69b5aaebc7d9", "identity": { "cognitoIdentityPoolId": "", "accountId": "", "cognitoIdentityId": "", "caller": "", "apiKey": "", "sourceIp": "192.168.100.1", "cognitoAuthenticationType": "", "cognitoAuthenticationProvider": "", "userArn": "", "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.82 Safari/537.36 OPR/39.0.2256.48", "user": "" }, "resourcePath": "/{proxy+}", "httpMethod": "GET", "apiId": "wt6mne2s9k" }, "resource": "/{proxy+}", "httpMethod": "GET", "queryStringParameters": { "name": "me" }, "stageVariables": { "stageVarName": "stageVarValue" } } ================================================ FILE: domovoi/examples/cloudformation-event.json ================================================ { "RequestType": "Create", "ServiceToken": "arn:aws:lambda:us-east-2:123456789012:function:lambda-error-processor-primer-14ROR2T3JKU66", "ResponseURL": "https://cloudformation-custom-resource-response-useast2.s3-us-east-2.amazonaws.com/arn%3Aaws%3Acloudformation%3Aus-east-2%3A123456789012%3Astack/lambda-error-processor/1134083a-2608-1e91-9897-022501a2c456%7Cprimerinvoke%7C5d478078-13e9-baf0-464a-7ef285ecc786?AWSAccessKeyId=AKIAIOSFODNN7EXAMPLE&Expires=1555451971&Signature=28UijZePE5I4dvukKQqM%2F9Rf1o4%3D", "StackId": "arn:aws:cloudformation:us-east-2:123456789012:stack/lambda-error-processor/1134083a-2608-1e91-9897-022501a2c456", "RequestId": "5d478078-13e9-baf0-464a-7ef285ecc786", "LogicalResourceId": "primerinvoke", "ResourceType": "AWS::CloudFormation::CustomResource", "ResourceProperties": { "ServiceToken": "arn:aws:lambda:us-east-2:123456789012:function:lambda-error-processor-primer-14ROR2T3JKU66", "FunctionName": "lambda-error-processor-randomerror-ZWUC391MQAJK" } } ================================================ FILE: domovoi/examples/cloudfront-event.json ================================================ { "Records": [ { "cf": { "config": { "distributionId": "EDFDVBD6EXAMPLE" }, "request": { "clientIp": "2001:0db8:85a3:0:0:8a2e:0370:7334", "method": "GET", "uri": "/picture.jpg", "headers": { "host": [ { "key": "Host", "value": "d111111abcdef8.cloudfront.net" } ], "user-agent": [ { "key": "User-Agent", "value": "curl/7.51.0" } ] } } } } ] } ================================================ FILE: domovoi/examples/cloudtrail-event.json ================================================ { "Records": [ { "eventVersion": "1.02", "userIdentity": { "type": "Root", "principalId": "123456789012", "arn": "arn:aws:iam::123456789012:root", "accountId": "123456789012", "accessKeyId": "access-key-id", "sessionContext": { "attributes": { "mfaAuthenticated": "false", "creationDate": "2015-01-24T22:41:54Z" } } }, "eventTime": "2015-01-24T23:26:50Z", "eventSource": "sns.amazonaws.com", "eventName": "CreateTopic", "awsRegion": "us-east-2", "sourceIPAddress": "205.251.233.176", "userAgent": "console.amazonaws.com", "requestParameters": { "name": "dropmeplease" }, "responseElements": { "topicArn": "arn:aws:sns:us-east-2:123456789012:exampletopic" }, "requestID": "3fdb7834-9079-557e-8ef2-350abc03536b", "eventID": "17b46459-dada-4278-b8e2-5a4ca9ff1a9c", "eventType": "AwsApiCall", "recipientAccountId": "123456789012" }, { "eventVersion": "1.02", "userIdentity": { "type": "Root", "principalId": "123456789012", "arn": "arn:aws:iam::123456789012:root", "accountId": "123456789012", "accessKeyId": "AKIAIOSFODNN7EXAMPLE", "sessionContext": { "attributes": { "mfaAuthenticated": "false", "creationDate": "2015-01-24T22:41:54Z" } } }, "eventTime": "2015-01-24T23:27:02Z", "eventSource": "sns.amazonaws.com", "eventName": "GetTopicAttributes", "awsRegion": "us-east-2", "sourceIPAddress": "205.251.233.176", "userAgent": "console.amazonaws.com", "requestParameters": { "topicArn": "arn:aws:sns:us-east-2:123456789012:exampletopic" }, "responseElements": null, "requestID": "4a0388f7-a0af-5df9-9587-c5c98c29cbec", "eventID": "ec5bb073-8fa1-4d45-b03c-f07b9fc9ea18", "eventType": "AwsApiCall", "recipientAccountId": "123456789012" } ] } ================================================ FILE: domovoi/examples/cloudwatch-event.json ================================================ { "account": "123456789012", "region": "us-east-2", "detail": {}, "detail-type": "Scheduled Event", "source": "aws.events", "time": "2019-03-01T01:23:45Z", "id": "cdc73f9d-aea9-11e3-9d5a-835b769c0d9c", "resources": [ "arn:aws:events:us-east-1:123456789012:rule/my-schedule" ] } ================================================ FILE: domovoi/examples/codecommit-event.json ================================================ { "Records": [ { "awsRegion": "us-east-2", "codecommit": { "references": [ { "commit": "5e493c6f3067653f3d04eca608b4901eb227078", "ref": "refs/heads/master" } ] }, "eventId": "31ade2c7-f889-47c5-a937-1cf99e2790e9", "eventName": "ReferenceChanges", "eventPartNumber": 1, "eventSource": "aws:codecommit", "eventSourceARN": "arn:aws:codecommit:us-east-2:123456789012:lambda-pipeline-repo", "eventTime": "2019-03-12T20:58:25.400+0000", "eventTotalParts": 1, "eventTriggerConfigId": "0d17d6a4-efeb-46f3-b3ab-a63741badeb8", "eventTriggerName": "index.handler", "eventVersion": "1.0", "userIdentityARN": "arn:aws:iam::123456789012:user/intern" } ] } ================================================ FILE: domovoi/examples/cognito-event.json ================================================ { "datasetName": "datasetName", "eventType": "SyncTrigger", "region": "us-east-1", "identityId": "identityId", "datasetRecords": { "SampleKey2": { "newValue": "newValue2", "oldValue": "oldValue2", "op": "replace" }, "SampleKey1": { "newValue": "newValue1", "oldValue": "oldValue1", "op": "replace" } }, "identityPoolId": "identityPoolId", "version": 2 } ================================================ FILE: domovoi/examples/config-event.json ================================================ { "invokingEvent": "{\"configurationItem\":{\"configurationItemCaptureTime\":\"2016-02-17T01:36:34.043Z\",\"awsAccountId\":\"000000000000\",\"configurationItemStatus\":\"OK\",\"resourceId\":\"i-00000000\",\"ARN\":\"arn:aws:ec2:us-east-1:000000000000:instance/i-00000000\",\"awsRegion\":\"us-east-1\",\"availabilityZone\":\"us-east-1a\",\"resourceType\":\"AWS::EC2::Instance\",\"tags\":{\"Foo\":\"Bar\"},\"relationships\":[{\"resourceId\":\"eipalloc-00000000\",\"resourceType\":\"AWS::EC2::EIP\",\"name\":\"Is attached to ElasticIp\"}],\"configuration\":{\"foo\":\"bar\"}},\"messageType\":\"ConfigurationItemChangeNotification\"}", "ruleParameters": "{\"myParameterKey\":\"myParameterValue\"}", "resultToken": "myResultToken", "eventLeftScope": false, "executionRoleArn": "arn:aws:iam::012345678912:role/config-role", "configRuleArn": "arn:aws:config:us-east-1:012345678912:config-rule/config-rule-0123456", "configRuleName": "change-triggered-config-rule", "configRuleId": "config-rule-0123456", "accountId": "012345678912", "version": "1.0" } ================================================ FILE: domovoi/examples/dynamodb-event.json ================================================ { "Records": [ { "eventID": "1", "eventVersion": "1.0", "dynamodb": { "Keys": { "Id": { "N": "101" } }, "NewImage": { "Message": { "S": "New item!" }, "Id": { "N": "101" } }, "StreamViewType": "NEW_AND_OLD_IMAGES", "SequenceNumber": "111", "SizeBytes": 26 }, "awsRegion": "us-west-2", "eventName": "INSERT", "eventSourceARN": "{{eventsourcearn}}", "eventSource": "aws:dynamodb" }, { "eventID": "2", "eventVersion": "1.0", "dynamodb": { "OldImage": { "Message": { "S": "New item!" }, "Id": { "N": "101" } }, "SequenceNumber": "222", "Keys": { "Id": { "N": "101" } }, "SizeBytes": 59, "NewImage": { "Message": { "S": "This item has changed" }, "Id": { "N": "101" } }, "StreamViewType": "NEW_AND_OLD_IMAGES" }, "awsRegion": "us-west-2", "eventName": "MODIFY", "eventSourceARN": "{{sourcearn}}", "eventSource": "aws:dynamodb" } ] } ================================================ FILE: domovoi/examples/firehose-event.json ================================================ { "invocationId": "invoked123", "deliveryStreamArn": "aws:lambda:events", "region": "us-west-2", "records": [ { "data": "SGVsbG8gV29ybGQ=", "recordId": "record1", "approximateArrivalTimestamp": 1510772160000, "kinesisRecordMetadata": { "shardId": "shardId-000000000000", "partitionKey": "4d1ad2b9-24f8-4b9d-a088-76e9947c317a", "approximateArrivalTimestamp": "2012-04-23T18:25:43.511Z", "sequenceNumber": "49546986683135544286507457936321625675700192471156785154", "subsequenceNumber": "" } }, { "data": "SGVsbG8gV29ybGQ=", "recordId": "record2", "approximateArrivalTimestamp": 151077216000, "kinesisRecordMetadata": { "shardId": "shardId-000000000001", "partitionKey": "4d1ad2b9-24f8-4b9d-a088-76e9947c318a", "approximateArrivalTimestamp": "2012-04-23T19:25:43.511Z", "sequenceNumber": "49546986683135544286507457936321625675700192471156785155", "subsequenceNumber": "" } } ] } ================================================ FILE: domovoi/examples/kinesis-event.json ================================================ { "Records": [ { "kinesis": { "kinesisSchemaVersion": "1.0", "partitionKey": "1", "sequenceNumber": "49590338271490256608559692538361571095921575989136588898", "data": "SGVsbG8sIHRoaXMgaXMgYSB0ZXN0Lg==", "approximateArrivalTimestamp": 1545084650.987 }, "eventSource": "aws:kinesis", "eventVersion": "1.0", "eventID": "shardId-000000000006:49590338271490256608559692538361571095921575989136588898", "eventName": "aws:kinesis:record", "invokeIdentityArn": "arn:aws:iam::123456789012:role/lambda-role", "awsRegion": "us-east-2", "eventSourceARN": "arn:aws:kinesis:us-east-2:123456789012:stream/lambda-stream" }, { "kinesis": { "kinesisSchemaVersion": "1.0", "partitionKey": "1", "sequenceNumber": "49590338271490256608559692540925702759324208523137515618", "data": "VGhpcyBpcyBvbmx5IGEgdGVzdC4=", "approximateArrivalTimestamp": 1545084711.166 }, "eventSource": "aws:kinesis", "eventVersion": "1.0", "eventID": "shardId-000000000006:49590338271490256608559692540925702759324208523137515618", "eventName": "aws:kinesis:record", "invokeIdentityArn": "arn:aws:iam::123456789012:role/lambda-role", "awsRegion": "us-east-2", "eventSourceARN": "arn:aws:kinesis:us-east-2:123456789012:stream/lambda-stream" } ] } ================================================ FILE: domovoi/examples/logs-event.json ================================================ { "awslogs": { "data": "ewogICAgIm1lc3NhZ2VUeXBlIjogIkRBVEFfTUVTU0FHRSIsCiAgICAib3duZXIiOiAiMTIzNDU2Nzg5MDEyIiwKICAgICJsb2dHcm91cCI6I..." } } ================================================ FILE: domovoi/examples/s3-event.json ================================================ { "Records": [ { "eventVersion": "2.0", "eventSource": "aws:s3", "awsRegion": "us-west-2", "eventTime": "1970-01-01T00:00:00.000Z", "eventName": "ObjectCreated:Put", "userIdentity": { "principalId": "AIDAJDPLRKLG7UEXAMPLE" }, "requestParameters": { "sourceIPAddress": "127.0.0.1" }, "responseElements": { "x-amz-request-id": "C3D13FE58DE4C810", "x-amz-id-2": "FMyUVURIY8/IgAtTv8xRjskZQpcIZ9KG4V5Wp6S7S/JRWeUWerMUE5JgHvANOjpD" }, "s3": { "s3SchemaVersion": "1.0", "configurationId": "testConfigRule", "bucket": { "name": "sourcebucket", "ownerIdentity": { "principalId": "A3NL1KOZZKExample" }, "arn": "arn:aws:s3:::sourcebucket" }, "object": { "key": "HappyFace.jpg", "size": 1024, "eTag": "d41d8cd98f00b204e9800998ecf8427e", "versionId": "096fKKXTRTtl3on89fVO.nfljtsv6qko" } } } ] } ================================================ FILE: domovoi/examples/ses-event.json ================================================ { "Records": [ { "eventVersion": "1.0", "ses": { "mail": { "commonHeaders": { "from": [ "Jane Doe " ], "to": [ "johndoe@example.com" ], "returnPath": "janedoe@example.com", "messageId": "<0123456789example.com>", "date": "Wed, 7 Oct 2015 12:34:56 -0700", "subject": "Test Subject" }, "source": "janedoe@example.com", "timestamp": "1970-01-01T00:00:00.000Z", "destination": [ "johndoe@example.com" ], "headers": [ { "name": "Return-Path", "value": "" }, { "name": "Received", "value": "from mailer.example.com (mailer.example.com [203.0.113.1]) by inbound-smtp.us-west-2.amazonaws.com with SMTP id o3vrnil0e2ic for johndoe@example.com; Wed, 07 Oct 2015 12:34:56 +0000 (UTC)" }, { "name": "DKIM-Signature", "value": "v=1; a=rsa-sha256; c=relaxed/relaxed; d=example.com; s=example; h=mime-version:from:date:message-id:subject:to:content-type; bh=jX3F0bCAI7sIbkHyy3mLYO28ieDQz2R0P8HwQkklFj4=; b=sQwJ+LMe9RjkesGu+vqU56asvMhrLRRYrWCbV" }, { "name": "MIME-Version", "value": "1.0" }, { "name": "From", "value": "Jane Doe " }, { "name": "Date", "value": "Wed, 7 Oct 2015 12:34:56 -0700" }, { "name": "Message-ID", "value": "<0123456789example.com>" }, { "name": "Subject", "value": "Test Subject" }, { "name": "To", "value": "johndoe@example.com" }, { "name": "Content-Type", "value": "text/plain; charset=UTF-8" } ], "headersTruncated": false, "messageId": "o3vrnil0e2ic28tr" }, "receipt": { "recipients": [ "johndoe@example.com" ], "timestamp": "1970-01-01T00:00:00.000Z", "spamVerdict": { "status": "PASS" }, "dkimVerdict": { "status": "PASS" }, "processingTimeMillis": 574, "action": { "type": "Lambda", "invocationType": "Event", "functionArn": "arn:aws:lambda:us-west-2:012345678912:function:Example" }, "spfVerdict": { "status": "PASS" }, "virusVerdict": { "status": "PASS" } } }, "eventSource": "aws:ses" } ] } ================================================ FILE: domovoi/examples/sns-event.json ================================================ { "Records": [ { "EventVersion": "1.0", "EventSubscriptionArn": "arn:aws:sns:us-east-2:123456789012:test-lambda:21be56ed-a058-49f5-8c98-aedd2564c486", "EventSource": "aws:sns", "Sns": { "SignatureVersion": "1", "Timestamp": "1970-01-01T00:00:00.000Z", "Signature": "tcc6faL2yUC6dgZdmrwh1Y4cGa/ebXEkAi6RibDsvpi+tE/1+82j...65r==", "SigningCertUrl": "https://sns.us-east-2.amazonaws.com/SimpleNotificationService-ac565b8b1a6c5d002d285f9598aa1d9b.pem", "MessageId": "95df01b4-ee98-5cb9-9903-4c221d41eb5e", "Message": "Hello from SNS!", "MessageAttributes": { "Test": { "Type": "String", "Value": "TestString" }, "TestBinary": { "Type": "Binary", "Value": "TestBinary" } }, "Type": "Notification", "UnsubscribeUrl": "https://sns.us-east-2.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=arn:aws:sns:us-east-2:123456789012:test-lambda:21be56ed-a058-49f5-8c98-aedd2564c486", "TopicArn": "{{topicarn}}", "Subject": "TestInvoke" } } ] } ================================================ FILE: domovoi/examples/sqs-event.json ================================================ { "Records": [ { "messageId": "059f36b4-87a3-44ab-83d2-661975830a7d", "receiptHandle": "AQEBwJnKyrHigUMZj6rYigCgxlaS3SLy0a...", "body": "test", "attributes": { "ApproximateReceiveCount": "1", "SentTimestamp": "1545082649183", "SenderId": "AIDAIENQZJOLO23YVJ4VO", "ApproximateFirstReceiveTimestamp": "1545082649185" }, "messageAttributes": {}, "md5OfBody": "098f6bcd4621d373cade4e832627b4f6", "eventSource": "aws:sqs", "eventSourceARN": "arn:aws:sqs:us-east-2:123456789012:my-queue", "awsRegion": "us-east-2" }, { "messageId": "2e1424d4-f796-459a-8184-9c92662be6da", "receiptHandle": "AQEBzWwaftRI0KuVm4tP+/7q1rGgNqicHq...", "body": "test", "attributes": { "ApproximateReceiveCount": "1", "SentTimestamp": "1545082650636", "SenderId": "AIDAIENQZJOLO23YVJ4VO", "ApproximateFirstReceiveTimestamp": "1545082650649" }, "messageAttributes": {}, "md5OfBody": "098f6bcd4621d373cade4e832627b4f6", "eventSource": "aws:sqs", "eventSourceARN": "arn:aws:sqs:us-east-2:123456789012:my-queue", "awsRegion": "us-east-2" } ] } ================================================ FILE: domovoi/examples/state_machine_app.py ================================================ #!/usr/bin/env python3.6 import os, sys, json, time, random, signal, base64, pickle, zlib import boto3, domovoi app = domovoi.Domovoi() sfn = { "Comment": """ This is a Domovoi integrated AWS Step Functions state machine. It uses AWS Lambda as the task executor. Domovoi will replace the Resource field of all Task states with the ARN of the appropriate lambda function managed by Domovoi. See AWS documentation of the state machine language here: http://docs.aws.amazon.com/step-functions/latest/dg/concepts-amazon-states-language.html See AWS documentation of *Choice* state conditionals here: http://docs.aws.amazon.com/step-functions/latest/dg/amazon-states-language-choice-state.html The *Sleep* state can be used to wait on other events without busy-waiting. Use the following command to invoke the state machine: $ aws stepfunctions start-execution --state-machine-arn ARN --input '{"x": 1}', where ARN is displayed in the result of `domovoi deploy`. Use the following command to monitor execution of the state machine: $ aws stepfunctions get-execution-history --execution-arn EXECUTION_ARN, where EXECUTION_ARN is displayed in the result of `aws stepfunctions start-execution`. State machine input is passed directly in the `event` argument to the task handlers. There is a 32KB I/O size limit. The name of the task state that the handler was called from is available via `context.stepfunctions_task_name`. """, "StartAt": "Worker", "States": { "Worker": { "Type": "Task", "Resource": None, # This will be set by Domovoi to the Lambda ARN "Next": "Branch" }, "Branch": { "Type": "Choice", "Choices": [{ "Variable": "$.finished", "BooleanEquals": True, "Next": "Sleep" }], "Default": "Worker" }, "Sleep": { "Type": "Wait", # This is a delay step that can be set by the worker lambda to avoid busy-waiting. "SecondsPath": "$.sleep_seconds", "Next": "Finalizer" }, "Finalizer": { "Type": "Task", "Resource": None, # This will be set by Domovoi to the Lambda ARN "End": True } } } class DomovoiTimeout(Exception): pass class Worker: def run(self, x): # The run() function should save its work in progress in attributes attached to self. # If the Lambda function runs out of time, the worker instance is pickled and restored when the lambda is # restarted, but all other state is lost. self.x = getattr(self, "x", 0) + x if random.random() < 0.8: while True: # This represents some long-running task that may not be interruptible from within Python. time.sleep(9000) return dict(x=self.x, sleep_seconds=random.randrange(8)) @app.step_function_task(state_name="Worker", state_machine_definition=sfn) def do_work(event, context): def alarm_handler(signum, frame): raise DomovoiTimeout("Time to save state") signal.signal(signal.SIGALRM, alarm_handler) timeout_seconds = (context.get_remaining_time_in_millis() / 1000) - 10 context.log("Setting timeout to {}".format(timeout_seconds)) signal.alarm(timeout_seconds) if "state" in event: worker = pickle.loads(zlib.decompress(base64.b64decode(event["state"]))) else: worker = Worker() try: result = worker.run(event["x"]) except DomovoiTimeout: event.update(state=base64.b64encode(zlib.compress(pickle.dumps(worker))).decode(), finished=False) return event event.update(result, finished=True) return event @app.step_function_task(state_name="Finalizer", state_machine_definition=sfn) def finish_work(event, context): return {"result": event["x"]} ================================================ FILE: domovoi/examples/state_machine_threadpool_app.py ================================================ #!/usr/bin/env python3.6 import os, sys, json, time, random, signal, base64, pickle, zlib import boto3, domovoi app = domovoi.Domovoi() sfn = { "Comment": """ This is a Domovoi integrated AWS Step Functions state machine using a threadpool pattern. See https://github.com/kislyuk/domovoi/blob/master/domovoi/examples/state_machine_app.py for more information on this state machine. """, "StartAt": "Scatter", "States": { "Scatter": { "Type": "Task", "Resource": None, # This will be set by Domovoi to the Lambda ARN "Next": "Threadpool" }, "Threadpool": { "Type": "Parallel", "Branches": [], # This will be filled in with an array of "thread" state machines below "Next": "Finalizer" }, "Finalizer": { "Type": "Task", "Resource": None, # This will be set by Domovoi to the Lambda ARN "End": True } } } sfn_thread = { "StartAt": "Worker{t}", "States": { "Worker{t}": { "Type": "Task", "Resource": None, # This will be set by Domovoi to the Lambda ARN "Next": "Branch{t}" }, "Branch{t}": { "Type": "Choice", "Choices": [{ "Variable": "$.finished", "BooleanEquals": True, "Next": "EndThread{t}" }], "Default": "Worker{t}" }, "EndThread{t}": { "Type": "Pass", "End": True } } } num_threads = 64 class DomovoiTimeout(Exception): pass @app.step_function_task(state_name="Scatter", state_machine_definition=sfn) def scatter(event, context): # The scatter function should initialize and partition work between workers. # Each worker will receive the same event payload. You can change this using the state machine I/O processing # directives described in # http://docs.aws.amazon.com/step-functions/latest/dg/amazon-states-language-input-output-processing.html, # or distribute work out-of-band through an SQS queue or similar. # Workers can introspect their state name (which contains the "thread ID") via context.invoked_function_arn. return event class Worker: def run(self, x): # The run() function should save its work in progress in attributes attached to self. # If the Lambda function runs out of time, the worker instance is pickled and restored when the lambda is # restarted, but all other state is lost. self.x = getattr(self, "x", 0) + x if random.random() < 0.8: while True: # This represents some long-running task that may not be interruptible from within Python. time.sleep(9000) return dict(x=self.x) def do_work(event, context): def alarm_handler(signum, frame): raise DomovoiTimeout("Time to save state") signal.signal(signal.SIGALRM, alarm_handler) timeout_seconds = int(context.get_remaining_time_in_millis() / 1000) - 8 context.log("Setting timeout to {}".format(timeout_seconds)) signal.alarm(timeout_seconds) if "state" in event: worker = pickle.loads(zlib.decompress(base64.b64decode(event["state"]))) else: worker = Worker() try: result = worker.run(event["x"]) except DomovoiTimeout: event.update(state=base64.b64encode(zlib.compress(pickle.dumps(worker))).decode(), finished=False) return event event.update(result, finished=True) return event # Construct the threadpool definition by explicitly mentioning each thread in the state machine definition. for t in range(num_threads): thread = json.loads(json.dumps(sfn_thread).replace("{t}", str(t))) sfn["States"]["Threadpool"]["Branches"].append(thread) app.step_function_task(state_name="Worker{}".format(t), state_machine_definition=sfn)(do_work) @app.step_function_task(state_name="Finalizer", state_machine_definition=sfn) def finish_work(event, context): # The finalizer - the state after the "Parallel" (Threadpool) state - receives the parallel execution results as an # array. The finalizer can do things like aggregate results from the array and do any post-processing actions. return {"result": event} ================================================ FILE: domovoi/utils.py ================================================ import os, inspect import attr, boto3 import chalice.deploy.models from chalice.deploy.packager import LambdaDeploymentPackager from chalice.cli.factory import create_botocore_session import domovoi class DomovoiDeploymentPackager(LambdaDeploymentPackager): _CHALICE_LIB_DIR = "domovoilib" def _add_app_files(self, zip_fileobj, project_dir): domovoi_router = inspect.getfile(domovoi.app) if domovoi_router.endswith(".pyc"): domovoi_router = domovoi_router[:-1] zip_fileobj.write(domovoi_router, "domovoi/app.py") domovoi_init = inspect.getfile(domovoi) if domovoi_init.endswith(".pyc"): domovoi_init = domovoi_init[:-1] zip_fileobj.write(domovoi_init, "domovoi/__init__.py") chalice_router = inspect.getfile(chalice.app) if chalice_router.endswith(".pyc"): chalice_router = chalice_router[:-1] zip_fileobj.write(chalice_router, "chalice/app.py") chalice_init = inspect.getfile(chalice) if chalice_init.endswith(".pyc"): chalice_init = chalice_init[:-1] zip_fileobj.write(chalice_init, "chalice/__init__.py") zip_fileobj.write(os.path.join(project_dir, "app.py"), "app.py") self._add_chalice_lib_if_needed(project_dir, zip_fileobj) def _needs_latest_version(self, filename): return filename == 'app.py' or filename.startswith(('domovoilib/', 'domovoi/')) def create_deployment_package(self, project_dir, python_version, package_filename=None): deployment_package_filename = self.deployment_package_filename(project_dir, python_version) if os.path.exists(deployment_package_filename): self.inject_latest_app(deployment_package_filename, project_dir) return deployment_package_filename else: return LambdaDeploymentPackager.create_deployment_package(self, project_dir, python_version, package_filename=package_filename) @attr.attrs class ManagedIAMRole(chalice.deploy.models.ManagedIAMRole): def __attrs_post_init__(self): self.role_name = self.role_name.rpartition("-")[0] @attr.attrs class LambdaFunction(chalice.deploy.models.LambdaFunction): def __attrs_post_init__(self): self.function_name = self.function_name.rpartition("-")[0] def patch_chalice(): chalice.deploy.packager.LambdaDeploymentPackager = DomovoiDeploymentPackager chalice.deploy.deployer.LambdaDeploymentPackager = DomovoiDeploymentPackager chalice.deploy.models.ManagedIAMRole = ManagedIAMRole chalice.deploy.models.LambdaFunction = LambdaFunction def add_filter_config(event_config, event_handler): cfg = dict(event_config) for fltr in "prefix", "suffix": if event_handler.get(fltr): cfg.setdefault("Filter", dict(Key=dict(FilterRules=[]))) cfg["Filter"]["Key"]["FilterRules"].append(dict(Name=fltr, Value=event_handler[fltr])) return cfg def get_boto3_session(user_agent_extra, profile, debug): botocore_session = create_botocore_session(profile=profile, debug=debug) botocore_session.user_agent_extra = user_agent_extra return boto3.session.Session(botocore_session=botocore_session) class DomovoiLambdaManager: def __init__(self, function_name, awslambda_client): self.function_name = function_name self.awslambda = awslambda_client def put_event_source_mapping(self, event_source_arn, source_data, dry_run=False): event_source_mapping_args = dict(EventSourceArn=event_source_arn, FunctionName=self.function_name, Enabled=True) if "dynamodb" in event_source_arn: event_source_mapping_args.update(StartingPosition="TRIM_HORIZON") if source_data["batch_size"] is not None: event_source_mapping_args.update(BatchSize=source_data["batch_size"]) esm = None try: if not dry_run: esm = self.awslambda.create_event_source_mapping(**event_source_mapping_args) except self.awslambda.exceptions.ResourceConflictException as e: assert "already exists" in str(e) and str(e).split()[-2] == "UUID" if source_data["batch_size"] is not None: esm_uuid = str(e).split()[-1] esm = self.awslambda.get_event_source_mapping(UUID=esm_uuid) if source_data["batch_size"] != esm["BatchSize"]: esm = self.awslambda.update_event_source_mapping(UUID=esm_uuid, BatchSize=source_data["batch_size"]) return esm ================================================ FILE: scripts/domovoi ================================================ #!/usr/bin/env python from __future__ import absolute_import, division, print_function, unicode_literals import os, argparse, json, hashlib, time, copy, shutil import botocore, boto3.session import chalice, chalice.app, chalice.awsclient, chalice.deploy.packager, chalice.deploy.deployer, chalice.deploy.models from chalice.cli.factory import CLIFactory from chalice.deploy.deployer import create_default_deployer from chalice.utils import UI from chalice.compat import urlparse from chalice.constants import DEFAULT_STAGE_NAME import domovoi from domovoi.utils import patch_chalice, add_filter_config, DomovoiLambdaManager, get_boto3_session try: import pkg_resources __version__ = pkg_resources.get_distribution("domovoi").version except Exception: __version__ = "0.0.0" patch_chalice() parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawTextHelpFormatter) parser.add_argument("--stage", default=DEFAULT_STAGE_NAME) parser.add_argument("--profile") parser.add_argument("--debug", action="store_true") parser.add_argument("--dry-run", action="store_true") parser.add_argument("--version", action="version", version="domovoi {}".format(__version__)) parser.add_argument("action", choices={"deploy", "new-project"}) parser.add_argument("project_dir", nargs="?", default=os.getcwd()) args = parser.parse_args() example_app = """# This is an example entry point for an app built with Domovoi, an AWS Lambda event handler manager. # See https://github.com/kislyuk/domovoi for Domovoi's documentation. import json, boto3, domovoi app = domovoi.Domovoi() @app.scheduled_function("rate(1 minute)") def test(event, context): pass """ if args.action == "new-project": from chalice.cli import click, create_new_project_skeleton from chalice.policy import PolicyBuilder if args.project_dir == os.getcwd(): args.project_dir = click.prompt("New Domovoi project name") assert not os.path.isdir(args.project_dir) create_new_project_skeleton(args.project_dir, profile=None) with open(os.path.join(args.project_dir, "app.py"), "w") as fh: fh.write(example_app) default_iam_policy_filename = os.path.join(os.path.dirname(domovoi.__file__), "default_iam_policy.json") stage_iam_policy_filename = os.path.join(args.project_dir, ".chalice", "policy-{}.json".format(DEFAULT_STAGE_NAME)) shutil.copy(default_iam_policy_filename, stage_iam_policy_filename) parser.exit(status=0, message="New Domovoi project created in {}\n".format(args.project_dir)) boto3_session = get_boto3_session(user_agent_extra="domovoi/%s" % __version__, profile=args.profile, debug=args.debug) events = boto3_session.client("events") sns = boto3_session.resource("sns") sqs = boto3_session.resource("sqs") awslambda = boto3_session.client("lambda") s3 = boto3_session.resource("s3") sts = boto3_session.client("sts") sfn = boto3.client("stepfunctions") iam = boto3_session.resource("iam") logs = boto3_session.client("logs") dynamodb = boto3_session.resource("dynamodb") ec2 = boto3_session.resource("ec2") elbv2 = boto3_session.client("elbv2") acm = boto3_session.client("acm") route53 = boto3_session.client("route53") trust_statement = copy.deepcopy(chalice.constants.LAMBDA_TRUST_POLICY["Statement"][0]) trust_statement["Principal"] = {"Service": urlparse(sfn.meta.endpoint_url).netloc} chalice.constants.LAMBDA_TRUST_POLICY["Statement"].append(trust_statement) config = CLIFactory(args.project_dir).create_config_obj(chalice_stage_name=args.stage, autogen_policy=False) config._user_provided_params.setdefault("tags", {}) config._user_provided_params["tags"]["domovoi"] = "version={}".format(__version__) deployer = create_default_deployer(session=boto3_session._session, config=config, ui=UI(confirm=lambda *args, **kwargs: True)) function_name = '%s-%s' % (config.app_name, args.stage) function_manager = DomovoiLambdaManager(function_name=function_name, awslambda_client=awslambda) if args.dry_run: deployed_values, lambda_arn = dict(resources=[]), "arn:aws:lambda:::" else: deployed_values = deployer.deploy(config, chalice_stage_name=args.stage) for resource in deployed_values["resources"]: if resource["resource_type"] == "lambda_function": lambda_arn = resource["lambda_arn"] client = chalice.awsclient.TypedAWSClient(boto3_session._session)._client("lambda") fn_updates = dict(Description=config._chain_lookup("description") or "Domovoi event handler") dlq_arn = config._chain_lookup('dead_letter_queue_target_arn') if dlq_arn: print("Setting DLQ {} for {}".format(dlq_arn, function_name)) fn_updates.update(DeadLetterConfig=dict(TargetArn=dlq_arn)) client.update_function_configuration(FunctionName=function_name, **fn_updates) reserved_concurrent_executions = config._chain_lookup('reserved_concurrent_executions') if reserved_concurrent_executions: print("Setting concurrency reservation {} for {}".format(reserved_concurrent_executions, function_name)) client.put_function_concurrency(FunctionName=function_name, ReservedConcurrentExecutions=reserved_concurrent_executions) alb_acm_cert_dns_name = config._chain_lookup('alb_acm_cert_dns_name') domovoi_app = config.chalice_app # TODO: consider narrowing trust policy for service in ("apigateway", "events", "sns", "sqs", "s3", "logs", "elasticloadbalancing"): service_uri = service + ".amazonaws.com" policy = dict(FunctionName=lambda_arn, Principal=service_uri, Action="lambda:InvokeFunction") policy_id = "domovoi-{}".format(hashlib.md5(json.dumps(policy).encode()).hexdigest()[:8]) print("Granting {} access to invoke Lambda function {}".format(service, lambda_arn)) if not args.dry_run: try: awslambda.add_permission(StatementId=policy_id, **policy) except awslambda.exceptions.ResourceConflictException: print("Found existing permission grant statement {}, skipping".format(policy_id)) def find_acm_cert(dns_name): for page in acm.get_paginator("list_certificates").paginate(CertificateStatuses=["ISSUED"]): for cert in page["CertificateSummaryList"]: if cert["DomainName"] == dns_name: return cert raise Exception("Unable to find ACM certificate for {}".format(dns_name)) def ensure_ingress_rule(security_group, **kwargs): cidr_ip = kwargs.pop("CidrIp") for rule in security_group.ip_permissions: ip_range_matches = any(cidr_ip == ip_range["CidrIp"] for ip_range in rule["IpRanges"]) opts_match = all(rule.get(arg) == kwargs[arg] for arg in kwargs) if ip_range_matches and opts_match: break else: security_group.authorize_ingress(CidrIp=cidr_ip, **kwargs) def find_route53_zone_for_name(dns_name): best_zone = None for page in route53.get_paginator("list_hosted_zones").paginate(): for zone in page["HostedZones"]: if dns_name.endswith(zone["Name"].rstrip(".")): if best_zone is None or len(best_zone["Name"]) < len(zone["Name"]): best_zone = zone return best_zone def update_alias_dns_record(source_dns_name, target_dns_name, hosted_zone_id): zone = find_route53_zone_for_name(source_dns_name) zone_update = { "Action": "UPSERT", "ResourceRecordSet": { "Name": source_dns_name, "Type": "A", "AliasTarget": { 'HostedZoneId': hosted_zone_id, 'DNSName': target_dns_name, 'EvaluateTargetHealth': False } } } route53.change_resource_record_sets(HostedZoneId=zone["Id"], ChangeBatch=dict(Changes=[zone_update])) def register_deployed_resource(resource_type, **kwargs): deployed_values["resources"].append(dict(kwargs, resource_type=resource_type)) deployer._recorder.record_results(deployed_values, args.stage, args.project_dir) if domovoi_app.alb_targets: if not args.dry_run: if not alb_acm_cert_dns_name: raise Exception('Please set the "alb_acm_cert_dns_name" config key in your .chalice/config.json ' 'to the DNS name of a validated ACM certificate in your account') for vpc in ec2.vpcs.filter(Filters=[dict(Name="isDefault", Values=["true"])]): break else: raise Exception("A default VPC is required") security_group_name = "domovoi-{}".format(function_name) try: security_group = vpc.create_security_group(GroupName=security_group_name, Description="Automatically managed by Domovoi for Lambda ALB") except botocore.exceptions.ClientError as e: if "InvalidGroup.Duplicate" not in str(e): raise security_group = list(vpc.security_groups.filter(GroupNames=[security_group_name]))[0] ensure_ingress_rule(security_group, IpProtocol="tcp", FromPort=443, ToPort=443, CidrIp="0.0.0.0/0") for prefix, handler in domovoi_app.alb_targets.items(): res = elbv2.create_load_balancer(Name=function_name, Subnets=[subnet.id for subnet in vpc.subnets.all()], SecurityGroups=[security_group.id]) alb = res["LoadBalancers"][0] print("Using ALB", alb["LoadBalancerArn"]) res = elbv2.create_target_group(Name=function_name, TargetType="lambda") target_group = res["TargetGroups"][0] print("Using target group", target_group["TargetGroupArn"]) cert = find_acm_cert(alb_acm_cert_dns_name) print("Using ACM certificate", cert["CertificateArn"]) default_action = dict(Type="forward", TargetGroupArn=target_group["TargetGroupArn"]) res = elbv2.create_listener(LoadBalancerArn=alb["LoadBalancerArn"], Protocol="HTTPS", Port=443, Certificates=[dict(CertificateArn=cert["CertificateArn"])], DefaultActions=[default_action]) listener = res["Listeners"][0] print("Using listener", listener["ListenerArn"]) res = elbv2.register_targets(TargetGroupArn=target_group["TargetGroupArn"], Targets=[dict(Id=lambda_arn)]) print("Updating the Route53 ALIAS DNS record for {} to {}".format(alb_acm_cert_dns_name, alb["DNSName"])) update_alias_dns_record(alb_acm_cert_dns_name, alb["DNSName"], hosted_zone_id=alb["CanonicalHostedZoneId"]) for task_name, task in domovoi_app.cloudwatch_events_rules.items(): print("Scheduling", task_name, "to run on schedule", task["schedule_expression"], "pattern", task["event_pattern"]) rule_args = dict(Name=task_name) if task.get("schedule_expression"): rule_args["ScheduleExpression"] = task["schedule_expression"] if task.get("event_pattern"): rule_args["EventPattern"] = json.dumps(task["event_pattern"]) if not args.dry_run: rule_arn = events.put_rule(**rule_args)["RuleArn"] lambda_input = '{"task_name": "%s", "event": }' % task_name ixform = dict(InputPathsMap=dict(event="$"), InputTemplate=lambda_input) events.put_targets(Rule=task_name, Targets=[dict(Id=task_name, Arn=lambda_arn, InputTransformer=ixform)]) print("Scheduled CloudWatch event", rule_arn) for sns_topic, event_handler in domovoi_app.sns_subscribers.items(): print("Subscribing", event_handler, "to SNS topic", sns_topic) if not args.dry_run: topic = sns.create_topic(Name=sns_topic) subscription = topic.subscribe(Protocol="lambda", Endpoint=lambda_arn) print("Subscribed to", subscription) for sqs_queue, source_data in domovoi_app.sqs_subscribers.items(): print("Subscribing", source_data["func"], "to SQS queue", sqs_queue) queue_attributes = dict(domovoi_app.sqs_default_queue_attributes) queue_attributes.update(source_data["queue_attributes"] or {}) if not args.dry_run: queue = sqs.create_queue(QueueName=sqs_queue) queue_arn = queue.attributes["QueueArn"] queue.set_attributes(Attributes=queue_attributes) function_manager.put_event_source_mapping(event_source_arn=queue_arn, source_data=source_data, dry_run=args.dry_run) def ensure_queue(queue_name, sender_arn, event_handler): sqs_queue = sqs.create_queue(QueueName=queue_name) policy = {"Statement": [{"Action": ["SQS:SendMessage"], "Effect": "Allow", "Resource": sqs_queue.attributes["QueueArn"], "Principal": {"AWS": "*"}, "Condition": {"ArnLike": {"aws:SourceArn": sender_arn}}}]} policy["Statement"][0]["Sid"] = "domovoi-{}".format(hashlib.md5(json.dumps(policy).encode()).hexdigest()[:8]) queue_attributes = dict(domovoi_app.sqs_default_queue_attributes) queue_attributes.update(event_handler["sqs_queue_attributes"] or {}, Policy=json.dumps(policy)) sqs_queue.set_attributes(Attributes=queue_attributes) return sqs_queue def ensure_topic(topic_name, s3_bucket): sns_topic = sns.create_topic(Name=topic_name) policy = {"Statement": [{"Action": ["SNS:Publish"], "Effect": "Allow", "Resource": sns_topic.arn, "Principal": {"Service": ["s3.amazonaws.com"]}, "Condition": {"ArnLike": {"aws:SourceArn": "arn:aws:s3:*:*:" + s3_bucket}}}]} policy["Statement"][0]["Sid"] = "domovoi-{}".format(hashlib.md5(json.dumps(policy).encode()).hexdigest()[:8]) sns_topic.set_attributes(AttributeName="Policy", AttributeValue=json.dumps(policy)) return sns_topic for s3_bucket, event_handler in domovoi_app.s3_subscribers.items(): print("Subscribing", event_handler["func"], "to events in S3 bucket", s3_bucket) if args.dry_run: continue if event_handler["use_sqs"] and event_handler["use_sns"]: # An SNS-SQS bridge is the only option for subscribing multiple Lambdas to the same S3 event type with SQS topic_name = "domovoi-s3-events-{}".format(s3_bucket.replace(".", "_")) sns_topic = ensure_topic(topic_name, s3_bucket) queue_name = "domovoi-s3-events-{}-{}".format(s3_bucket.replace(".", "_"), function_name) sqs_queue = ensure_queue(queue_name, sender_arn="arn:aws:sns:*:*:" + topic_name, event_handler=event_handler) queue_arn = sqs_queue.attributes["QueueArn"] subscription = sns_topic.subscribe(Protocol="sqs", Endpoint=queue_arn) print("Subscribed", queue_arn, "to", subscription) esm = function_manager.put_event_source_mapping(event_source_arn=queue_arn, source_data=dict(batch_size=event_handler["sqs_batch_size"])) print("Created event source mapping", esm["UUID"]) topic_configuration = dict(TopicArn=sns_topic.arn, Events=event_handler["events"]) topic_configuration = add_filter_config(topic_configuration, event_handler) elif event_handler["use_sqs"]: queue_name = "domovoi-s3-events-{}-{}".format(s3_bucket.replace(".", "_"), function_name) sqs_queue = ensure_queue(queue_name, sender_arn="arn:aws:s3:*:*:" + s3_bucket, event_handler=event_handler) queue_arn = sqs_queue.attributes["QueueArn"] esm = function_manager.put_event_source_mapping(event_source_arn=queue_arn, source_data=dict(batch_size=event_handler["sqs_batch_size"])) print("Created event source mapping", esm["UUID"]) queue_configuration = dict(QueueArn=queue_arn, Events=event_handler["events"]) queue_configuration = add_filter_config(queue_configuration, event_handler) elif event_handler["use_sns"]: topic_name = "domovoi-s3-events-{}".format(s3_bucket.replace(".", "_")) sns_topic = ensure_topic(topic_name, s3_bucket) subscription = sns_topic.subscribe(Protocol="lambda", Endpoint=lambda_arn) print("Subscribed", lambda_arn, "to", subscription) topic_configuration = dict(TopicArn=sns_topic.arn, Events=event_handler["events"]) topic_configuration = add_filter_config(topic_configuration, event_handler) else: lambda_function_configuration = dict(LambdaFunctionArn=lambda_arn, Events=event_handler["events"]) lambda_function_configuration = add_filter_config(lambda_function_configuration, event_handler) for t in range(8): try: notification, last_exception = s3.Bucket(s3_bucket).Notification(), None new_config = dict(LambdaFunctionConfigurations=notification.lambda_function_configurations or [], QueueConfigurations=notification.queue_configurations or [], TopicConfigurations=notification.topic_configurations or []) if event_handler["use_sqs"] and not event_handler["use_sns"]: old_cfgs = [cfg for cfg in new_config["QueueConfigurations"] if cfg["QueueArn"] != queue_arn] new_config["QueueConfigurations"] = [queue_configuration] + old_cfgs elif event_handler["use_sns"]: old_cfgs = [cfg for cfg in new_config["TopicConfigurations"] if cfg["TopicArn"] != sns_topic.arn] new_config["TopicConfigurations"] = [topic_configuration] + old_cfgs else: old_cfgs = [cfg for cfg in new_config["LambdaFunctionConfigurations"] if cfg["LambdaFunctionArn"] != lambda_arn] new_config["LambdaFunctionConfigurations"] = [lambda_function_configuration] + old_cfgs notification.put(NotificationConfiguration=new_config) break except botocore.exceptions.ClientError as e: if "A conflicting conditional operation is currently in progress" not in str(e): raise last_exception = e print("Waiting", int(1.6**t), "seconds for concurrent operation to complete") time.sleep(1.6**t) else: raise last_exception for cwl_log_group_name, cwl_sub_filter_data in domovoi_app.cwl_sub_filters.items(): print("Subscribing", cwl_sub_filter_data, "to CloudWatch Logs filter for", cwl_log_group_name) if not args.dry_run: logs.put_subscription_filter(logGroupName=cwl_log_group_name, filterName="domovoi-cwl-filter", filterPattern=cwl_sub_filter_data["filter_pattern"], destinationArn=lambda_arn) for table_name, source_data in domovoi_app.dynamodb_event_sources.items(): print("Subscribing to DynamoDB event stream for table", table_name) stream_arn = dynamodb.Table(table_name).latest_stream_arn if not args.dry_run else "arn:aws::::" function_manager.put_event_source_mapping(event_source_arn=stream_arn, source_data=source_data, dry_run=args.dry_run) existing_aliases = [] if not args.dry_run: for page in awslambda.get_paginator('list_aliases').paginate(FunctionName=function_name): existing_aliases.extend(page["Aliases"]) state_machine = None for sfn_task_name, sfn_task in domovoi_app.sfn_tasks.items(): print("Registering step function state machine for", sfn_task_name) if state_machine is None: state_machine = sfn_task["state_machine_definition"] else: msg = "Multiple state machine definitions are not supported" assert state_machine == sfn_task["state_machine_definition"], msg lambda_alias = "domovoi-stepfunctions-task-" + sfn_task_name alias_args = dict(FunctionName=function_name, Name=lambda_alias, FunctionVersion="$LATEST", Description="Domovoi Lambda routing label for a Step Functions state machine task") all_states = domovoi.Domovoi.get_all_states(state_machine) state = all_states[sfn_task["state_name"]] if not args.dry_run: for alias in existing_aliases: if alias["Name"] == lambda_alias and alias["FunctionVersion"] == "$LATEST": break else: try: awslambda.create_alias(**alias_args) except awslambda.exceptions.ResourceConflictException: awslambda.update_alias(**alias_args) state["Resource"] = lambda_arn + ":" + lambda_alias if state_machine and not args.dry_run: iam_role_arn = config.iam_role_arn or iam.Role(function_name).arn sm_args = dict(name=function_name, definition=json.dumps(state_machine), roleArn=iam_role_arn) try: sm = sfn.create_state_machine(**sm_args) print("Created new state machine", sm["stateMachineArn"]) except botocore.exceptions.ClientError as e: for page in sfn.get_paginator("list_state_machines").paginate(): for sm in page["stateMachines"]: if sm["name"] == function_name: break if sm["name"] != function_name: raise e sm = sfn.describe_state_machine(stateMachineArn=sm["stateMachineArn"]) sm_args.clear() if json.loads(sm["definition"]) != state_machine: sm_args["definition"] = json.dumps(state_machine) if sm["roleArn"] != iam_role_arn: sm_args["roleArn"] = iam_role_arn if sm_args: print("Updating state machine", sm["stateMachineArn"]) sfn.update_state_machine(stateMachineArn=sm["stateMachineArn"], **sm_args) else: print("No changes required to existing state machine", sm["stateMachineArn"]) register_deployed_resource("state_machine", name=function_name, arn=sm["stateMachineArn"]) print("State machine:", sm["stateMachineArn"]) if args.dry_run: print("Dry run successful") ================================================ FILE: setup.cfg ================================================ [bdist_wheel] universal=1 [flake8] max-line-length=120 ignore: E401, F401 ================================================ FILE: setup.py ================================================ #!/usr/bin/env python import glob from setuptools import setup, find_packages setup( name="domovoi", version="2.0.2", url='https://github.com/kislyuk/domovoi', license='Apache Software License', author='Andrey Kislyuk', author_email='kislyuk@gmail.com', description='AWS Lambda event handler manager', long_description=open('README.rst').read(), install_requires=[ 'boto3 >= 1.7.19, < 2', 'chalice >= 1.3.0, < 2' ], extras_require={ ':python_version == "2.7"': ['enum34 >= 1.1.6, < 2'] }, packages=find_packages(exclude=['test']), scripts=glob.glob('scripts/*'), platforms=['MacOS X', 'Posix'], package_data={'domovoi': ['*.json']}, zip_safe=False, include_package_data=True, test_suite='test', classifiers=[ 'Intended Audience :: Developers', 'License :: OSI Approved :: Apache Software License', 'Operating System :: MacOS :: MacOS X', 'Operating System :: POSIX', 'Programming Language :: Python', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Topic :: Software Development :: Libraries :: Python Modules' ] ) ================================================ FILE: test/test.py ================================================ #!/usr/bin/env python # coding: utf-8 from __future__ import absolute_import, division, print_function, unicode_literals import os, sys, unittest, tempfile, json, subprocess, shutil, textwrap sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) from domovoi import Domovoi # noqa class TestDomovoi(unittest.TestCase): def test_basic_statements(self): state_machine = { "StartAt": "Worker", "States": { "Worker": { "Type": "Task", "Resource": None, "End": True } } } subprocess.check_call(["domovoi", "new-project", "testproject"]) readme_filename = os.path.join(os.path.dirname(__file__), "..", "README.rst") with open(readme_filename) as readme_fh, open("testproject/app.py", "w") as app_fh: for line in readme_fh.readlines(): if line.strip() == ".. code-block:: python": app_fh.write("# Domovoi test\nstate_machine = {}\n".format(state_machine)) elif line.strip() == "Installation": break elif app_fh.tell(): app_fh.write(line[4:]) subprocess.check_call(["domovoi", "--dry-run", "deploy"], cwd="testproject") def test_state_machine_examples(self): subprocess.check_call(["domovoi", "new-project", "testproject-sfn"]) shutil.copy(os.path.join(os.path.dirname(__file__), "..", "domovoi", "examples", "state_machine_app.py"), os.path.join("testproject-sfn", "app.py")) subprocess.check_call(["domovoi", "--dry-run", "deploy"], cwd="testproject-sfn") def test_state_machine_registration(self): sm_app = """ import json, boto3, domovoi app = domovoi.Domovoi() def handler(event, context): pass state_machine = { "StartAt": "Worker", "States": { "Worker": { "Type": "Task", "Resource": handler, "End": True } } } app.register_state_machine(state_machine) """ subprocess.check_call(["domovoi", "new-project", "testproject2"]) with open("testproject2/app.py", "w") as app_fh: app_fh.write(textwrap.dedent(sm_app)) subprocess.check_call(["domovoi", "--dry-run", "deploy"], cwd="testproject2") if __name__ == '__main__': unittest.main()