[ { "path": ".gitignore", "content": "# Reminder:\n# - A leading slash means the pattern is anchored at the root.\n# - No leading slash means the pattern matches at any depth.\n\n# Python files\n*.pyc\n__pycache__/\n.tox/\n*.egg-info/\n/build/\n/dist/\n/.eggs/\n\n# Sphinx documentation\n/docs/_build/\n\n# IDE project files\n/.pydevproject\n\n# vim python-mode plugin\n/.ropeproject\n\n# IntelliJ IDEA / PyCharm project files\n/.idea\n/*.iml\n\n# JS/node/npm/web dev files\nnode_modules\nnpm-debug.log\n\n# OS X metadata files\n.DS_Store\n" }, { "path": ".travis.yml", "content": "language: python\nsudo: required\ndist: bionic\ncache: pip\n\npython:\n - 2.7\n - 3.6\n - 3.7\n - 3.8\n\nenv:\n global:\n - AWS_DEFAULT_REGION=us-east-1\n\nbefore_install:\n - pip install --quiet coverage flake8 pyyaml\n\ninstall:\n - make install\n\nscript:\n - make test\n\nafter_success:\n - bash <(curl -s https://codecov.io/bash)\n\nsudo: false\n" }, { "path": "Changes.rst", "content": "Changes for v2.0.2 (2019-08-26)\n===============================\n\n- Record step function ARN in deployed values. Fixes #22\n\n- Return enclosed function in ALB decorator\n\n- Minor documentation improvements\n\nChanges for v2.0.1 (2019-04-15)\n===============================\n\n- Manage route53 records for ALB\n\nChanges for v2.0.0 (2019-04-15)\n===============================\n\n- Initial support for ALB\n\nChanges for v1.9.0 (2019-01-10)\n===============================\n\n- Condense SNS/SQS names for brevity\n\nChanges for v1.8.3 (2018-10-11)\n===============================\n\n- Add app.state_machine.start_named_execution(name, \\*args)\n\nChanges for v1.8.2 (2018-09-14)\n===============================\n\nRetain S3-SQS mux ability\n\nChanges for v1.8.1 (2018-09-14)\n===============================\n\nBuild SNS-SQS bridge to mux lambdas onto S3 event types\n\nChanges for v1.8.0 (2018-08-29)\n===============================\n\nFix essential logging, take 4\n\nChanges for v1.7.10 (2018-08-29)\n================================\n\nFix essential logging, take 3\n\nChanges for v1.7.9 (2018-08-29)\n===============================\n\nFix essential logging, take 2\n\nChanges for v1.7.8 (2018-08-29)\n===============================\n\nFix essential logging\n\nChanges for v1.7.7 (2018-08-29)\n===============================\n\n- Enable refresh deployment in packager\n\n- Use Python logging library instead of Lambda context.log (#11, thanks\n to @irgeek)\n\nChanges for v1.7.6 (2018-08-28)\n===============================\n\n- Remove debug statement\n\nChanges for v1.7.5 (2018-08-14)\n===============================\n\n- Grant SQS permissions to invoke lambda\n\nChanges for v1.7.4 (2018-07-10)\n===============================\n\n- Allow queue attributes to be set in SQS\n\nChanges for v1.7.3 (2018-07-06)\n===============================\n\n- SQS S3 event envelope support\n\nChanges for v1.7.2 (2018-07-05)\n===============================\n\n- Support state machine introspection and SQS-SFN springboard\n\n- Update docs for SQS\n\nChanges for v1.7.1 (2018-07-05)\n===============================\n\n- Don’t assume eventSource exists\n\nChanges for v1.7.0 (2018-07-05)\n===============================\n\n- Add support for SQS event sources\n\nChanges for v1.6.1 (2018-06-06)\n===============================\n\n- Set function description from “description” config key\n\n- Fix version reporting and provide it in published Lambda tag\n\nChanges for v1.6.0 (2018-06-05)\n===============================\n\n- Domovoi is now compatible with Chalice 1.2+. This should be a\n backwards compatible change. However, Chalice underwent a complete\n deployment system rewrite in version 1.2, so deployment state or\n other aspects of your app’s deployment may be affected. You may need\n to clear the deployment state of your app.\n\nChanges for v1.5.8 (2018-05-04)\n===============================\n\n- Allow customizable rule name for @scheduled_function (fixes #9)\n\nChanges for v1.5.7 (2018-05-03)\n===============================\n\n- Ensure domovoi errors when there is no handler\n\nChanges for v1.5.6 (2018-04-09)\n===============================\n\n- Allow configurable concurrency reservation\n\nChanges for v1.5.5 (2018-03-16)\n===============================\n\n- Add support for new-project and update docs\n\nChanges for v1.5.4 (2018-03-03)\n===============================\n\n- Skip updating event source mapping when diff is null\n\nChanges for v1.5.3 (2018-02-21)\n===============================\n\n- Avoid triggering Lambda API rate limits when managing state aliases\n\nChanges for v1.5.2 (2018-02-05)\n===============================\n\n- Ensure SNS topic names can represent all DNS-compliant S3 bucket\n names. Fixes #5\n\nChanges for v1.5.1 (2018-02-01)\n===============================\n\n- Fix routing of domovoi dynamodb handlers\n\nChanges for v1.5.0 (2018-02-01)\n===============================\n\n- Add DynamoDB streams support\n\n- Bypass prompt when writing IAM policy for the first time\n\nChanges for v1.4.5 (2017-12-12)\n===============================\n\n- Only call step\\_function\\_task if the state has a Resource field\n that's callable\n\nChanges for v1.4.4 (2017-12-11)\n===============================\n\n- Allow state machine registration; pass state name in context\n\n- Deconflict concurrent S3 notification config operations\n\nChanges for v1.4.3 (2017-11-29)\n===============================\n\n- Improve SM updates: use update\\_state\\_machine\n\nChanges for v1.4.2 (2017-11-14)\n===============================\n\nAccommodate eventual consistency in SM update loop\n\nChanges for v1.4.1 (2017-11-14)\n===============================\n\n- Add statement to debug SM deploy loop crash\n\nChanges for v1.4.0 (2017-11-09)\n===============================\n\n- Add support for CloudWatch Logs subscription filter events\n\n- Expand docs for step function / state machine examples\n\nChanges for v1.3.2 (2017-11-07)\n===============================\n\n- Support nested states\n\nChanges for v1.3.1 (2017-10-30)\n===============================\n\n- Key state machine tasks by state name, not function name\n\n- Parameterize sfn trust statement by region\n\nChanges for v1.3.0 (2017-10-26)\n===============================\n\n- Add step functions support\n\nChanges for v1.2.6 (2017-08-26)\n===============================\n\n- Monkey-patch chalice to avoid dependency wheel management bug\n\n- Use more intuitive errors when handler not found\n\nChanges for v1.2.5 (2017-08-17)\n===============================\n\nAvoid running privileged op on update\n\nChanges for v1.2.4 (2017-08-17)\n===============================\n\n- Chalice 1.0 compat, part 3\n\nChanges for v1.2.3 (2017-08-17)\n===============================\n\n- Chalice 1.0 compat, part 2\n\nChanges for v1.2.2 (2017-08-17)\n===============================\n\nChalice 1.0 compatibility fixes\n\nChanges for v1.2.1 (2017-07-14)\n===============================\n\n- Simplify DLQ handling; add docs for DLQ\n\nChanges for v1.2.0 (2017-07-14)\n===============================\n\n- Support DLQ lambda config\n\nChanges for v1.1.1 (2017-07-05)\n===============================\n\n- Parameterize stage name, part 2\n\nChanges for v1.1.0 (2017-07-05)\n===============================\n\n- Parameterize stage name\n\nChanges for v1.0.9 (2017-06-24)\n===============================\n\n- Forward S3 notifications through SNS by default\n\nChanges for v1.0.8 (2017-06-24)\n===============================\n\n- Don't clobber existing S3 bucket notifications\n\nChanges for v1.0.7 (2017-06-22)\n===============================\n\n- Pass through configure\\_logs\n\n- Test improvements\n\nChanges for v1.0.6 (2017-06-15)\n===============================\n\nFix error in release\n\nChanges for v1.0.5 (2017-06-15)\n===============================\n\nEnable idempotent Lambda permission grants\n\nChanges for v1.0.4 (2017-06-09)\n===============================\n\n- Hardcode no autogen policy\n\nChanges for v1.0.3 (2017-06-08)\n===============================\n\n- Ensure S3 bucket notifications work without filters specified\n\nChanges for v1.0.2 (2017-06-01)\n===============================\n\n- Fix dispatching of S3 events\n\n- Fixes to deploy procedure\n\nChanges for v1.0.1 (2017-06-01)\n===============================\n\n- Fix event subscriptions\n\nChanges for v1.0.0 (2017-05-28)\n===============================\n\n- Update to be compatible with Chalice 0.8 and Python 3.6\n\n\n\n\nChanges for v0.0.3 (2016-12-19)\n===============================\n\n- Autogenerate IAM policy\n\n- Release automation\n\nVersion 0.0.1 (2016-12-14)\n--------------------------\n- Initial release.\n" }, { "path": "LICENSE", "content": "Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n APPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"{}\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\n Copyright {yyyy} {name of copyright owner}\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n" }, { "path": "MANIFEST.in", "content": "include *.rst\ninclude test/*\ninclude domovoi/*.json\ninclude domovoi/examples/*.json\n" }, { "path": "Makefile", "content": "SHELL=/bin/bash\n\nlint:\n\t./setup.py flake8\n\tflake8 scripts/*\n\ntest: lint\n\t-rm -rf testproject testproject2 testproject-sfn\n\tpython ./test/test.py -v\n\ninit_docs:\n\tcd docs; sphinx-quickstart\n\ndocs:\n\t$(MAKE) -C docs html\n\ninstall:\n\t-rm -rf dist\n\tpython setup.py bdist_wheel\n\tpip install --upgrade dist/*.whl\n\n.PHONY: test release docs\n\ninclude common.mk\n" }, { "path": "README.rst", "content": "Domovoi: AWS Lambda event handler manager\n=========================================\n\n*Domovoi* is an extension to `AWS Chalice `_ to handle `AWS Lambda\n`_ `event sources\n`_ other\nthan HTTP requests through API Gateway. Domovoi lets you easily configure and deploy a Lambda function to serve HTTP\nrequests through `ALB `_,\non a schedule, or in response to a variety of events like an `SNS `_\nor `SQS `_ message, S3 event, or custom\n`state machine `_ transition:\n\n.. code-block:: python\n\n import json, boto3, domovoi\n\n app = domovoi.Domovoi()\n\n # Compared to API Gateway, ALB increases the response timeout from 30s to 900s, but reduces the payload\n # limit from 10MB to 1MB. It also does not try to negotiate on the Accept/Content-Type headers.\n @app.alb_target()\n def serve(event, context):\n return dict(statusCode=200,\n statusDescription=\"200 OK\",\n isBase64Encoded=False,\n headers={\"Content-Type\": \"application/json\"},\n body=json.dumps({\"hello\": \"world\"}))\n\n @app.scheduled_function(\"cron(0 18 ? * MON-FRI *)\")\n def foo(event, context):\n context.log(\"foo invoked at 06:00pm (UTC) every Mon-Fri\")\n return dict(result=True)\n\n @app.scheduled_function(\"rate(1 minute)\")\n def bar(event, context):\n context.log(\"bar invoked once a minute\")\n boto3.resource(\"sns\").create_topic(Name=\"bartender\").publish(Message=json.dumps({\"beer\": 1}))\n return dict(result=\"Work work work\")\n\n @app.sns_topic_subscriber(\"bartender\")\n def tend(event, context):\n message = json.loads(event[\"Records\"][0][\"Sns\"][\"Message\"])\n context.log(dict(beer=\"Quadrupel\", quantity=message[\"beer\"]))\n\n # SQS messages are deleted upon successful exit, requeued otherwise.\n # See https://docs.aws.amazon.com/lambda/latest/dg/with-sqs.html\n @app.sqs_queue_subscriber(\"my_queue\", batch_size=64)\n def process_queue_messages(event, context):\n message = json.loads(event[\"Records\"][0][\"body\"])\n message_attributes = event[\"Records\"][0][\"messageAttributes\"]\n # You can colocate a state machine definition with an SQS handler to launch a SFN driven lambda from SQS.\n return app.state_machine.start_execution(**message)[\"executionArn\"]\n\n @app.cloudwatch_event_handler(source=[\"aws.ecs\"])\n def monitor_ecs_events(event, context):\n message = json.loads(event[\"Records\"][0][\"Sns\"][\"Message\"])\n context.log(\"Got an event from ECS: {}\".format(message))\n\n @app.s3_event_handler(bucket=\"myS3bucket\", events=[\"s3:ObjectCreated:*\"], prefix=\"foo\", suffix=\".bar\")\n def monitor_s3(event, context):\n context.log(\"Got an event from S3: {}\".format(event))\n\n # Set use_sns=False, use_sqs=False to subscribe your Lambda directly to S3 events without forwarding them through an SNS-SQS bridge.\n # That approach has fewer moving parts, but you can only subscribe one Lambda function to events in a given S3 bucket.\n @app.s3_event_handler(bucket=\"myS3bucket\", events=[\"s3:ObjectCreated:*\"], prefix=\"foo\", suffix=\".bar\", use_sns=False, use_sqs=False)\n def monitor_s3(event, context):\n context.log(\"Got an event from S3: {}\".format(event))\n\n # DynamoDB event format: https://docs.aws.amazon.com/lambda/latest/dg/with-ddb.html\n @app.dynamodb_stream_handler(table_name=\"MyDynamoTable\", batch_size=200)\n def handle_dynamodb_stream(event, context):\n context.log(\"Got {} events from DynamoDB\".format(len(event[\"Records\"])))\n context.log(\"First event: {}\".format(event[\"Records\"][0][\"dynamodb\"]))\n\n # Use the following command to log a CloudWatch Logs message that will trigger this handler:\n # python -c'import watchtower as w, logging as l; L=l.getLogger(); L.addHandler(w.CloudWatchLogHandler()); L.error(dict(x=8))'\n # See http://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/FilterAndPatternSyntax.html for the filter pattern syntax\n @app.cloudwatch_logs_sub_filter_handler(log_group_name=\"watchtower\", filter_pattern=\"{$.x = 8}\")\n def monitor_cloudwatch_logs(event, context):\n print(\"Got a CWL subscription filter event:\", event)\n\n # See http://docs.aws.amazon.com/step-functions/latest/dg/concepts-amazon-states-language.html\n # See the \"AWS Step Functions state machines\" section below for a complete example of setting up a state machine.\n @app.step_function_task(state_name=\"Worker\", state_machine_definition=state_machine)\n def worker(event, context):\n return {\"result\": event[\"input\"] + 1, \"my_state\": context.stepfunctions_task_name}\n\nInstallation\n------------\n::\n\n pip install domovoi\n\nUsage\n-----\nFirst-time setup::\n\n domovoi new-project\n\n* Edit the Domovoi app entry point in ``app.py`` using examples above.\n* Edit the IAM policy for your Lambda function in ``my_project/.chalice/policy-dev.json`` to add any permissions it\n needs.\n* Deploy the event handlers::\n\n domovoi deploy\n\nTo stage files into the deployment package, use a ``domovoilib`` directory in your project where you would use\n``chalicelib`` in Chalice. For example, ``my_project/domovoilib/rds_cert.pem`` becomes ``/var/task/domovoilib/rds_cert.pem``\nwith your function executing in ``/var/task/app.py`` with ``/var/task`` as the working directory. See the\n`Chalice docs `_ for more information on how to set up Chalice configuration.\n\nSupported event types\n~~~~~~~~~~~~~~~~~~~~~\nSee `Supported Event Sources `_ for an\noverview of event sources that can be used to trigger Lambda functions. Domovoi supports the following event sources:\n\n* `ALB HTTPS requests `_\n* `SNS subscriptions `_\n* `SQS queues `_\n* CloudWatch Events rule targets, including\n `CloudWatch Scheduled Events `_ (see\n `CloudWatch Events Event Examples `_ for a\n list of event types supported by CloudWatch Events)\n* `S3 events `_\n* AWS Step Functions state machine tasks\n* `CloudWatch Logs filter subscriptions `_\n* `DynamoDB stream events `_\n\nPossible future event sources to support:\n\n* Kinesis stream events\n* SES (email) events\n\nAWS Step Functions state machines\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\nDomovoi supports AWS Lambda integration with `AWS Step Functions\n`_. Step Functions state machines can be started using the\n`StartExecution `_ method or the\n`API Gateway Step Functions integration\n`_.\n\nSee the `domovoi/examples `_ directory for examples of Domovoi ``app.py`` apps using a state machine,\nincluding a loop that restarts the Lambda when it's about to hit its execution time limit, and a threadpool pattern that\ndivides work between multiple Lambdas.\n\nWhen creating a Step Functions State Machine driven Domovoi daemon Lambda, the State Machine assumes the same IAM role as\nthe Lambda itself. To allow the State Machine to invoke the Lambda, edit the IAM policy (under your app directory, in\n``.chalice/policy-dev.json``) to include a statement allowing the \"lambda:InvokeFunction\" action on all resources, or on the\nARN of the Lambda itself.\n\nConfiguration\n~~~~~~~~~~~~~\n\nALB\n^^^\nTo use your Lambda as an ALB target with the ``@alb_target(prefix=\"...\")`` decorator, you should pre-configure the\nfollowing resources in your AWS account:\n\n* A Route 53 hosted DNS zone such as ``example.com.``, with a domain (``example.com``) pointing to it\n* An active (verified/issued) ACM certificate for a DNS name within your DNS zone, such as ``domovoi.example.com``\n\nAfter configuring these, set the ``alb_acm_cert_dns_name`` configuration key in the file ``.chalice/config.json`` to\nyour DNS name. For example::\n\n {\n \"app_name\": \"my_app\",\n ...\n \"alb_acm_cert_dns_name\": \"domovoi.example.com\"\n }\n\nDomovoi will automatically create, manage, and link the ALB and DNS record in your Route 53 zone.\n\nDead Letter Queues\n^^^^^^^^^^^^^^^^^^\nTo enable your Lambda function to forward failed invocation notifications to `dead letter queues\n`_, set the configuration key ``dead_letter_queue_target_arn`` in\nthe file ``.chalice/config.json`` to the target DLQ ARN. For example::\n\n {\n \"app_name\": \"my_app\",\n ...\n \"dead_letter_queue_target_arn\": \"arn:aws:sns:us-east-1:123456789012:my-dlq\"\n }\n\nYou may need to update your Lambda IAM policy (``.chalice/policy-dev.json``) to give your Lambda access to SNS or SQS.\n\nConcurrency Reservations\n^^^^^^^^^^^^^^^^^^^^^^^^\nFor high volume Lambda invocations in accounts with multiple Lambdas, you may need to set `per-function concurrency\nlimits `_ to partition the overall concurrency\nquota and prevent one set of Lambdas from overloading another. In Domovoi, you can do so by setting the configuration\nkey ``reserved_concurrent_executions`` in the file ``.chalice/config.json`` to the desired concurrency reservation. For\nexample::\n\n {\n \"app_name\": \"my_app\",\n ...\n \"reserved_concurrent_executions\": 500\n }\n\n\nLinks\n-----\n* `Project home page (GitHub) `_\n* `Documentation (Read the Docs) `_\n* `Package distribution (PyPI) `_\n* `Change log `_\n\nBugs\n~~~~\nPlease report bugs, issues, feature requests, etc. on `GitHub `_.\n\nLicense\n-------\nLicensed under the terms of the `Apache License, Version 2.0 `_.\n\n.. image:: https://travis-ci.org/kislyuk/domovoi.png\n :target: https://travis-ci.org/kislyuk/domovoi\n.. image:: https://codecov.io/github/kislyuk/domovoi/coverage.svg?branch=master\n :target: https://codecov.io/github/kislyuk/domovoi?branch=master\n.. image:: https://img.shields.io/pypi/v/domovoi.svg\n :target: https://pypi.python.org/pypi/domovoi\n.. image:: https://img.shields.io/pypi/l/domovoi.svg\n :target: https://pypi.python.org/pypi/domovoi\n.. image:: https://readthedocs.org/projects/domovoi/badge/?version=latest\n :target: https://domovoi.readthedocs.org/\n" }, { "path": "common.mk", "content": "SHELL=/bin/bash -eo pipefail\n\nrelease_major:\n\t$(eval export TAG=$(shell git describe --tags --match 'v*.*.*' | perl -ne '/^v(\\d+)\\.(\\d+)\\.(\\d+)/; print \"v@{[$$1+1]}.0.0\"'))\n\t$(MAKE) release\n\nrelease_minor:\n\t$(eval export TAG=$(shell git describe --tags --match 'v*.*.*' | perl -ne '/^v(\\d+)\\.(\\d+)\\.(\\d+)/; print \"v$$1.@{[$$2+1]}.0\"'))\n\t$(MAKE) release\n\nrelease_patch:\n\t$(eval export TAG=$(shell git describe --tags --match 'v*.*.*' | perl -ne '/^v(\\d+)\\.(\\d+)\\.(\\d+)/; print \"v$$1.$$2.@{[$$3+1]}\"'))\n\t$(MAKE) release\n\nrelease:\n\t@if [[ -z $$TAG ]]; then echo \"Use release_{major,minor,patch}\"; exit 1; fi\n\t@if ! type -P pandoc; then echo \"Please install pandoc\"; exit 1; fi\n\t@if ! type -P sponge; then echo \"Please install moreutils\"; exit 1; fi\n\t@if ! type -P http; then echo \"Please install httpie\"; exit 1; fi\n\t@if ! type -P twine; then echo \"Please install twine\"; exit 1; fi\n\t$(eval REMOTE=$(shell git remote get-url origin | perl -ne '/([^\\/\\:]+\\/.+?)(\\.git)?$$/; print $$1'))\n\t$(eval GIT_USER=$(shell git config --get user.email))\n\t$(eval GH_AUTH=$(shell if grep -q '@github.com' ~/.git-credentials; then echo $$(grep '@github.com' ~/.git-credentials | python3 -c 'import sys, urllib.parse as p; print(p.urlparse(sys.stdin.read()).netloc.split(\"@\")[0])'); else echo $(GIT_USER); fi))\n\t$(eval RELEASES_API=https://api.github.com/repos/${REMOTE}/releases)\n\t$(eval UPLOADS_API=https://uploads.github.com/repos/${REMOTE}/releases)\n\tgit pull\n\tgit clean -x --force $$(python setup.py --name)\n\tsed -i -e \"s/version=\\([\\'\\\"]\\)[0-9]*\\.[0-9]*\\.[0-9]*/version=\\1$${TAG:1}/\" setup.py\n\tgit add setup.py\n\tTAG_MSG=$$(mktemp); \\\n\t echo \"# Changes for ${TAG} ($$(date +%Y-%m-%d))\" > $$TAG_MSG; \\\n\t git log --pretty=format:%s $$(git describe --abbrev=0)..HEAD >> $$TAG_MSG; \\\n\t $${EDITOR:-emacs} $$TAG_MSG; \\\n\t if [[ -f Changes.md ]]; then cat $$TAG_MSG <(echo) Changes.md | sponge Changes.md; git add Changes.md; fi; \\\n\t if [[ -f Changes.rst ]]; then cat <(pandoc --from markdown --to rst $$TAG_MSG) <(echo) Changes.rst | sponge Changes.rst; git add Changes.rst; fi; \\\n\t git commit -m ${TAG}; \\\n\t git tag --sign --annotate --file $$TAG_MSG ${TAG}\n\tgit push --follow-tags\n\thttp --auth ${GH_AUTH} ${RELEASES_API} tag_name=${TAG} name=${TAG} \\\n\t body=\"$$(git tag --list ${TAG} -n99 | perl -pe 's/^\\S+\\s*// if $$. == 1' | sed 's/^\\s\\s\\s\\s//')\"\n\t$(MAKE) install\n\thttp --auth ${GH_AUTH} POST ${UPLOADS_API}/$$(http --auth ${GH_AUTH} ${RELEASES_API}/latest | jq .id)/assets \\\n\t name==$$(basename dist/*.whl) label==\"Python Wheel\" < dist/*.whl\n\t$(MAKE) pypi_release\n\npypi_release:\n\tpython setup.py sdist bdist_wheel\n\ttwine upload dist/*.tar.gz dist/*.whl --sign --verbose\n\n.PHONY: release\n" }, { "path": "docs/Makefile", "content": "# Makefile for Sphinx documentation\n#\n\n# You can set these variables from the command line.\nSPHINXOPTS =\nSPHINXBUILD = sphinx-build\nPAPER =\nBUILDDIR = _build\n\n# Internal variables.\nPAPEROPT_a4 = -D latex_paper_size=a4\nPAPEROPT_letter = -D latex_paper_size=letter\nALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .\n# the i18n builder cannot share the environment and doctrees with the others\nI18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .\n\n.PHONY: help\nhelp:\n\t@echo \"Please use \\`make ' where is one of\"\n\t@echo \" html to make standalone HTML files\"\n\t@echo \" dirhtml to make HTML files named index.html in directories\"\n\t@echo \" singlehtml to make a single large HTML file\"\n\t@echo \" pickle to make pickle files\"\n\t@echo \" json to make JSON files\"\n\t@echo \" htmlhelp to make HTML files and a HTML help project\"\n\t@echo \" qthelp to make HTML files and a qthelp project\"\n\t@echo \" applehelp to make an Apple Help Book\"\n\t@echo \" devhelp to make HTML files and a Devhelp project\"\n\t@echo \" epub to make an epub\"\n\t@echo \" epub3 to make an epub3\"\n\t@echo \" latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter\"\n\t@echo \" latexpdf to make LaTeX files and run them through pdflatex\"\n\t@echo \" latexpdfja to make LaTeX files and run them through platex/dvipdfmx\"\n\t@echo \" text to make text files\"\n\t@echo \" man to make manual pages\"\n\t@echo \" texinfo to make Texinfo files\"\n\t@echo \" info to make Texinfo files and run them through makeinfo\"\n\t@echo \" gettext to make PO message catalogs\"\n\t@echo \" changes to make an overview of all changed/added/deprecated items\"\n\t@echo \" xml to make Docutils-native XML files\"\n\t@echo \" pseudoxml to make pseudoxml-XML files for display purposes\"\n\t@echo \" linkcheck to check all external links for integrity\"\n\t@echo \" doctest to run all doctests embedded in the documentation (if enabled)\"\n\t@echo \" coverage to run coverage check of the documentation (if enabled)\"\n\t@echo \" dummy to check syntax errors of document sources\"\n\n.PHONY: clean\nclean:\n\trm -rf $(BUILDDIR)/*\n\n.PHONY: html\nhtml:\n\t$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html\n\t@echo\n\t@echo \"Build finished. The HTML pages are in $(BUILDDIR)/html.\"\n\n.PHONY: dirhtml\ndirhtml:\n\t$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml\n\t@echo\n\t@echo \"Build finished. The HTML pages are in $(BUILDDIR)/dirhtml.\"\n\n.PHONY: singlehtml\nsinglehtml:\n\t$(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml\n\t@echo\n\t@echo \"Build finished. The HTML page is in $(BUILDDIR)/singlehtml.\"\n\n.PHONY: pickle\npickle:\n\t$(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle\n\t@echo\n\t@echo \"Build finished; now you can process the pickle files.\"\n\n.PHONY: json\njson:\n\t$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json\n\t@echo\n\t@echo \"Build finished; now you can process the JSON files.\"\n\n.PHONY: htmlhelp\nhtmlhelp:\n\t$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp\n\t@echo\n\t@echo \"Build finished; now you can run HTML Help Workshop with the\" \\\n\t \".hhp project file in $(BUILDDIR)/htmlhelp.\"\n\n.PHONY: qthelp\nqthelp:\n\t$(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp\n\t@echo\n\t@echo \"Build finished; now you can run \"qcollectiongenerator\" with the\" \\\n\t \".qhcp project file in $(BUILDDIR)/qthelp, like this:\"\n\t@echo \"# qcollectiongenerator $(BUILDDIR)/qthelp/Domovoi.qhcp\"\n\t@echo \"To view the help file:\"\n\t@echo \"# assistant -collectionFile $(BUILDDIR)/qthelp/Domovoi.qhc\"\n\n.PHONY: applehelp\napplehelp:\n\t$(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp\n\t@echo\n\t@echo \"Build finished. The help book is in $(BUILDDIR)/applehelp.\"\n\t@echo \"N.B. You won't be able to view it unless you put it in\" \\\n\t \"~/Library/Documentation/Help or install it in your application\" \\\n\t \"bundle.\"\n\n.PHONY: devhelp\ndevhelp:\n\t$(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp\n\t@echo\n\t@echo \"Build finished.\"\n\t@echo \"To view the help file:\"\n\t@echo \"# mkdir -p $$HOME/.local/share/devhelp/Domovoi\"\n\t@echo \"# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/Domovoi\"\n\t@echo \"# devhelp\"\n\n.PHONY: epub\nepub:\n\t$(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub\n\t@echo\n\t@echo \"Build finished. The epub file is in $(BUILDDIR)/epub.\"\n\n.PHONY: epub3\nepub3:\n\t$(SPHINXBUILD) -b epub3 $(ALLSPHINXOPTS) $(BUILDDIR)/epub3\n\t@echo\n\t@echo \"Build finished. The epub3 file is in $(BUILDDIR)/epub3.\"\n\n.PHONY: latex\nlatex:\n\t$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex\n\t@echo\n\t@echo \"Build finished; the LaTeX files are in $(BUILDDIR)/latex.\"\n\t@echo \"Run \\`make' in that directory to run these through (pdf)latex\" \\\n\t \"(use \\`make latexpdf' here to do that automatically).\"\n\n.PHONY: latexpdf\nlatexpdf:\n\t$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex\n\t@echo \"Running LaTeX files through pdflatex...\"\n\t$(MAKE) -C $(BUILDDIR)/latex all-pdf\n\t@echo \"pdflatex finished; the PDF files are in $(BUILDDIR)/latex.\"\n\n.PHONY: latexpdfja\nlatexpdfja:\n\t$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex\n\t@echo \"Running LaTeX files through platex and dvipdfmx...\"\n\t$(MAKE) -C $(BUILDDIR)/latex all-pdf-ja\n\t@echo \"pdflatex finished; the PDF files are in $(BUILDDIR)/latex.\"\n\n.PHONY: text\ntext:\n\t$(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text\n\t@echo\n\t@echo \"Build finished. The text files are in $(BUILDDIR)/text.\"\n\n.PHONY: man\nman:\n\t$(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man\n\t@echo\n\t@echo \"Build finished. The manual pages are in $(BUILDDIR)/man.\"\n\n.PHONY: texinfo\ntexinfo:\n\t$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo\n\t@echo\n\t@echo \"Build finished. The Texinfo files are in $(BUILDDIR)/texinfo.\"\n\t@echo \"Run \\`make' in that directory to run these through makeinfo\" \\\n\t \"(use \\`make info' here to do that automatically).\"\n\n.PHONY: info\ninfo:\n\t$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo\n\t@echo \"Running Texinfo files through makeinfo...\"\n\tmake -C $(BUILDDIR)/texinfo info\n\t@echo \"makeinfo finished; the Info files are in $(BUILDDIR)/texinfo.\"\n\n.PHONY: gettext\ngettext:\n\t$(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale\n\t@echo\n\t@echo \"Build finished. The message catalogs are in $(BUILDDIR)/locale.\"\n\n.PHONY: changes\nchanges:\n\t$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes\n\t@echo\n\t@echo \"The overview file is in $(BUILDDIR)/changes.\"\n\n.PHONY: linkcheck\nlinkcheck:\n\t$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck\n\t@echo\n\t@echo \"Link check complete; look for any errors in the above output \" \\\n\t \"or in $(BUILDDIR)/linkcheck/output.txt.\"\n\n.PHONY: doctest\ndoctest:\n\t$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest\n\t@echo \"Testing of doctests in the sources finished, look at the \" \\\n\t \"results in $(BUILDDIR)/doctest/output.txt.\"\n\n.PHONY: coverage\ncoverage:\n\t$(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage\n\t@echo \"Testing of coverage in the sources finished, look at the \" \\\n\t \"results in $(BUILDDIR)/coverage/python.txt.\"\n\n.PHONY: xml\nxml:\n\t$(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml\n\t@echo\n\t@echo \"Build finished. The XML files are in $(BUILDDIR)/xml.\"\n\n.PHONY: pseudoxml\npseudoxml:\n\t$(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml\n\t@echo\n\t@echo \"Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml.\"\n\n.PHONY: dummy\ndummy:\n\t$(SPHINXBUILD) -b dummy $(ALLSPHINXOPTS) $(BUILDDIR)/dummy\n\t@echo\n\t@echo \"Build finished. Dummy builder generates no files.\"\n" }, { "path": "docs/conf.py", "content": "# -*- coding: utf-8 -*-\n#\n# Domovoi documentation build configuration file, created by\n# sphinx-quickstart on Wed Dec 14 20:23:38 2016.\n#\n# This file is execfile()d with the current directory set to its\n# containing dir.\n#\n# Note that not all possible configuration values are present in this\n# autogenerated file.\n#\n# All configuration values have a default; values that are commented out\n# serve to show the default.\n\n# If extensions (or modules to document with autodoc) are in another directory,\n# add these directories to sys.path here. If the directory is relative to the\n# documentation root, use os.path.abspath to make it absolute, like shown here.\n#\n# import os\n# import sys\n# sys.path.insert(0, os.path.abspath('.'))\n\n# -- General configuration ------------------------------------------------\n\n# If your documentation needs a minimal Sphinx version, state it here.\n#\n# needs_sphinx = '1.0'\n\n# Add any Sphinx extension module names here, as strings. They can be\n# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom\n# ones.\nextensions = [\n 'sphinx.ext.autodoc',\n 'sphinx.ext.viewcode',\n 'sphinx.ext.githubpages',\n]\n\n# Add any paths that contain templates here, relative to this directory.\ntemplates_path = ['_templates']\n\n# The suffix(es) of source filenames.\n# You can specify multiple suffix as a list of string:\n#\n# source_suffix = ['.rst', '.md']\nsource_suffix = '.rst'\n\n# The encoding of source files.\n#\n# source_encoding = 'utf-8-sig'\n\n# The master toctree document.\nmaster_doc = 'index'\n\n# General information about the project.\nproject = u'Domovoi'\ncopyright = u'2016, Andrey Kislyuk'\nauthor = u'Andrey Kislyuk'\n\n# The version info for the project you're documenting, acts as replacement for\n# |version| and |release|, also used in various other places throughout the\n# built documents.\n#\n# The short X.Y version.\nversion = u'0.0.1'\n# The full version, including alpha/beta/rc tags.\nrelease = u'0.0.1'\n\n# The language for content autogenerated by Sphinx. Refer to documentation\n# for a list of supported languages.\n#\n# This is also used if you do content translation via gettext catalogs.\n# Usually you set \"language\" from the command line for these cases.\nlanguage = None\n\n# There are two options for replacing |today|: either, you set today to some\n# non-false value, then it is used:\n#\n# today = ''\n#\n# Else, today_fmt is used as the format for a strftime call.\n#\n# today_fmt = '%B %d, %Y'\n\n# List of patterns, relative to source directory, that match files and\n# directories to ignore when looking for source files.\n# This patterns also effect to html_static_path and html_extra_path\nexclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']\n\n# The reST default role (used for this markup: `text`) to use for all\n# documents.\n#\n# default_role = None\n\n# If true, '()' will be appended to :func: etc. cross-reference text.\n#\n# add_function_parentheses = True\n\n# If true, the current module name will be prepended to all description\n# unit titles (such as .. function::).\n#\n# add_module_names = True\n\n# If true, sectionauthor and moduleauthor directives will be shown in the\n# output. They are ignored by default.\n#\n# show_authors = False\n\n# The name of the Pygments (syntax highlighting) style to use.\npygments_style = 'sphinx'\n\n# A list of ignored prefixes for module index sorting.\n# modindex_common_prefix = []\n\n# If true, keep warnings as \"system message\" paragraphs in the built documents.\n# keep_warnings = False\n\n# If true, `todo` and `todoList` produce output, else they produce nothing.\ntodo_include_todos = False\n\n\n# -- Options for HTML output ----------------------------------------------\n\n# The theme to use for HTML and HTML Help pages. See the documentation for\n# a list of builtin themes.\n#\nhtml_theme = 'default'\n\n# Theme options are theme-specific and customize the look and feel of a theme\n# further. For a list of options available for each theme, see the\n# documentation.\n#\n# html_theme_options = {}\n\n# Add any paths that contain custom themes here, relative to this directory.\n# html_theme_path = []\n\n# The name for this set of Sphinx documents.\n# \" v documentation\" by default.\n#\n# html_title = u'Domovoi v0.0.1'\n\n# A shorter title for the navigation bar. Default is the same as html_title.\n#\n# html_short_title = None\n\n# The name of an image file (relative to this directory) to place at the top\n# of the sidebar.\n#\n# html_logo = None\n\n# The name of an image file (relative to this directory) to use as a favicon of\n# the docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32\n# pixels large.\n#\n# html_favicon = None\n\n# Add any paths that contain custom static files (such as style sheets) here,\n# relative to this directory. They are copied after the builtin static files,\n# so a file named \"default.css\" will overwrite the builtin \"default.css\".\nhtml_static_path = ['_static']\n\n# Add any extra paths that contain custom files (such as robots.txt or\n# .htaccess) here, relative to this directory. These files are copied\n# directly to the root of the documentation.\n#\n# html_extra_path = []\n\n# If not None, a 'Last updated on:' timestamp is inserted at every page\n# bottom, using the given strftime format.\n# The empty string is equivalent to '%b %d, %Y'.\n#\n# html_last_updated_fmt = None\n\n# If true, SmartyPants will be used to convert quotes and dashes to\n# typographically correct entities.\n#\n# html_use_smartypants = True\n\n# Custom sidebar templates, maps document names to template names.\n#\n# html_sidebars = {}\n\n# Additional templates that should be rendered to pages, maps page names to\n# template names.\n#\n# html_additional_pages = {}\n\n# If false, no module index is generated.\n#\n# html_domain_indices = True\n\n# If false, no index is generated.\n#\n# html_use_index = True\n\n# If true, the index is split into individual pages for each letter.\n#\n# html_split_index = False\n\n# If true, links to the reST sources are added to the pages.\n#\n# html_show_sourcelink = True\n\n# If true, \"Created using Sphinx\" is shown in the HTML footer. Default is True.\n#\n# html_show_sphinx = True\n\n# If true, \"(C) Copyright ...\" is shown in the HTML footer. Default is True.\n#\n# html_show_copyright = True\n\n# If true, an OpenSearch description file will be output, and all pages will\n# contain a tag referring to it. The value of this option must be the\n# base URL from which the finished HTML is served.\n#\n# html_use_opensearch = ''\n\n# This is the file name suffix for HTML files (e.g. \".xhtml\").\n# html_file_suffix = None\n\n# Language to be used for generating the HTML full-text search index.\n# Sphinx supports the following languages:\n# 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja'\n# 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr', 'zh'\n#\n# html_search_language = 'en'\n\n# A dictionary with options for the search language support, empty by default.\n# 'ja' uses this config value.\n# 'zh' user can custom change `jieba` dictionary path.\n#\n# html_search_options = {'type': 'default'}\n\n# The name of a javascript file (relative to the configuration directory) that\n# implements a search results scorer. If empty, the default will be used.\n#\n# html_search_scorer = 'scorer.js'\n\n# Output file base name for HTML help builder.\nhtmlhelp_basename = 'Domovoidoc'\n\n# -- Options for LaTeX output ---------------------------------------------\n\nlatex_elements = {\n # The paper size ('letterpaper' or 'a4paper').\n #\n # 'papersize': 'letterpaper',\n\n # The font size ('10pt', '11pt' or '12pt').\n #\n # 'pointsize': '10pt',\n\n # Additional stuff for the LaTeX preamble.\n #\n # 'preamble': '',\n\n # Latex figure (float) alignment\n #\n # 'figure_align': 'htbp',\n}\n\n# Grouping the document tree into LaTeX files. List of tuples\n# (source start file, target name, title,\n# author, documentclass [howto, manual, or own class]).\nlatex_documents = [\n (master_doc, 'Domovoi.tex', u'Domovoi Documentation',\n u'Andrey Kislyuk', 'manual'),\n]\n\n# The name of an image file (relative to this directory) to place at the top of\n# the title page.\n#\n# latex_logo = None\n\n# For \"manual\" documents, if this is true, then toplevel headings are parts,\n# not chapters.\n#\n# latex_use_parts = False\n\n# If true, show page references after internal links.\n#\n# latex_show_pagerefs = False\n\n# If true, show URL addresses after external links.\n#\n# latex_show_urls = False\n\n# Documents to append as an appendix to all manuals.\n#\n# latex_appendices = []\n\n# If false, no module index is generated.\n#\n# latex_domain_indices = True\n\n\n# -- Options for manual page output ---------------------------------------\n\n# One entry per manual page. List of tuples\n# (source start file, name, description, authors, manual section).\nman_pages = [\n (master_doc, 'domovoi', u'Domovoi Documentation',\n [author], 1)\n]\n\n# If true, show URL addresses after external links.\n#\n# man_show_urls = False\n\n\n# -- Options for Texinfo output -------------------------------------------\n\n# Grouping the document tree into Texinfo files. List of tuples\n# (source start file, target name, title, author,\n# dir menu entry, description, category)\ntexinfo_documents = [\n (master_doc, 'Domovoi', u'Domovoi Documentation',\n author, 'Domovoi', 'One line description of project.',\n 'Miscellaneous'),\n]\n\n# Documents to append as an appendix to all manuals.\n#\n# texinfo_appendices = []\n\n# If false, no module index is generated.\n#\n# texinfo_domain_indices = True\n\n# How to display URL addresses: 'footnote', 'no', or 'inline'.\n#\n# texinfo_show_urls = 'footnote'\n\n# If true, do not generate a @detailmenu in the \"Top\" node's menu.\n#\n# texinfo_no_detailmenu = False\n" }, { "path": "docs/index.rst", "content": ".. include:: ../README.rst\n\nAPI documentation\n=================\n\n.. automodule:: domovoi\n :members:\n\n\nTable of Contents\n=================\n\n.. toctree::\n :maxdepth: 5\n\n index\n\n* :ref:`genindex`\n* :ref:`modindex`\n* :ref:`search`\n" }, { "path": "domovoi/__init__.py", "content": "from domovoi.app import Domovoi\n" }, { "path": "domovoi/app.py", "content": "from __future__ import absolute_import, division, print_function, unicode_literals\n\nimport json, gzip, base64, logging\n\nfrom chalice.app import Chalice, LambdaFunction, DecoratorAPI as ChaliceDecoratorAPI\n\n\nclass DomovoiException(Exception):\n pass\n\n\nclass ARN:\n fields = \"arn partition service region account_id resource\".split()\n\n def __init__(self, arn=\"arn:aws::::\", **kwargs):\n self.__dict__.update(dict(zip(self.fields, arn.split(\":\", 5)), **kwargs))\n\n def __str__(self):\n return \":\".join(getattr(self, field) for field in self.fields)\n\n\nclass StateMachine:\n def __init__(self, app, client=None):\n self.app = app\n self._client = client\n\n @property\n def stepfunctions(self):\n if self._client is None:\n import boto3\n self._client = boto3.client(\"stepfunctions\")\n return self._client\n\n def start_execution(self, **input):\n return self.start_named_execution(None, **input)\n\n def start_named_execution(self, name, **input):\n lambda_arn = ARN(self.app.lambda_context.invoked_function_arn)\n lambda_name = lambda_arn.resource.split(\":\")[1]\n state_machine_arn = ARN(str(lambda_arn), service=\"states\", resource=\"stateMachine:\" + lambda_name)\n start_execution_args = dict(stateMachineArn=str(state_machine_arn), input=json.dumps(input))\n if name is not None:\n start_execution_args.update(name=name)\n return self.stepfunctions.start_execution(**start_execution_args)\n\n\nclass Domovoi(Chalice):\n cloudwatch_events_rules = {}\n sns_subscribers = {}\n sqs_subscribers = {}\n s3_subscribers = {}\n sfn_tasks = {}\n cwl_sub_filters = {}\n dynamodb_event_sources = {}\n alb_targets = {}\n\n sqs_default_queue_attributes = {\"VisibilityTimeout\": \"320\"}\n\n def unsupported_decorator(*args, **kwargs):\n raise NotImplementedError(\"Domovoi does not support this Chalice decorator\")\n\n def __init__(self, app_name=\"Domovoi\", configure_logs=True):\n Chalice.__init__(self, app_name=app_name, configure_logs=configure_logs)\n self.pure_lambda_functions = [LambdaFunction(self, name=app_name, handler_string=\"app.app\")]\n for f in dir(ChaliceDecoratorAPI):\n if callable(getattr(ChaliceDecoratorAPI, f)) and not f.startswith(\"_\"):\n setattr(self, f, Domovoi.unsupported_decorator)\n\n def _configure_log_level(self):\n if self._debug:\n level = logging.DEBUG\n else:\n level = logging.INFO\n self.log.setLevel(level)\n\n def alb_target(self, prefix=\"\"):\n def register_alb_target(func):\n self.alb_targets[prefix] = dict(func=func, prefix=prefix)\n return func\n return register_alb_target\n\n def scheduled_function(self, schedule, rule_name=None):\n return self.cloudwatch_rule(schedule_expression=schedule, event_pattern=None, rule_name=rule_name)\n\n def sns_topic_subscriber(self, topic_name):\n def register_sns_subscriber(func):\n self.sns_subscribers[topic_name] = func\n return func\n return register_sns_subscriber\n\n def sqs_queue_subscriber(self, queue_name, batch_size=None, queue_attributes=None):\n def register_sqs_subscriber(func):\n self.sqs_subscribers[queue_name] = dict(func=func, batch_size=batch_size, queue_attributes=queue_attributes)\n return func\n return register_sqs_subscriber\n\n def dynamodb_stream_handler(self, table_name, batch_size=None):\n def register_dynamodb_event_source(func):\n self.dynamodb_event_sources[table_name] = dict(batch_size=batch_size, func=func)\n return func\n return register_dynamodb_event_source\n\n def kinesis_stream_handler(self, **kwargs):\n raise NotImplementedError()\n\n def email_receipt_handler(self):\n # http://boto3.readthedocs.io/en/latest/reference/services/ses.html#SES.Client.create_receipt_rule\n raise NotImplementedError()\n\n def cloudwatch_logs_sub_filter_handler(self, log_group_name, filter_pattern):\n def register_cwl_subscription_filter(func):\n self.cwl_sub_filters[log_group_name] = dict(log_group_name=log_group_name, filter_pattern=filter_pattern,\n func=func)\n return func\n return register_cwl_subscription_filter\n\n def cloudwatch_event_handler(self, **kwargs):\n return self.cloudwatch_rule(schedule_expression=None, event_pattern=kwargs)\n\n def s3_event_handler(self, bucket, events, prefix=None, suffix=None, use_sns=True, use_sqs=False, sqs_batch_size=1,\n sqs_queue_attributes=None):\n def register_s3_subscriber(func):\n self.s3_subscribers[bucket] = dict(events=events, prefix=prefix, suffix=suffix, func=func, use_sns=use_sns,\n use_sqs=use_sqs, sqs_batch_size=sqs_batch_size,\n sqs_queue_attributes=sqs_queue_attributes)\n return func\n return register_s3_subscriber\n\n def cloudwatch_rule(self, schedule_expression, event_pattern, rule_name=None):\n def register_rule(func):\n _rule_name = rule_name or func.__name__\n if _rule_name in self.cloudwatch_events_rules:\n raise KeyError(func.__name__)\n rule = dict(schedule_expression=schedule_expression, event_pattern=event_pattern, func=func)\n self.cloudwatch_events_rules[_rule_name] = rule\n return func\n return register_rule\n\n def step_function_task(self, state_name, state_machine_definition):\n def register_sfn_task(func):\n if state_name in self.sfn_tasks:\n raise KeyError(state_name)\n self.sfn_tasks[state_name] = dict(state_name=state_name,\n state_machine_definition=state_machine_definition,\n func=func)\n return func\n return register_sfn_task\n\n def register_state_machine(self, state_machine_definition):\n for state_name, state_data in self.get_all_states(state_machine_definition).items():\n if callable(state_data.get(\"Resource\", None)):\n self.step_function_task(state_name, state_machine_definition)(state_data[\"Resource\"])\n\n @classmethod\n def get_all_states(cls, state_machine):\n states = dict(state_machine[\"States\"])\n for state_name, state_data in state_machine[\"States\"].items():\n for sub_sm in state_data.get(\"Branches\", []):\n states.update(cls.get_all_states(sub_sm))\n return states\n\n @property\n def state_machine(self):\n return StateMachine(app=self)\n\n def _find_forwarded_s3_event(self, s3_event_envelope, forwarding_service):\n assert forwarding_service in {\"sns\", \"sqs\"}\n if forwarding_service == \"sns\":\n assert s3_event_envelope['Records'][0][\"Sns\"][\"Subject\"] == \"Amazon S3 Notification\"\n s3_event = json.loads(s3_event_envelope['Records'][0][\"Sns\"][\"Message\"])\n elif forwarding_service == \"sqs\":\n forwarded_event = json.loads(s3_event_envelope[\"Records\"][0][\"body\"])\n if forwarded_event.get(\"TopicArn\") and forwarded_event.get(\"Subject\") == \"Amazon S3 Notification\":\n s3_event = json.loads(forwarded_event[\"Message\"])\n else:\n s3_event = forwarded_event\n assert s3_event.get(\"Event\") == \"s3:TestEvent\" or s3_event['Records'][0].get(\"eventSource\") == \"aws:s3\"\n s3_bucket_name = s3_event.get(\"Bucket\") or s3_event['Records'][0][\"s3\"][\"bucket\"][\"name\"]\n handler = self.s3_subscribers[s3_bucket_name][\"func\"] if s3_bucket_name in self.s3_subscribers else None\n return s3_event, handler\n\n def __call__(self, event, context):\n self.log.info(\"Domovoi dispatch of event %s\", event)\n self.lambda_context = context\n invoked_function_arn = ARN(context.invoked_function_arn)\n handler = None\n if \"requestContext\" in event and \"elb\" in event[\"requestContext\"]:\n target = None\n # TODO: use suffix tree to avoid O(N) scan of route table\n for prefix, alb_target in self.alb_targets.items():\n if event[\"path\"].startswith(prefix):\n if target is None or len(target[\"prefix\"]) < len(alb_target[\"prefix\"]):\n target = alb_target\n handler = target[\"func\"]\n if \"task_name\" in event:\n if event[\"task_name\"] not in self.cloudwatch_events_rules:\n raise DomovoiException(\"Received CloudWatch event for a task with no known handler\")\n handler = self.cloudwatch_events_rules[event[\"task_name\"]][\"func\"]\n event = event[\"event\"]\n elif \"Records\" in event and \"s3\" in event[\"Records\"][0]:\n s3_bucket_name = event[\"Records\"][0][\"s3\"][\"bucket\"][\"name\"]\n if s3_bucket_name not in self.s3_subscribers:\n raise DomovoiException(\"Received S3 event for a bucket with no known handler\")\n handler = self.s3_subscribers[s3_bucket_name][\"func\"]\n elif \"Records\" in event and \"Sns\" in event[\"Records\"][0]:\n try:\n event, handler = self._find_forwarded_s3_event(event, forwarding_service=\"sns\")\n except Exception:\n sns_topic = ARN(event[\"Records\"][0][\"Sns\"][\"TopicArn\"]).resource\n if sns_topic not in self.sns_subscribers:\n raise DomovoiException(\"Received SNS or S3-SNS event with no known handler\")\n handler = self.sns_subscribers[sns_topic]\n elif \"Records\" in event and event[\"Records\"][0].get(\"eventSource\") == \"aws:sqs\":\n try:\n event, handler = self._find_forwarded_s3_event(event, forwarding_service=\"sqs\")\n except Exception:\n queue_name = ARN(event[\"Records\"][0][\"eventSourceARN\"]).resource\n handler = self.sqs_subscribers[queue_name][\"func\"]\n elif \"Records\" in event and \"dynamodb\" in event[\"Records\"][0]:\n event_source_arn = ARN(event[\"Records\"][0][\"eventSourceARN\"])\n table_name = event_source_arn.resource.split(\"/\")[1]\n handler = self.dynamodb_event_sources[table_name][\"func\"]\n elif \"awslogs\" in event:\n event = json.loads(gzip.decompress(base64.b64decode(event[\"awslogs\"][\"data\"])))\n handler = self.cwl_sub_filters[event[\"logGroup\"]][\"func\"]\n elif \"domovoi-stepfunctions-task\" in invoked_function_arn.resource:\n _, lambda_name, lambda_alias = invoked_function_arn.resource.split(\":\")\n assert lambda_alias.startswith(\"domovoi-stepfunctions-task-\")\n task_name = lambda_alias[len(\"domovoi-stepfunctions-task-\"):]\n context.stepfunctions_task_name = task_name\n handler = self.sfn_tasks[task_name][\"func\"]\n\n if handler is None:\n raise DomovoiException(\"No handler found for event {}\".format(event))\n result = handler(event, context)\n self.log.info(\"%s\", result)\n return result\n" }, { "path": "domovoi/default_iam_policy.json", "content": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Action\": [\n \"events:*\",\n \"iam:ListAttachedRolePolicies\",\n \"iam:ListRolePolicies\",\n \"iam:ListRoles\",\n \"iam:PassRole\"\n ],\n \"Resource\": \"*\",\n \"Effect\": \"Allow\"\n },\n {\n \"Action\": [\n \"sns:CreateTopic\",\n \"sns:Publish\"\n ],\n \"Resource\": \"arn:aws:sns:*:*:*\",\n \"Effect\": \"Allow\"\n },\n {\n \"Action\": [\n \"sqs:ReceiveMessage\",\n \"sqs:DeleteMessage\",\n \"sqs:GetQueueAttributes\"\n ],\n \"Resource\": [\n \"arn:aws:sqs:*:*:*\"\n ],\n \"Effect\": \"Allow\"\n },\n {\n \"Action\": [\n \"logs:CreateLogGroup\",\n \"logs:CreateLogStream\",\n \"logs:PutLogEvents\",\n \"logs:DescribeLogStreams\"\n ],\n \"Resource\": [\n \"arn:aws:logs:*:*:*\"\n ],\n \"Effect\": \"Allow\"\n }\n ]\n}\n" }, { "path": "domovoi/examples/alb-event.json", "content": "{\n \"requestContext\": {\n \"elb\": {\n \"targetGroupArn\": \"arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/lambda-279XGJDqGZ5rsrHC2Fjr/49e9d65c45c6791a\"\n }\n },\n \"httpMethod\": \"GET\",\n \"path\": \"/lambda\",\n \"queryStringParameters\": {\n \"query\": \"1234ABCD\"\n },\n \"headers\": {\n \"accept\": \"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8\",\n \"accept-encoding\": \"gzip\",\n \"accept-language\": \"en-US,en;q=0.9\",\n \"connection\": \"keep-alive\",\n \"host\": \"lambda-alb-123578498.us-east-2.elb.amazonaws.com\",\n \"upgrade-insecure-requests\": \"1\",\n \"user-agent\": \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36\",\n \"x-amzn-trace-id\": \"Root=1-5c536348-3d683b8b04734faae651f476\",\n \"x-forwarded-for\": \"72.12.164.125\",\n \"x-forwarded-port\": \"80\",\n \"x-forwarded-proto\": \"http\",\n \"x-imforwards\": \"20\"\n },\n \"body\": \"\",\n \"isBase64Encoded\": false\n}\n" }, { "path": "domovoi/examples/alexa-event.json", "content": "{\n \"header\": {\n \"payloadVersion\": \"1\",\n \"namespace\": \"Control\",\n \"name\": \"SwitchOnOffRequest\"\n },\n \"payload\": {\n \"switchControlAction\": \"TURN_ON\",\n \"appliance\": {\n \"additionalApplianceDetails\": {\n \"key2\": \"value2\",\n \"key1\": \"value1\"\n },\n \"applianceId\": \"sampleId\"\n },\n \"accessToken\": \"sampleAccessToken\"\n }\n}\n" }, { "path": "domovoi/examples/apigateway-event.json", "content": "{\n \"path\": \"/test/hello\",\n \"headers\": {\n \"Accept\": \"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8\",\n \"Accept-Encoding\": \"gzip, deflate, lzma, sdch, br\",\n \"Accept-Language\": \"en-US,en;q=0.8\",\n \"CloudFront-Forwarded-Proto\": \"https\",\n \"CloudFront-Is-Desktop-Viewer\": \"true\",\n \"CloudFront-Is-Mobile-Viewer\": \"false\",\n \"CloudFront-Is-SmartTV-Viewer\": \"false\",\n \"CloudFront-Is-Tablet-Viewer\": \"false\",\n \"CloudFront-Viewer-Country\": \"US\",\n \"Host\": \"wt6mne2s9k.execute-api.us-west-2.amazonaws.com\",\n \"Upgrade-Insecure-Requests\": \"1\",\n \"User-Agent\": \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.82 Safari/537.36 OPR/39.0.2256.48\",\n \"Via\": \"1.1 fb7cca60f0ecd82ce07790c9c5eef16c.cloudfront.net (CloudFront)\",\n \"X-Amz-Cf-Id\": \"nBsWBOrSHMgnaROZJK1wGCZ9PcRcSpq_oSXZNQwQ10OTZL4cimZo3g==\",\n \"X-Forwarded-For\": \"192.168.100.1, 192.168.1.1\",\n \"X-Forwarded-Port\": \"443\",\n \"X-Forwarded-Proto\": \"https\"\n },\n \"pathParameters\": {\n \"proxy\": \"hello\"\n },\n \"requestContext\": {\n \"accountId\": \"123456789012\",\n \"resourceId\": \"us4z18\",\n \"stage\": \"test\",\n \"requestId\": \"41b45ea3-70b5-11e6-b7bd-69b5aaebc7d9\",\n \"identity\": {\n \"cognitoIdentityPoolId\": \"\",\n \"accountId\": \"\",\n \"cognitoIdentityId\": \"\",\n \"caller\": \"\",\n \"apiKey\": \"\",\n \"sourceIp\": \"192.168.100.1\",\n \"cognitoAuthenticationType\": \"\",\n \"cognitoAuthenticationProvider\": \"\",\n \"userArn\": \"\",\n \"userAgent\": \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.82 Safari/537.36 OPR/39.0.2256.48\",\n \"user\": \"\"\n },\n \"resourcePath\": \"/{proxy+}\",\n \"httpMethod\": \"GET\",\n \"apiId\": \"wt6mne2s9k\"\n },\n \"resource\": \"/{proxy+}\",\n \"httpMethod\": \"GET\",\n \"queryStringParameters\": {\n \"name\": \"me\"\n },\n \"stageVariables\": {\n \"stageVarName\": \"stageVarValue\"\n }\n}\n" }, { "path": "domovoi/examples/cloudformation-event.json", "content": "{\n \"RequestType\": \"Create\",\n \"ServiceToken\": \"arn:aws:lambda:us-east-2:123456789012:function:lambda-error-processor-primer-14ROR2T3JKU66\",\n \"ResponseURL\": \"https://cloudformation-custom-resource-response-useast2.s3-us-east-2.amazonaws.com/arn%3Aaws%3Acloudformation%3Aus-east-2%3A123456789012%3Astack/lambda-error-processor/1134083a-2608-1e91-9897-022501a2c456%7Cprimerinvoke%7C5d478078-13e9-baf0-464a-7ef285ecc786?AWSAccessKeyId=AKIAIOSFODNN7EXAMPLE&Expires=1555451971&Signature=28UijZePE5I4dvukKQqM%2F9Rf1o4%3D\",\n \"StackId\": \"arn:aws:cloudformation:us-east-2:123456789012:stack/lambda-error-processor/1134083a-2608-1e91-9897-022501a2c456\",\n \"RequestId\": \"5d478078-13e9-baf0-464a-7ef285ecc786\",\n \"LogicalResourceId\": \"primerinvoke\",\n \"ResourceType\": \"AWS::CloudFormation::CustomResource\",\n \"ResourceProperties\": {\n \"ServiceToken\": \"arn:aws:lambda:us-east-2:123456789012:function:lambda-error-processor-primer-14ROR2T3JKU66\",\n \"FunctionName\": \"lambda-error-processor-randomerror-ZWUC391MQAJK\"\n }\n}\n" }, { "path": "domovoi/examples/cloudfront-event.json", "content": "{\n \"Records\": [\n {\n \"cf\": {\n \"config\": {\n \"distributionId\": \"EDFDVBD6EXAMPLE\"\n },\n \"request\": {\n \"clientIp\": \"2001:0db8:85a3:0:0:8a2e:0370:7334\",\n \"method\": \"GET\",\n \"uri\": \"/picture.jpg\",\n \"headers\": {\n \"host\": [\n {\n \"key\": \"Host\",\n \"value\": \"d111111abcdef8.cloudfront.net\"\n }\n ],\n \"user-agent\": [\n {\n \"key\": \"User-Agent\",\n \"value\": \"curl/7.51.0\"\n }\n ]\n }\n }\n }\n }\n ]\n}\n" }, { "path": "domovoi/examples/cloudtrail-event.json", "content": "{\n \"Records\": [\n {\n \"eventVersion\": \"1.02\",\n \"userIdentity\": {\n \"type\": \"Root\",\n \"principalId\": \"123456789012\",\n \"arn\": \"arn:aws:iam::123456789012:root\",\n \"accountId\": \"123456789012\",\n \"accessKeyId\": \"access-key-id\",\n \"sessionContext\": {\n \"attributes\": {\n \"mfaAuthenticated\": \"false\",\n \"creationDate\": \"2015-01-24T22:41:54Z\"\n }\n }\n },\n \"eventTime\": \"2015-01-24T23:26:50Z\",\n \"eventSource\": \"sns.amazonaws.com\",\n \"eventName\": \"CreateTopic\",\n \"awsRegion\": \"us-east-2\",\n \"sourceIPAddress\": \"205.251.233.176\",\n \"userAgent\": \"console.amazonaws.com\",\n \"requestParameters\": {\n \"name\": \"dropmeplease\"\n },\n \"responseElements\": {\n \"topicArn\": \"arn:aws:sns:us-east-2:123456789012:exampletopic\"\n },\n \"requestID\": \"3fdb7834-9079-557e-8ef2-350abc03536b\",\n \"eventID\": \"17b46459-dada-4278-b8e2-5a4ca9ff1a9c\",\n \"eventType\": \"AwsApiCall\",\n \"recipientAccountId\": \"123456789012\"\n },\n {\n \"eventVersion\": \"1.02\",\n \"userIdentity\": {\n \"type\": \"Root\",\n \"principalId\": \"123456789012\",\n \"arn\": \"arn:aws:iam::123456789012:root\",\n \"accountId\": \"123456789012\",\n \"accessKeyId\": \"AKIAIOSFODNN7EXAMPLE\",\n \"sessionContext\": {\n \"attributes\": {\n \"mfaAuthenticated\": \"false\",\n \"creationDate\": \"2015-01-24T22:41:54Z\"\n }\n }\n },\n \"eventTime\": \"2015-01-24T23:27:02Z\",\n \"eventSource\": \"sns.amazonaws.com\",\n \"eventName\": \"GetTopicAttributes\",\n \"awsRegion\": \"us-east-2\",\n \"sourceIPAddress\": \"205.251.233.176\",\n \"userAgent\": \"console.amazonaws.com\",\n \"requestParameters\": {\n \"topicArn\": \"arn:aws:sns:us-east-2:123456789012:exampletopic\"\n },\n \"responseElements\": null,\n \"requestID\": \"4a0388f7-a0af-5df9-9587-c5c98c29cbec\",\n \"eventID\": \"ec5bb073-8fa1-4d45-b03c-f07b9fc9ea18\",\n \"eventType\": \"AwsApiCall\",\n \"recipientAccountId\": \"123456789012\"\n }\n ]\n}\n" }, { "path": "domovoi/examples/cloudwatch-event.json", "content": "{\n \"account\": \"123456789012\",\n \"region\": \"us-east-2\",\n \"detail\": {},\n \"detail-type\": \"Scheduled Event\",\n \"source\": \"aws.events\",\n \"time\": \"2019-03-01T01:23:45Z\",\n \"id\": \"cdc73f9d-aea9-11e3-9d5a-835b769c0d9c\",\n \"resources\": [\n \"arn:aws:events:us-east-1:123456789012:rule/my-schedule\"\n ]\n}\n" }, { "path": "domovoi/examples/codecommit-event.json", "content": "{\n \"Records\": [\n {\n \"awsRegion\": \"us-east-2\",\n \"codecommit\": {\n \"references\": [\n {\n \"commit\": \"5e493c6f3067653f3d04eca608b4901eb227078\",\n \"ref\": \"refs/heads/master\"\n }\n ]\n },\n \"eventId\": \"31ade2c7-f889-47c5-a937-1cf99e2790e9\",\n \"eventName\": \"ReferenceChanges\",\n \"eventPartNumber\": 1,\n \"eventSource\": \"aws:codecommit\",\n \"eventSourceARN\": \"arn:aws:codecommit:us-east-2:123456789012:lambda-pipeline-repo\",\n \"eventTime\": \"2019-03-12T20:58:25.400+0000\",\n \"eventTotalParts\": 1,\n \"eventTriggerConfigId\": \"0d17d6a4-efeb-46f3-b3ab-a63741badeb8\",\n \"eventTriggerName\": \"index.handler\",\n \"eventVersion\": \"1.0\",\n \"userIdentityARN\": \"arn:aws:iam::123456789012:user/intern\"\n }\n ]\n}\n" }, { "path": "domovoi/examples/cognito-event.json", "content": "{\n \"datasetName\": \"datasetName\",\n \"eventType\": \"SyncTrigger\",\n \"region\": \"us-east-1\",\n \"identityId\": \"identityId\",\n \"datasetRecords\": {\n \"SampleKey2\": {\n \"newValue\": \"newValue2\",\n \"oldValue\": \"oldValue2\",\n \"op\": \"replace\"\n },\n \"SampleKey1\": {\n \"newValue\": \"newValue1\",\n \"oldValue\": \"oldValue1\",\n \"op\": \"replace\"\n }\n },\n \"identityPoolId\": \"identityPoolId\",\n \"version\": 2\n}\n" }, { "path": "domovoi/examples/config-event.json", "content": "{\n \"invokingEvent\": \"{\\\"configurationItem\\\":{\\\"configurationItemCaptureTime\\\":\\\"2016-02-17T01:36:34.043Z\\\",\\\"awsAccountId\\\":\\\"000000000000\\\",\\\"configurationItemStatus\\\":\\\"OK\\\",\\\"resourceId\\\":\\\"i-00000000\\\",\\\"ARN\\\":\\\"arn:aws:ec2:us-east-1:000000000000:instance/i-00000000\\\",\\\"awsRegion\\\":\\\"us-east-1\\\",\\\"availabilityZone\\\":\\\"us-east-1a\\\",\\\"resourceType\\\":\\\"AWS::EC2::Instance\\\",\\\"tags\\\":{\\\"Foo\\\":\\\"Bar\\\"},\\\"relationships\\\":[{\\\"resourceId\\\":\\\"eipalloc-00000000\\\",\\\"resourceType\\\":\\\"AWS::EC2::EIP\\\",\\\"name\\\":\\\"Is attached to ElasticIp\\\"}],\\\"configuration\\\":{\\\"foo\\\":\\\"bar\\\"}},\\\"messageType\\\":\\\"ConfigurationItemChangeNotification\\\"}\",\n \"ruleParameters\": \"{\\\"myParameterKey\\\":\\\"myParameterValue\\\"}\",\n \"resultToken\": \"myResultToken\",\n \"eventLeftScope\": false,\n \"executionRoleArn\": \"arn:aws:iam::012345678912:role/config-role\",\n \"configRuleArn\": \"arn:aws:config:us-east-1:012345678912:config-rule/config-rule-0123456\",\n \"configRuleName\": \"change-triggered-config-rule\",\n \"configRuleId\": \"config-rule-0123456\",\n \"accountId\": \"012345678912\",\n \"version\": \"1.0\"\n}\n" }, { "path": "domovoi/examples/dynamodb-event.json", "content": "{\n \"Records\": [\n {\n \"eventID\": \"1\",\n \"eventVersion\": \"1.0\",\n \"dynamodb\": {\n \"Keys\": {\n \"Id\": {\n \"N\": \"101\"\n }\n },\n \"NewImage\": {\n \"Message\": {\n \"S\": \"New item!\"\n },\n \"Id\": {\n \"N\": \"101\"\n }\n },\n \"StreamViewType\": \"NEW_AND_OLD_IMAGES\",\n \"SequenceNumber\": \"111\",\n \"SizeBytes\": 26\n },\n \"awsRegion\": \"us-west-2\",\n \"eventName\": \"INSERT\",\n \"eventSourceARN\": \"{{eventsourcearn}}\",\n \"eventSource\": \"aws:dynamodb\"\n },\n {\n \"eventID\": \"2\",\n \"eventVersion\": \"1.0\",\n \"dynamodb\": {\n \"OldImage\": {\n \"Message\": {\n \"S\": \"New item!\"\n },\n \"Id\": {\n \"N\": \"101\"\n }\n },\n \"SequenceNumber\": \"222\",\n \"Keys\": {\n \"Id\": {\n \"N\": \"101\"\n }\n },\n \"SizeBytes\": 59,\n \"NewImage\": {\n \"Message\": {\n \"S\": \"This item has changed\"\n },\n \"Id\": {\n \"N\": \"101\"\n }\n },\n \"StreamViewType\": \"NEW_AND_OLD_IMAGES\"\n },\n \"awsRegion\": \"us-west-2\",\n \"eventName\": \"MODIFY\",\n \"eventSourceARN\": \"{{sourcearn}}\",\n \"eventSource\": \"aws:dynamodb\"\n }\n ]\n}\n" }, { "path": "domovoi/examples/firehose-event.json", "content": "{\n \"invocationId\": \"invoked123\",\n \"deliveryStreamArn\": \"aws:lambda:events\",\n \"region\": \"us-west-2\",\n \"records\": [\n {\n \"data\": \"SGVsbG8gV29ybGQ=\",\n \"recordId\": \"record1\",\n \"approximateArrivalTimestamp\": 1510772160000,\n \"kinesisRecordMetadata\": {\n \"shardId\": \"shardId-000000000000\",\n \"partitionKey\": \"4d1ad2b9-24f8-4b9d-a088-76e9947c317a\",\n \"approximateArrivalTimestamp\": \"2012-04-23T18:25:43.511Z\",\n \"sequenceNumber\": \"49546986683135544286507457936321625675700192471156785154\",\n \"subsequenceNumber\": \"\"\n }\n },\n {\n \"data\": \"SGVsbG8gV29ybGQ=\",\n \"recordId\": \"record2\",\n \"approximateArrivalTimestamp\": 151077216000,\n \"kinesisRecordMetadata\": {\n \"shardId\": \"shardId-000000000001\",\n \"partitionKey\": \"4d1ad2b9-24f8-4b9d-a088-76e9947c318a\",\n \"approximateArrivalTimestamp\": \"2012-04-23T19:25:43.511Z\",\n \"sequenceNumber\": \"49546986683135544286507457936321625675700192471156785155\",\n \"subsequenceNumber\": \"\"\n }\n }\n ]\n}\n" }, { "path": "domovoi/examples/kinesis-event.json", "content": "{\n \"Records\": [\n {\n \"kinesis\": {\n \"kinesisSchemaVersion\": \"1.0\",\n \"partitionKey\": \"1\",\n \"sequenceNumber\": \"49590338271490256608559692538361571095921575989136588898\",\n \"data\": \"SGVsbG8sIHRoaXMgaXMgYSB0ZXN0Lg==\",\n \"approximateArrivalTimestamp\": 1545084650.987\n },\n \"eventSource\": \"aws:kinesis\",\n \"eventVersion\": \"1.0\",\n \"eventID\": \"shardId-000000000006:49590338271490256608559692538361571095921575989136588898\",\n \"eventName\": \"aws:kinesis:record\",\n \"invokeIdentityArn\": \"arn:aws:iam::123456789012:role/lambda-role\",\n \"awsRegion\": \"us-east-2\",\n \"eventSourceARN\": \"arn:aws:kinesis:us-east-2:123456789012:stream/lambda-stream\"\n },\n {\n \"kinesis\": {\n \"kinesisSchemaVersion\": \"1.0\",\n \"partitionKey\": \"1\",\n \"sequenceNumber\": \"49590338271490256608559692540925702759324208523137515618\",\n \"data\": \"VGhpcyBpcyBvbmx5IGEgdGVzdC4=\",\n \"approximateArrivalTimestamp\": 1545084711.166\n },\n \"eventSource\": \"aws:kinesis\",\n \"eventVersion\": \"1.0\",\n \"eventID\": \"shardId-000000000006:49590338271490256608559692540925702759324208523137515618\",\n \"eventName\": \"aws:kinesis:record\",\n \"invokeIdentityArn\": \"arn:aws:iam::123456789012:role/lambda-role\",\n \"awsRegion\": \"us-east-2\",\n \"eventSourceARN\": \"arn:aws:kinesis:us-east-2:123456789012:stream/lambda-stream\"\n }\n ]\n}\n" }, { "path": "domovoi/examples/logs-event.json", "content": "{\n \"awslogs\": {\n \"data\": \"ewogICAgIm1lc3NhZ2VUeXBlIjogIkRBVEFfTUVTU0FHRSIsCiAgICAib3duZXIiOiAiMTIzNDU2Nzg5MDEyIiwKICAgICJsb2dHcm91cCI6I...\"\n }\n}\n" }, { "path": "domovoi/examples/s3-event.json", "content": "{\n \"Records\": [\n {\n \"eventVersion\": \"2.0\",\n \"eventSource\": \"aws:s3\",\n \"awsRegion\": \"us-west-2\",\n \"eventTime\": \"1970-01-01T00:00:00.000Z\",\n \"eventName\": \"ObjectCreated:Put\",\n \"userIdentity\": {\n \"principalId\": \"AIDAJDPLRKLG7UEXAMPLE\"\n },\n \"requestParameters\": {\n \"sourceIPAddress\": \"127.0.0.1\"\n },\n \"responseElements\": {\n \"x-amz-request-id\": \"C3D13FE58DE4C810\",\n \"x-amz-id-2\": \"FMyUVURIY8/IgAtTv8xRjskZQpcIZ9KG4V5Wp6S7S/JRWeUWerMUE5JgHvANOjpD\"\n },\n \"s3\": {\n \"s3SchemaVersion\": \"1.0\",\n \"configurationId\": \"testConfigRule\",\n \"bucket\": {\n \"name\": \"sourcebucket\",\n \"ownerIdentity\": {\n \"principalId\": \"A3NL1KOZZKExample\"\n },\n \"arn\": \"arn:aws:s3:::sourcebucket\"\n },\n \"object\": {\n \"key\": \"HappyFace.jpg\",\n \"size\": 1024,\n \"eTag\": \"d41d8cd98f00b204e9800998ecf8427e\",\n \"versionId\": \"096fKKXTRTtl3on89fVO.nfljtsv6qko\"\n }\n }\n }\n ]\n}\n" }, { "path": "domovoi/examples/ses-event.json", "content": "{\n \"Records\": [\n {\n \"eventVersion\": \"1.0\",\n \"ses\": {\n \"mail\": {\n \"commonHeaders\": {\n \"from\": [\n \"Jane Doe \"\n ],\n \"to\": [\n \"johndoe@example.com\"\n ],\n \"returnPath\": \"janedoe@example.com\",\n \"messageId\": \"<0123456789example.com>\",\n \"date\": \"Wed, 7 Oct 2015 12:34:56 -0700\",\n \"subject\": \"Test Subject\"\n },\n \"source\": \"janedoe@example.com\",\n \"timestamp\": \"1970-01-01T00:00:00.000Z\",\n \"destination\": [\n \"johndoe@example.com\"\n ],\n \"headers\": [\n {\n \"name\": \"Return-Path\",\n \"value\": \"\"\n },\n {\n \"name\": \"Received\",\n \"value\": \"from mailer.example.com (mailer.example.com [203.0.113.1]) by inbound-smtp.us-west-2.amazonaws.com with SMTP id o3vrnil0e2ic for johndoe@example.com; Wed, 07 Oct 2015 12:34:56 +0000 (UTC)\"\n },\n {\n \"name\": \"DKIM-Signature\",\n \"value\": \"v=1; a=rsa-sha256; c=relaxed/relaxed; d=example.com; s=example; h=mime-version:from:date:message-id:subject:to:content-type; bh=jX3F0bCAI7sIbkHyy3mLYO28ieDQz2R0P8HwQkklFj4=; b=sQwJ+LMe9RjkesGu+vqU56asvMhrLRRYrWCbV\"\n },\n {\n \"name\": \"MIME-Version\",\n \"value\": \"1.0\"\n },\n {\n \"name\": \"From\",\n \"value\": \"Jane Doe \"\n },\n {\n \"name\": \"Date\",\n \"value\": \"Wed, 7 Oct 2015 12:34:56 -0700\"\n },\n {\n \"name\": \"Message-ID\",\n \"value\": \"<0123456789example.com>\"\n },\n {\n \"name\": \"Subject\",\n \"value\": \"Test Subject\"\n },\n {\n \"name\": \"To\",\n \"value\": \"johndoe@example.com\"\n },\n {\n \"name\": \"Content-Type\",\n \"value\": \"text/plain; charset=UTF-8\"\n }\n ],\n \"headersTruncated\": false,\n \"messageId\": \"o3vrnil0e2ic28tr\"\n },\n \"receipt\": {\n \"recipients\": [\n \"johndoe@example.com\"\n ],\n \"timestamp\": \"1970-01-01T00:00:00.000Z\",\n \"spamVerdict\": {\n \"status\": \"PASS\"\n },\n \"dkimVerdict\": {\n \"status\": \"PASS\"\n },\n \"processingTimeMillis\": 574,\n \"action\": {\n \"type\": \"Lambda\",\n \"invocationType\": \"Event\",\n \"functionArn\": \"arn:aws:lambda:us-west-2:012345678912:function:Example\"\n },\n \"spfVerdict\": {\n \"status\": \"PASS\"\n },\n \"virusVerdict\": {\n \"status\": \"PASS\"\n }\n }\n },\n \"eventSource\": \"aws:ses\"\n }\n ]\n}\n" }, { "path": "domovoi/examples/sns-event.json", "content": "{\n \"Records\": [\n {\n \"EventVersion\": \"1.0\",\n \"EventSubscriptionArn\": \"arn:aws:sns:us-east-2:123456789012:test-lambda:21be56ed-a058-49f5-8c98-aedd2564c486\",\n \"EventSource\": \"aws:sns\",\n \"Sns\": {\n \"SignatureVersion\": \"1\",\n \"Timestamp\": \"1970-01-01T00:00:00.000Z\",\n \"Signature\": \"tcc6faL2yUC6dgZdmrwh1Y4cGa/ebXEkAi6RibDsvpi+tE/1+82j...65r==\",\n \"SigningCertUrl\": \"https://sns.us-east-2.amazonaws.com/SimpleNotificationService-ac565b8b1a6c5d002d285f9598aa1d9b.pem\",\n \"MessageId\": \"95df01b4-ee98-5cb9-9903-4c221d41eb5e\",\n \"Message\": \"Hello from SNS!\",\n \"MessageAttributes\": {\n \"Test\": {\n \"Type\": \"String\",\n \"Value\": \"TestString\"\n },\n \"TestBinary\": {\n \"Type\": \"Binary\",\n \"Value\": \"TestBinary\"\n }\n },\n \"Type\": \"Notification\",\n \"UnsubscribeUrl\": \"https://sns.us-east-2.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=arn:aws:sns:us-east-2:123456789012:test-lambda:21be56ed-a058-49f5-8c98-aedd2564c486\",\n \"TopicArn\": \"{{topicarn}}\",\n \"Subject\": \"TestInvoke\"\n }\n }\n ]\n}\n" }, { "path": "domovoi/examples/sqs-event.json", "content": "{\n \"Records\": [\n {\n \"messageId\": \"059f36b4-87a3-44ab-83d2-661975830a7d\",\n \"receiptHandle\": \"AQEBwJnKyrHigUMZj6rYigCgxlaS3SLy0a...\",\n \"body\": \"test\",\n \"attributes\": {\n \"ApproximateReceiveCount\": \"1\",\n \"SentTimestamp\": \"1545082649183\",\n \"SenderId\": \"AIDAIENQZJOLO23YVJ4VO\",\n \"ApproximateFirstReceiveTimestamp\": \"1545082649185\"\n },\n \"messageAttributes\": {},\n \"md5OfBody\": \"098f6bcd4621d373cade4e832627b4f6\",\n \"eventSource\": \"aws:sqs\",\n \"eventSourceARN\": \"arn:aws:sqs:us-east-2:123456789012:my-queue\",\n \"awsRegion\": \"us-east-2\"\n },\n {\n \"messageId\": \"2e1424d4-f796-459a-8184-9c92662be6da\",\n \"receiptHandle\": \"AQEBzWwaftRI0KuVm4tP+/7q1rGgNqicHq...\",\n \"body\": \"test\",\n \"attributes\": {\n \"ApproximateReceiveCount\": \"1\",\n \"SentTimestamp\": \"1545082650636\",\n \"SenderId\": \"AIDAIENQZJOLO23YVJ4VO\",\n \"ApproximateFirstReceiveTimestamp\": \"1545082650649\"\n },\n \"messageAttributes\": {},\n \"md5OfBody\": \"098f6bcd4621d373cade4e832627b4f6\",\n \"eventSource\": \"aws:sqs\",\n \"eventSourceARN\": \"arn:aws:sqs:us-east-2:123456789012:my-queue\",\n \"awsRegion\": \"us-east-2\"\n }\n ]\n}\n" }, { "path": "domovoi/examples/state_machine_app.py", "content": "#!/usr/bin/env python3.6\n\nimport os, sys, json, time, random, signal, base64, pickle, zlib\nimport boto3, domovoi\n\napp = domovoi.Domovoi()\n\nsfn = {\n \"Comment\": \"\"\"\n This is a Domovoi integrated AWS Step Functions state machine. It uses AWS Lambda as the task executor.\n Domovoi will replace the Resource field of all Task states with the ARN of the appropriate lambda function managed\n by Domovoi.\n See AWS documentation of the state machine language here:\n http://docs.aws.amazon.com/step-functions/latest/dg/concepts-amazon-states-language.html\n See AWS documentation of *Choice* state conditionals here:\n http://docs.aws.amazon.com/step-functions/latest/dg/amazon-states-language-choice-state.html\n The *Sleep* state can be used to wait on other events without busy-waiting.\n Use the following command to invoke the state machine:\n $ aws stepfunctions start-execution --state-machine-arn ARN --input '{\"x\": 1}',\n where ARN is displayed in the result of `domovoi deploy`.\n Use the following command to monitor execution of the state machine:\n $ aws stepfunctions get-execution-history --execution-arn EXECUTION_ARN,\n where EXECUTION_ARN is displayed in the result of `aws stepfunctions start-execution`.\n State machine input is passed directly in the `event` argument to the task handlers. There is a 32KB I/O size limit.\n The name of the task state that the handler was called from is available via `context.stepfunctions_task_name`.\n \"\"\",\n \"StartAt\": \"Worker\",\n \"States\": {\n \"Worker\": {\n \"Type\": \"Task\",\n \"Resource\": None, # This will be set by Domovoi to the Lambda ARN\n \"Next\": \"Branch\"\n },\n \"Branch\": {\n \"Type\": \"Choice\",\n \"Choices\": [{\n \"Variable\": \"$.finished\",\n \"BooleanEquals\": True,\n \"Next\": \"Sleep\"\n }],\n \"Default\": \"Worker\"\n },\n \"Sleep\": {\n \"Type\": \"Wait\",\n # This is a delay step that can be set by the worker lambda to avoid busy-waiting.\n \"SecondsPath\": \"$.sleep_seconds\",\n \"Next\": \"Finalizer\"\n },\n \"Finalizer\": {\n \"Type\": \"Task\",\n \"Resource\": None, # This will be set by Domovoi to the Lambda ARN\n \"End\": True\n }\n }\n}\n\n\nclass DomovoiTimeout(Exception):\n pass\n\n\nclass Worker:\n def run(self, x):\n # The run() function should save its work in progress in attributes attached to self.\n # If the Lambda function runs out of time, the worker instance is pickled and restored when the lambda is\n # restarted, but all other state is lost.\n self.x = getattr(self, \"x\", 0) + x\n if random.random() < 0.8:\n while True:\n # This represents some long-running task that may not be interruptible from within Python.\n time.sleep(9000)\n return dict(x=self.x, sleep_seconds=random.randrange(8))\n\n\n@app.step_function_task(state_name=\"Worker\", state_machine_definition=sfn)\ndef do_work(event, context):\n def alarm_handler(signum, frame):\n raise DomovoiTimeout(\"Time to save state\")\n\n signal.signal(signal.SIGALRM, alarm_handler)\n timeout_seconds = (context.get_remaining_time_in_millis() / 1000) - 10\n context.log(\"Setting timeout to {}\".format(timeout_seconds))\n signal.alarm(timeout_seconds)\n\n if \"state\" in event:\n worker = pickle.loads(zlib.decompress(base64.b64decode(event[\"state\"])))\n else:\n worker = Worker()\n\n try:\n result = worker.run(event[\"x\"])\n except DomovoiTimeout:\n event.update(state=base64.b64encode(zlib.compress(pickle.dumps(worker))).decode(), finished=False)\n return event\n\n event.update(result, finished=True)\n return event\n\n\n@app.step_function_task(state_name=\"Finalizer\", state_machine_definition=sfn)\ndef finish_work(event, context):\n return {\"result\": event[\"x\"]}\n" }, { "path": "domovoi/examples/state_machine_threadpool_app.py", "content": "#!/usr/bin/env python3.6\n\nimport os, sys, json, time, random, signal, base64, pickle, zlib\nimport boto3, domovoi\n\napp = domovoi.Domovoi()\n\nsfn = {\n \"Comment\": \"\"\"\n This is a Domovoi integrated AWS Step Functions state machine using a threadpool pattern.\n See https://github.com/kislyuk/domovoi/blob/master/domovoi/examples/state_machine_app.py for more information on\n this state machine.\n \"\"\",\n \"StartAt\": \"Scatter\",\n \"States\": {\n \"Scatter\": {\n \"Type\": \"Task\",\n \"Resource\": None, # This will be set by Domovoi to the Lambda ARN\n \"Next\": \"Threadpool\"\n },\n \"Threadpool\": {\n \"Type\": \"Parallel\",\n \"Branches\": [], # This will be filled in with an array of \"thread\" state machines below\n \"Next\": \"Finalizer\"\n },\n \"Finalizer\": {\n \"Type\": \"Task\",\n \"Resource\": None, # This will be set by Domovoi to the Lambda ARN\n \"End\": True\n }\n }\n}\n\nsfn_thread = {\n \"StartAt\": \"Worker{t}\",\n \"States\": {\n \"Worker{t}\": {\n \"Type\": \"Task\",\n \"Resource\": None, # This will be set by Domovoi to the Lambda ARN\n \"Next\": \"Branch{t}\"\n },\n \"Branch{t}\": {\n \"Type\": \"Choice\",\n \"Choices\": [{\n \"Variable\": \"$.finished\",\n \"BooleanEquals\": True,\n \"Next\": \"EndThread{t}\"\n }],\n \"Default\": \"Worker{t}\"\n },\n \"EndThread{t}\": {\n \"Type\": \"Pass\",\n \"End\": True\n }\n }\n}\n\nnum_threads = 64\n\n\nclass DomovoiTimeout(Exception):\n pass\n\n\n@app.step_function_task(state_name=\"Scatter\", state_machine_definition=sfn)\ndef scatter(event, context):\n # The scatter function should initialize and partition work between workers.\n # Each worker will receive the same event payload. You can change this using the state machine I/O processing\n # directives described in\n # http://docs.aws.amazon.com/step-functions/latest/dg/amazon-states-language-input-output-processing.html,\n # or distribute work out-of-band through an SQS queue or similar.\n # Workers can introspect their state name (which contains the \"thread ID\") via context.invoked_function_arn.\n return event\n\n\nclass Worker:\n def run(self, x):\n # The run() function should save its work in progress in attributes attached to self.\n # If the Lambda function runs out of time, the worker instance is pickled and restored when the lambda is\n # restarted, but all other state is lost.\n self.x = getattr(self, \"x\", 0) + x\n if random.random() < 0.8:\n while True:\n # This represents some long-running task that may not be interruptible from within Python.\n time.sleep(9000)\n return dict(x=self.x)\n\n\ndef do_work(event, context):\n def alarm_handler(signum, frame):\n raise DomovoiTimeout(\"Time to save state\")\n\n signal.signal(signal.SIGALRM, alarm_handler)\n timeout_seconds = int(context.get_remaining_time_in_millis() / 1000) - 8\n context.log(\"Setting timeout to {}\".format(timeout_seconds))\n signal.alarm(timeout_seconds)\n\n if \"state\" in event:\n worker = pickle.loads(zlib.decompress(base64.b64decode(event[\"state\"])))\n else:\n worker = Worker()\n\n try:\n result = worker.run(event[\"x\"])\n except DomovoiTimeout:\n event.update(state=base64.b64encode(zlib.compress(pickle.dumps(worker))).decode(), finished=False)\n return event\n\n event.update(result, finished=True)\n return event\n\n\n# Construct the threadpool definition by explicitly mentioning each thread in the state machine definition.\nfor t in range(num_threads):\n thread = json.loads(json.dumps(sfn_thread).replace(\"{t}\", str(t)))\n sfn[\"States\"][\"Threadpool\"][\"Branches\"].append(thread)\n app.step_function_task(state_name=\"Worker{}\".format(t), state_machine_definition=sfn)(do_work)\n\n\n@app.step_function_task(state_name=\"Finalizer\", state_machine_definition=sfn)\ndef finish_work(event, context):\n # The finalizer - the state after the \"Parallel\" (Threadpool) state - receives the parallel execution results as an\n # array. The finalizer can do things like aggregate results from the array and do any post-processing actions.\n return {\"result\": event}\n" }, { "path": "domovoi/utils.py", "content": "import os, inspect\n\nimport attr, boto3\nimport chalice.deploy.models\nfrom chalice.deploy.packager import LambdaDeploymentPackager\nfrom chalice.cli.factory import create_botocore_session\n\nimport domovoi\n\n\nclass DomovoiDeploymentPackager(LambdaDeploymentPackager):\n _CHALICE_LIB_DIR = \"domovoilib\"\n\n def _add_app_files(self, zip_fileobj, project_dir):\n domovoi_router = inspect.getfile(domovoi.app)\n if domovoi_router.endswith(\".pyc\"):\n domovoi_router = domovoi_router[:-1]\n zip_fileobj.write(domovoi_router, \"domovoi/app.py\")\n\n domovoi_init = inspect.getfile(domovoi)\n if domovoi_init.endswith(\".pyc\"):\n domovoi_init = domovoi_init[:-1]\n zip_fileobj.write(domovoi_init, \"domovoi/__init__.py\")\n\n chalice_router = inspect.getfile(chalice.app)\n if chalice_router.endswith(\".pyc\"):\n chalice_router = chalice_router[:-1]\n zip_fileobj.write(chalice_router, \"chalice/app.py\")\n\n chalice_init = inspect.getfile(chalice)\n if chalice_init.endswith(\".pyc\"):\n chalice_init = chalice_init[:-1]\n zip_fileobj.write(chalice_init, \"chalice/__init__.py\")\n\n zip_fileobj.write(os.path.join(project_dir, \"app.py\"), \"app.py\")\n self._add_chalice_lib_if_needed(project_dir, zip_fileobj)\n\n def _needs_latest_version(self, filename):\n return filename == 'app.py' or filename.startswith(('domovoilib/', 'domovoi/'))\n\n def create_deployment_package(self, project_dir, python_version, package_filename=None):\n deployment_package_filename = self.deployment_package_filename(project_dir, python_version)\n if os.path.exists(deployment_package_filename):\n self.inject_latest_app(deployment_package_filename, project_dir)\n return deployment_package_filename\n else:\n return LambdaDeploymentPackager.create_deployment_package(self, project_dir, python_version,\n package_filename=package_filename)\n\n\n@attr.attrs\nclass ManagedIAMRole(chalice.deploy.models.ManagedIAMRole):\n def __attrs_post_init__(self):\n self.role_name = self.role_name.rpartition(\"-\")[0]\n\n\n@attr.attrs\nclass LambdaFunction(chalice.deploy.models.LambdaFunction):\n def __attrs_post_init__(self):\n self.function_name = self.function_name.rpartition(\"-\")[0]\n\n\ndef patch_chalice():\n chalice.deploy.packager.LambdaDeploymentPackager = DomovoiDeploymentPackager\n chalice.deploy.deployer.LambdaDeploymentPackager = DomovoiDeploymentPackager\n chalice.deploy.models.ManagedIAMRole = ManagedIAMRole\n chalice.deploy.models.LambdaFunction = LambdaFunction\n\n\ndef add_filter_config(event_config, event_handler):\n cfg = dict(event_config)\n for fltr in \"prefix\", \"suffix\":\n if event_handler.get(fltr):\n cfg.setdefault(\"Filter\", dict(Key=dict(FilterRules=[])))\n cfg[\"Filter\"][\"Key\"][\"FilterRules\"].append(dict(Name=fltr, Value=event_handler[fltr]))\n return cfg\n\n\ndef get_boto3_session(user_agent_extra, profile, debug):\n botocore_session = create_botocore_session(profile=profile, debug=debug)\n botocore_session.user_agent_extra = user_agent_extra\n return boto3.session.Session(botocore_session=botocore_session)\n\n\nclass DomovoiLambdaManager:\n def __init__(self, function_name, awslambda_client):\n self.function_name = function_name\n self.awslambda = awslambda_client\n\n def put_event_source_mapping(self, event_source_arn, source_data, dry_run=False):\n event_source_mapping_args = dict(EventSourceArn=event_source_arn,\n FunctionName=self.function_name,\n Enabled=True)\n if \"dynamodb\" in event_source_arn:\n event_source_mapping_args.update(StartingPosition=\"TRIM_HORIZON\")\n if source_data[\"batch_size\"] is not None:\n event_source_mapping_args.update(BatchSize=source_data[\"batch_size\"])\n esm = None\n try:\n if not dry_run:\n esm = self.awslambda.create_event_source_mapping(**event_source_mapping_args)\n except self.awslambda.exceptions.ResourceConflictException as e:\n assert \"already exists\" in str(e) and str(e).split()[-2] == \"UUID\"\n if source_data[\"batch_size\"] is not None:\n esm_uuid = str(e).split()[-1]\n esm = self.awslambda.get_event_source_mapping(UUID=esm_uuid)\n if source_data[\"batch_size\"] != esm[\"BatchSize\"]:\n esm = self.awslambda.update_event_source_mapping(UUID=esm_uuid, BatchSize=source_data[\"batch_size\"])\n return esm\n" }, { "path": "scripts/domovoi", "content": "#!/usr/bin/env python\n\nfrom __future__ import absolute_import, division, print_function, unicode_literals\n\nimport os, argparse, json, hashlib, time, copy, shutil\n\nimport botocore, boto3.session\nimport chalice, chalice.app, chalice.awsclient, chalice.deploy.packager, chalice.deploy.deployer, chalice.deploy.models\nfrom chalice.cli.factory import CLIFactory\nfrom chalice.deploy.deployer import create_default_deployer\nfrom chalice.utils import UI\nfrom chalice.compat import urlparse\nfrom chalice.constants import DEFAULT_STAGE_NAME\n\nimport domovoi\nfrom domovoi.utils import patch_chalice, add_filter_config, DomovoiLambdaManager, get_boto3_session\n\ntry:\n import pkg_resources\n __version__ = pkg_resources.get_distribution(\"domovoi\").version\nexcept Exception:\n __version__ = \"0.0.0\"\n\npatch_chalice()\n\nparser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawTextHelpFormatter)\nparser.add_argument(\"--stage\", default=DEFAULT_STAGE_NAME)\nparser.add_argument(\"--profile\")\nparser.add_argument(\"--debug\", action=\"store_true\")\nparser.add_argument(\"--dry-run\", action=\"store_true\")\nparser.add_argument(\"--version\", action=\"version\", version=\"domovoi {}\".format(__version__))\nparser.add_argument(\"action\", choices={\"deploy\", \"new-project\"})\nparser.add_argument(\"project_dir\", nargs=\"?\", default=os.getcwd())\nargs = parser.parse_args()\n\nexample_app = \"\"\"# This is an example entry point for an app built with Domovoi, an AWS Lambda event handler manager.\n# See https://github.com/kislyuk/domovoi for Domovoi's documentation.\n\nimport json, boto3, domovoi\n\napp = domovoi.Domovoi()\n\n@app.scheduled_function(\"rate(1 minute)\")\ndef test(event, context):\n pass\n\"\"\"\n\nif args.action == \"new-project\":\n from chalice.cli import click, create_new_project_skeleton\n from chalice.policy import PolicyBuilder\n if args.project_dir == os.getcwd():\n args.project_dir = click.prompt(\"New Domovoi project name\")\n assert not os.path.isdir(args.project_dir)\n create_new_project_skeleton(args.project_dir, profile=None)\n with open(os.path.join(args.project_dir, \"app.py\"), \"w\") as fh:\n fh.write(example_app)\n default_iam_policy_filename = os.path.join(os.path.dirname(domovoi.__file__), \"default_iam_policy.json\")\n stage_iam_policy_filename = os.path.join(args.project_dir, \".chalice\", \"policy-{}.json\".format(DEFAULT_STAGE_NAME))\n shutil.copy(default_iam_policy_filename, stage_iam_policy_filename)\n parser.exit(status=0, message=\"New Domovoi project created in {}\\n\".format(args.project_dir))\n\nboto3_session = get_boto3_session(user_agent_extra=\"domovoi/%s\" % __version__, profile=args.profile, debug=args.debug)\nevents = boto3_session.client(\"events\")\nsns = boto3_session.resource(\"sns\")\nsqs = boto3_session.resource(\"sqs\")\nawslambda = boto3_session.client(\"lambda\")\ns3 = boto3_session.resource(\"s3\")\nsts = boto3_session.client(\"sts\")\nsfn = boto3.client(\"stepfunctions\")\niam = boto3_session.resource(\"iam\")\nlogs = boto3_session.client(\"logs\")\ndynamodb = boto3_session.resource(\"dynamodb\")\nec2 = boto3_session.resource(\"ec2\")\nelbv2 = boto3_session.client(\"elbv2\")\nacm = boto3_session.client(\"acm\")\nroute53 = boto3_session.client(\"route53\")\n\ntrust_statement = copy.deepcopy(chalice.constants.LAMBDA_TRUST_POLICY[\"Statement\"][0])\ntrust_statement[\"Principal\"] = {\"Service\": urlparse(sfn.meta.endpoint_url).netloc}\nchalice.constants.LAMBDA_TRUST_POLICY[\"Statement\"].append(trust_statement)\n\nconfig = CLIFactory(args.project_dir).create_config_obj(chalice_stage_name=args.stage, autogen_policy=False)\nconfig._user_provided_params.setdefault(\"tags\", {})\nconfig._user_provided_params[\"tags\"][\"domovoi\"] = \"version={}\".format(__version__)\ndeployer = create_default_deployer(session=boto3_session._session,\n config=config,\n ui=UI(confirm=lambda *args, **kwargs: True))\nfunction_name = '%s-%s' % (config.app_name, args.stage)\nfunction_manager = DomovoiLambdaManager(function_name=function_name, awslambda_client=awslambda)\n\nif args.dry_run:\n deployed_values, lambda_arn = dict(resources=[]), \"arn:aws:lambda:::\"\nelse:\n deployed_values = deployer.deploy(config, chalice_stage_name=args.stage)\n for resource in deployed_values[\"resources\"]:\n if resource[\"resource_type\"] == \"lambda_function\":\n lambda_arn = resource[\"lambda_arn\"]\n\n client = chalice.awsclient.TypedAWSClient(boto3_session._session)._client(\"lambda\")\n fn_updates = dict(Description=config._chain_lookup(\"description\") or \"Domovoi event handler\")\n dlq_arn = config._chain_lookup('dead_letter_queue_target_arn')\n if dlq_arn:\n print(\"Setting DLQ {} for {}\".format(dlq_arn, function_name))\n fn_updates.update(DeadLetterConfig=dict(TargetArn=dlq_arn))\n client.update_function_configuration(FunctionName=function_name, **fn_updates)\n reserved_concurrent_executions = config._chain_lookup('reserved_concurrent_executions')\n if reserved_concurrent_executions:\n print(\"Setting concurrency reservation {} for {}\".format(reserved_concurrent_executions, function_name))\n client.put_function_concurrency(FunctionName=function_name,\n ReservedConcurrentExecutions=reserved_concurrent_executions)\n alb_acm_cert_dns_name = config._chain_lookup('alb_acm_cert_dns_name')\n\ndomovoi_app = config.chalice_app\n\n# TODO: consider narrowing trust policy\nfor service in (\"apigateway\", \"events\", \"sns\", \"sqs\", \"s3\", \"logs\", \"elasticloadbalancing\"):\n service_uri = service + \".amazonaws.com\"\n policy = dict(FunctionName=lambda_arn, Principal=service_uri, Action=\"lambda:InvokeFunction\")\n policy_id = \"domovoi-{}\".format(hashlib.md5(json.dumps(policy).encode()).hexdigest()[:8])\n print(\"Granting {} access to invoke Lambda function {}\".format(service, lambda_arn))\n if not args.dry_run:\n try:\n awslambda.add_permission(StatementId=policy_id, **policy)\n except awslambda.exceptions.ResourceConflictException:\n print(\"Found existing permission grant statement {}, skipping\".format(policy_id))\n\n\ndef find_acm_cert(dns_name):\n for page in acm.get_paginator(\"list_certificates\").paginate(CertificateStatuses=[\"ISSUED\"]):\n for cert in page[\"CertificateSummaryList\"]:\n if cert[\"DomainName\"] == dns_name:\n return cert\n raise Exception(\"Unable to find ACM certificate for {}\".format(dns_name))\n\n\ndef ensure_ingress_rule(security_group, **kwargs):\n cidr_ip = kwargs.pop(\"CidrIp\")\n for rule in security_group.ip_permissions:\n ip_range_matches = any(cidr_ip == ip_range[\"CidrIp\"] for ip_range in rule[\"IpRanges\"])\n opts_match = all(rule.get(arg) == kwargs[arg] for arg in kwargs)\n if ip_range_matches and opts_match:\n break\n else:\n security_group.authorize_ingress(CidrIp=cidr_ip, **kwargs)\n\n\ndef find_route53_zone_for_name(dns_name):\n best_zone = None\n for page in route53.get_paginator(\"list_hosted_zones\").paginate():\n for zone in page[\"HostedZones\"]:\n if dns_name.endswith(zone[\"Name\"].rstrip(\".\")):\n if best_zone is None or len(best_zone[\"Name\"]) < len(zone[\"Name\"]):\n best_zone = zone\n return best_zone\n\n\ndef update_alias_dns_record(source_dns_name, target_dns_name, hosted_zone_id):\n zone = find_route53_zone_for_name(source_dns_name)\n zone_update = {\n \"Action\": \"UPSERT\",\n \"ResourceRecordSet\": {\n \"Name\": source_dns_name,\n \"Type\": \"A\",\n \"AliasTarget\": {\n 'HostedZoneId': hosted_zone_id,\n 'DNSName': target_dns_name,\n 'EvaluateTargetHealth': False\n }\n }\n }\n route53.change_resource_record_sets(HostedZoneId=zone[\"Id\"], ChangeBatch=dict(Changes=[zone_update]))\n\n\ndef register_deployed_resource(resource_type, **kwargs):\n deployed_values[\"resources\"].append(dict(kwargs, resource_type=resource_type))\n deployer._recorder.record_results(deployed_values, args.stage, args.project_dir)\n\n\nif domovoi_app.alb_targets:\n if not args.dry_run:\n if not alb_acm_cert_dns_name:\n raise Exception('Please set the \"alb_acm_cert_dns_name\" config key in your .chalice/config.json '\n 'to the DNS name of a validated ACM certificate in your account')\n for vpc in ec2.vpcs.filter(Filters=[dict(Name=\"isDefault\", Values=[\"true\"])]):\n break\n else:\n raise Exception(\"A default VPC is required\")\n security_group_name = \"domovoi-{}\".format(function_name)\n try:\n security_group = vpc.create_security_group(GroupName=security_group_name,\n Description=\"Automatically managed by Domovoi for Lambda ALB\")\n except botocore.exceptions.ClientError as e:\n if \"InvalidGroup.Duplicate\" not in str(e):\n raise\n security_group = list(vpc.security_groups.filter(GroupNames=[security_group_name]))[0]\n\n ensure_ingress_rule(security_group, IpProtocol=\"tcp\", FromPort=443, ToPort=443, CidrIp=\"0.0.0.0/0\")\n\n for prefix, handler in domovoi_app.alb_targets.items():\n res = elbv2.create_load_balancer(Name=function_name,\n Subnets=[subnet.id for subnet in vpc.subnets.all()],\n SecurityGroups=[security_group.id])\n alb = res[\"LoadBalancers\"][0]\n print(\"Using ALB\", alb[\"LoadBalancerArn\"])\n res = elbv2.create_target_group(Name=function_name, TargetType=\"lambda\")\n target_group = res[\"TargetGroups\"][0]\n print(\"Using target group\", target_group[\"TargetGroupArn\"])\n cert = find_acm_cert(alb_acm_cert_dns_name)\n print(\"Using ACM certificate\", cert[\"CertificateArn\"])\n default_action = dict(Type=\"forward\", TargetGroupArn=target_group[\"TargetGroupArn\"])\n res = elbv2.create_listener(LoadBalancerArn=alb[\"LoadBalancerArn\"],\n Protocol=\"HTTPS\",\n Port=443,\n Certificates=[dict(CertificateArn=cert[\"CertificateArn\"])],\n DefaultActions=[default_action])\n listener = res[\"Listeners\"][0]\n print(\"Using listener\", listener[\"ListenerArn\"])\n res = elbv2.register_targets(TargetGroupArn=target_group[\"TargetGroupArn\"], Targets=[dict(Id=lambda_arn)])\n print(\"Updating the Route53 ALIAS DNS record for {} to {}\".format(alb_acm_cert_dns_name, alb[\"DNSName\"]))\n update_alias_dns_record(alb_acm_cert_dns_name, alb[\"DNSName\"], hosted_zone_id=alb[\"CanonicalHostedZoneId\"])\n\nfor task_name, task in domovoi_app.cloudwatch_events_rules.items():\n print(\"Scheduling\", task_name, \"to run on schedule\", task[\"schedule_expression\"], \"pattern\", task[\"event_pattern\"])\n rule_args = dict(Name=task_name)\n if task.get(\"schedule_expression\"):\n rule_args[\"ScheduleExpression\"] = task[\"schedule_expression\"]\n if task.get(\"event_pattern\"):\n rule_args[\"EventPattern\"] = json.dumps(task[\"event_pattern\"])\n if not args.dry_run:\n rule_arn = events.put_rule(**rule_args)[\"RuleArn\"]\n lambda_input = '{\"task_name\": \"%s\", \"event\": }' % task_name\n ixform = dict(InputPathsMap=dict(event=\"$\"), InputTemplate=lambda_input)\n events.put_targets(Rule=task_name, Targets=[dict(Id=task_name, Arn=lambda_arn, InputTransformer=ixform)])\n print(\"Scheduled CloudWatch event\", rule_arn)\n\nfor sns_topic, event_handler in domovoi_app.sns_subscribers.items():\n print(\"Subscribing\", event_handler, \"to SNS topic\", sns_topic)\n if not args.dry_run:\n topic = sns.create_topic(Name=sns_topic)\n subscription = topic.subscribe(Protocol=\"lambda\", Endpoint=lambda_arn)\n print(\"Subscribed to\", subscription)\n\nfor sqs_queue, source_data in domovoi_app.sqs_subscribers.items():\n print(\"Subscribing\", source_data[\"func\"], \"to SQS queue\", sqs_queue)\n queue_attributes = dict(domovoi_app.sqs_default_queue_attributes)\n queue_attributes.update(source_data[\"queue_attributes\"] or {})\n if not args.dry_run:\n queue = sqs.create_queue(QueueName=sqs_queue)\n queue_arn = queue.attributes[\"QueueArn\"]\n queue.set_attributes(Attributes=queue_attributes)\n function_manager.put_event_source_mapping(event_source_arn=queue_arn,\n source_data=source_data,\n dry_run=args.dry_run)\n\n\ndef ensure_queue(queue_name, sender_arn, event_handler):\n sqs_queue = sqs.create_queue(QueueName=queue_name)\n policy = {\"Statement\": [{\"Action\": [\"SQS:SendMessage\"],\n \"Effect\": \"Allow\",\n \"Resource\": sqs_queue.attributes[\"QueueArn\"],\n \"Principal\": {\"AWS\": \"*\"},\n \"Condition\": {\"ArnLike\": {\"aws:SourceArn\": sender_arn}}}]}\n policy[\"Statement\"][0][\"Sid\"] = \"domovoi-{}\".format(hashlib.md5(json.dumps(policy).encode()).hexdigest()[:8])\n queue_attributes = dict(domovoi_app.sqs_default_queue_attributes)\n queue_attributes.update(event_handler[\"sqs_queue_attributes\"] or {}, Policy=json.dumps(policy))\n sqs_queue.set_attributes(Attributes=queue_attributes)\n return sqs_queue\n\n\ndef ensure_topic(topic_name, s3_bucket):\n sns_topic = sns.create_topic(Name=topic_name)\n policy = {\"Statement\": [{\"Action\": [\"SNS:Publish\"],\n \"Effect\": \"Allow\",\n \"Resource\": sns_topic.arn,\n \"Principal\": {\"Service\": [\"s3.amazonaws.com\"]},\n \"Condition\": {\"ArnLike\": {\"aws:SourceArn\": \"arn:aws:s3:*:*:\" + s3_bucket}}}]}\n policy[\"Statement\"][0][\"Sid\"] = \"domovoi-{}\".format(hashlib.md5(json.dumps(policy).encode()).hexdigest()[:8])\n sns_topic.set_attributes(AttributeName=\"Policy\", AttributeValue=json.dumps(policy))\n return sns_topic\n\n\nfor s3_bucket, event_handler in domovoi_app.s3_subscribers.items():\n print(\"Subscribing\", event_handler[\"func\"], \"to events in S3 bucket\", s3_bucket)\n if args.dry_run:\n continue\n if event_handler[\"use_sqs\"] and event_handler[\"use_sns\"]:\n # An SNS-SQS bridge is the only option for subscribing multiple Lambdas to the same S3 event type with SQS\n topic_name = \"domovoi-s3-events-{}\".format(s3_bucket.replace(\".\", \"_\"))\n sns_topic = ensure_topic(topic_name, s3_bucket)\n queue_name = \"domovoi-s3-events-{}-{}\".format(s3_bucket.replace(\".\", \"_\"), function_name)\n sqs_queue = ensure_queue(queue_name, sender_arn=\"arn:aws:sns:*:*:\" + topic_name, event_handler=event_handler)\n queue_arn = sqs_queue.attributes[\"QueueArn\"]\n subscription = sns_topic.subscribe(Protocol=\"sqs\", Endpoint=queue_arn)\n print(\"Subscribed\", queue_arn, \"to\", subscription)\n esm = function_manager.put_event_source_mapping(event_source_arn=queue_arn,\n source_data=dict(batch_size=event_handler[\"sqs_batch_size\"]))\n print(\"Created event source mapping\", esm[\"UUID\"])\n topic_configuration = dict(TopicArn=sns_topic.arn, Events=event_handler[\"events\"])\n topic_configuration = add_filter_config(topic_configuration, event_handler)\n elif event_handler[\"use_sqs\"]:\n queue_name = \"domovoi-s3-events-{}-{}\".format(s3_bucket.replace(\".\", \"_\"), function_name)\n sqs_queue = ensure_queue(queue_name, sender_arn=\"arn:aws:s3:*:*:\" + s3_bucket, event_handler=event_handler)\n queue_arn = sqs_queue.attributes[\"QueueArn\"]\n esm = function_manager.put_event_source_mapping(event_source_arn=queue_arn,\n source_data=dict(batch_size=event_handler[\"sqs_batch_size\"]))\n print(\"Created event source mapping\", esm[\"UUID\"])\n queue_configuration = dict(QueueArn=queue_arn, Events=event_handler[\"events\"])\n queue_configuration = add_filter_config(queue_configuration, event_handler)\n elif event_handler[\"use_sns\"]:\n topic_name = \"domovoi-s3-events-{}\".format(s3_bucket.replace(\".\", \"_\"))\n sns_topic = ensure_topic(topic_name, s3_bucket)\n subscription = sns_topic.subscribe(Protocol=\"lambda\", Endpoint=lambda_arn)\n print(\"Subscribed\", lambda_arn, \"to\", subscription)\n topic_configuration = dict(TopicArn=sns_topic.arn, Events=event_handler[\"events\"])\n topic_configuration = add_filter_config(topic_configuration, event_handler)\n else:\n lambda_function_configuration = dict(LambdaFunctionArn=lambda_arn, Events=event_handler[\"events\"])\n lambda_function_configuration = add_filter_config(lambda_function_configuration, event_handler)\n for t in range(8):\n try:\n notification, last_exception = s3.Bucket(s3_bucket).Notification(), None\n new_config = dict(LambdaFunctionConfigurations=notification.lambda_function_configurations or [],\n QueueConfigurations=notification.queue_configurations or [],\n TopicConfigurations=notification.topic_configurations or [])\n if event_handler[\"use_sqs\"] and not event_handler[\"use_sns\"]:\n old_cfgs = [cfg for cfg in new_config[\"QueueConfigurations\"] if cfg[\"QueueArn\"] != queue_arn]\n new_config[\"QueueConfigurations\"] = [queue_configuration] + old_cfgs\n elif event_handler[\"use_sns\"]:\n old_cfgs = [cfg for cfg in new_config[\"TopicConfigurations\"] if cfg[\"TopicArn\"] != sns_topic.arn]\n new_config[\"TopicConfigurations\"] = [topic_configuration] + old_cfgs\n else:\n old_cfgs = [cfg for cfg in new_config[\"LambdaFunctionConfigurations\"]\n if cfg[\"LambdaFunctionArn\"] != lambda_arn]\n new_config[\"LambdaFunctionConfigurations\"] = [lambda_function_configuration] + old_cfgs\n notification.put(NotificationConfiguration=new_config)\n break\n except botocore.exceptions.ClientError as e:\n if \"A conflicting conditional operation is currently in progress\" not in str(e):\n raise\n last_exception = e\n print(\"Waiting\", int(1.6**t), \"seconds for concurrent operation to complete\")\n time.sleep(1.6**t)\n else:\n raise last_exception\n\nfor cwl_log_group_name, cwl_sub_filter_data in domovoi_app.cwl_sub_filters.items():\n print(\"Subscribing\", cwl_sub_filter_data, \"to CloudWatch Logs filter for\", cwl_log_group_name)\n if not args.dry_run:\n logs.put_subscription_filter(logGroupName=cwl_log_group_name,\n filterName=\"domovoi-cwl-filter\",\n filterPattern=cwl_sub_filter_data[\"filter_pattern\"],\n destinationArn=lambda_arn)\n\nfor table_name, source_data in domovoi_app.dynamodb_event_sources.items():\n print(\"Subscribing to DynamoDB event stream for table\", table_name)\n stream_arn = dynamodb.Table(table_name).latest_stream_arn if not args.dry_run else \"arn:aws::::\"\n function_manager.put_event_source_mapping(event_source_arn=stream_arn,\n source_data=source_data,\n dry_run=args.dry_run)\n\nexisting_aliases = []\nif not args.dry_run:\n for page in awslambda.get_paginator('list_aliases').paginate(FunctionName=function_name):\n existing_aliases.extend(page[\"Aliases\"])\n\nstate_machine = None\nfor sfn_task_name, sfn_task in domovoi_app.sfn_tasks.items():\n print(\"Registering step function state machine for\", sfn_task_name)\n if state_machine is None:\n state_machine = sfn_task[\"state_machine_definition\"]\n else:\n msg = \"Multiple state machine definitions are not supported\"\n assert state_machine == sfn_task[\"state_machine_definition\"], msg\n lambda_alias = \"domovoi-stepfunctions-task-\" + sfn_task_name\n alias_args = dict(FunctionName=function_name,\n Name=lambda_alias,\n FunctionVersion=\"$LATEST\",\n Description=\"Domovoi Lambda routing label for a Step Functions state machine task\")\n all_states = domovoi.Domovoi.get_all_states(state_machine)\n state = all_states[sfn_task[\"state_name\"]]\n if not args.dry_run:\n for alias in existing_aliases:\n if alias[\"Name\"] == lambda_alias and alias[\"FunctionVersion\"] == \"$LATEST\":\n break\n else:\n try:\n awslambda.create_alias(**alias_args)\n except awslambda.exceptions.ResourceConflictException:\n awslambda.update_alias(**alias_args)\n state[\"Resource\"] = lambda_arn + \":\" + lambda_alias\n\nif state_machine and not args.dry_run:\n iam_role_arn = config.iam_role_arn or iam.Role(function_name).arn\n sm_args = dict(name=function_name,\n definition=json.dumps(state_machine),\n roleArn=iam_role_arn)\n try:\n sm = sfn.create_state_machine(**sm_args)\n print(\"Created new state machine\", sm[\"stateMachineArn\"])\n except botocore.exceptions.ClientError as e:\n for page in sfn.get_paginator(\"list_state_machines\").paginate():\n for sm in page[\"stateMachines\"]:\n if sm[\"name\"] == function_name:\n break\n if sm[\"name\"] != function_name:\n raise e\n sm = sfn.describe_state_machine(stateMachineArn=sm[\"stateMachineArn\"])\n sm_args.clear()\n if json.loads(sm[\"definition\"]) != state_machine:\n sm_args[\"definition\"] = json.dumps(state_machine)\n if sm[\"roleArn\"] != iam_role_arn:\n sm_args[\"roleArn\"] = iam_role_arn\n if sm_args:\n print(\"Updating state machine\", sm[\"stateMachineArn\"])\n sfn.update_state_machine(stateMachineArn=sm[\"stateMachineArn\"], **sm_args)\n else:\n print(\"No changes required to existing state machine\", sm[\"stateMachineArn\"])\n register_deployed_resource(\"state_machine\", name=function_name, arn=sm[\"stateMachineArn\"])\n print(\"State machine:\", sm[\"stateMachineArn\"])\n\nif args.dry_run:\n print(\"Dry run successful\")\n" }, { "path": "setup.cfg", "content": "[bdist_wheel]\nuniversal=1\n[flake8]\nmax-line-length=120\nignore: E401, F401\n" }, { "path": "setup.py", "content": "#!/usr/bin/env python\n\nimport glob\nfrom setuptools import setup, find_packages\n\nsetup(\n name=\"domovoi\",\n version=\"2.0.2\",\n url='https://github.com/kislyuk/domovoi',\n license='Apache Software License',\n author='Andrey Kislyuk',\n author_email='kislyuk@gmail.com',\n description='AWS Lambda event handler manager',\n long_description=open('README.rst').read(),\n install_requires=[\n 'boto3 >= 1.7.19, < 2',\n 'chalice >= 1.3.0, < 2'\n ],\n extras_require={\n ':python_version == \"2.7\"': ['enum34 >= 1.1.6, < 2']\n },\n packages=find_packages(exclude=['test']),\n scripts=glob.glob('scripts/*'),\n platforms=['MacOS X', 'Posix'],\n package_data={'domovoi': ['*.json']},\n zip_safe=False,\n include_package_data=True,\n test_suite='test',\n classifiers=[\n 'Intended Audience :: Developers',\n 'License :: OSI Approved :: Apache Software License',\n 'Operating System :: MacOS :: MacOS X',\n 'Operating System :: POSIX',\n 'Programming Language :: Python',\n 'Programming Language :: Python :: 2.7',\n 'Programming Language :: Python :: 3.3',\n 'Programming Language :: Python :: 3.4',\n 'Programming Language :: Python :: 3.5',\n 'Topic :: Software Development :: Libraries :: Python Modules'\n ]\n)\n" }, { "path": "test/test.py", "content": "#!/usr/bin/env python\n# coding: utf-8\n\nfrom __future__ import absolute_import, division, print_function, unicode_literals\n\nimport os, sys, unittest, tempfile, json, subprocess, shutil, textwrap\n\nsys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))\nfrom domovoi import Domovoi # noqa\n\nclass TestDomovoi(unittest.TestCase):\n def test_basic_statements(self):\n state_machine = {\n \"StartAt\": \"Worker\",\n \"States\": {\n \"Worker\": {\n \"Type\": \"Task\",\n \"Resource\": None,\n \"End\": True\n }\n }\n }\n\n subprocess.check_call([\"domovoi\", \"new-project\", \"testproject\"])\n readme_filename = os.path.join(os.path.dirname(__file__), \"..\", \"README.rst\")\n with open(readme_filename) as readme_fh, open(\"testproject/app.py\", \"w\") as app_fh:\n for line in readme_fh.readlines():\n if line.strip() == \".. code-block:: python\":\n app_fh.write(\"# Domovoi test\\nstate_machine = {}\\n\".format(state_machine))\n elif line.strip() == \"Installation\":\n break\n elif app_fh.tell():\n app_fh.write(line[4:])\n\n subprocess.check_call([\"domovoi\", \"--dry-run\", \"deploy\"], cwd=\"testproject\")\n\n def test_state_machine_examples(self):\n subprocess.check_call([\"domovoi\", \"new-project\", \"testproject-sfn\"])\n shutil.copy(os.path.join(os.path.dirname(__file__), \"..\", \"domovoi\", \"examples\", \"state_machine_app.py\"),\n os.path.join(\"testproject-sfn\", \"app.py\"))\n subprocess.check_call([\"domovoi\", \"--dry-run\", \"deploy\"], cwd=\"testproject-sfn\")\n\n def test_state_machine_registration(self):\n sm_app = \"\"\"\n import json, boto3, domovoi\n\n app = domovoi.Domovoi()\n\n def handler(event, context):\n pass\n\n state_machine = {\n \"StartAt\": \"Worker\",\n \"States\": {\n \"Worker\": {\n \"Type\": \"Task\",\n \"Resource\": handler,\n \"End\": True\n }\n }\n }\n app.register_state_machine(state_machine)\n \"\"\"\n\n subprocess.check_call([\"domovoi\", \"new-project\", \"testproject2\"])\n with open(\"testproject2/app.py\", \"w\") as app_fh:\n app_fh.write(textwrap.dedent(sm_app))\n\n subprocess.check_call([\"domovoi\", \"--dry-run\", \"deploy\"], cwd=\"testproject2\")\n\n\nif __name__ == '__main__':\n unittest.main()\n" } ]