[ { "path": ".github/FUNDING.yml", "content": "github: [apls777]\n" }, { "path": ".github/workflows/generate-docs.yml", "content": "# This workflows will upload a Python Package using Twine when a release is created\n# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries\n\nname: Generate Docs\n\non:\n push:\n branches:\n - master\n\njobs:\n update-doc:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v2\n - uses: actions/setup-python@v1\n with:\n python-version: 3.6\n\n - name: generate docs\n run: |\n cd docs\n pip install -r requirements.txt\n make html\n cd build/html\n\n touch .nojekyll\n echo \"spotty.cloud\" > CNAME\n\n git init\n git config --local user.email \"github-bot@spotty.cloud\"\n git config --local user.name \"Spotty Dev Bot\"\n git add .\n git commit -m \"generated docs\" -a\n\n - uses: ad-m/github-push-action@master\n with:\n github_token: ${{ secrets.BOT_GITHUB_TOKEN }}\n repository: spotty-cloud/website\n force: true\n directory: docs/build/html\n" }, { "path": ".github/workflows/python-publish.yml", "content": "# This workflows will upload a Python Package using Twine when a release is created\n# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries\n\nname: Upload Python Package\n\non:\n release:\n types: [created]\n\njobs:\n deploy:\n\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/checkout@v2\n - name: Set up Python\n uses: actions/setup-python@v2\n with:\n python-version: '3.6'\n - name: Install dependencies\n run: |\n python -m pip install --upgrade pip\n pip install setuptools wheel twine\n - name: Build and publish\n env:\n TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}\n TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}\n run: |\n python setup.py sdist bdist_wheel\n twine upload dist/*\n" }, { "path": ".gitignore", "content": ".idea/\nbuild/\ndist/\n*.egg-info/\n__pycache__/\ntodo\n" }, { "path": "CONTRIBUTING.md", "content": "# Contributing to Spotty\n\n**Thank you for your interest in Spotty. Your contributions are highly welcome.**\n\nThere are multiple ways of getting involved:\n\n- [Report a bug](#report-a-bug)\n- [Suggest a feature](#suggest-a-feature)\n- [Contribute code](#contribute-code)\n\nBelow are a few guidelines we would like you to follow.\nIf you need help, please reach out to us by opening an issue.\n\n## Report a bug \n\nReporting bugs is one of the best ways to contribute. Before creating a bug report, \nplease check that an [issue](https://github.com/spotty-cloud/spotty/issues) reporting the same problem does not already \nexist. If there is such an issue, you may add your information as a comment.\n\nTo report a new bug you should open an issue that summarizes the bug and set the label to \"bug\".\n\nIf you want to provide a fix along with your bug report: that is great! In this case please send us a pull request as \ndescribed in section [Contribute Code](#contribute-code).\n\n## Suggest a feature\n\nTo request a new feature you should open an [issue](https://github.com/spotty-cloud/spotty/issues/new) and summarize \nthe desired functionality and its use case. Set the issue label to \"feature\". \n\n## Contribute code\n\nThis is a rough outline of what the workflow for code contributions looks like:\n- Check the list of open [issues](https://github.com/spotty-cloud/spotty/issues). Either assign an existing issue to \nyourself or create a new one that you would like to work on and discuss your ideas and use cases. It is always best to \ndiscuss your plans beforehand, to ensure that your contribution is in line with our goals for Spotty.\n- Fork the repository on GitHub\n- Create a topic branch from where you want to base your work. This is usually the master.\n- Make commits of logical units\n- Write good commit messages (see below)\n- Push your changes to a topic branch in your fork of the repository\n- Submit a pull request to [spotty-cloud/spotty](https://github.com/spotty-cloud/spotty)\n\nThanks for your contributions!\n\n### Commit messages\n\nYour commit messages ideally can answer two questions: what changed and why. The subject line should feature \nthe \"what\" and the body of the commit should describe the \"why\".\n\nWhen creating a pull request, its comment should reference the corresponding issue ID.\n\n**Have fun and enjoy hacking!**\n" }, { "path": "LICENSE", "content": "The MIT License (MIT)\n\nCopyright (c) 2017 Oleg Polosin\n\nPermission is hereby granted, free of charge, to any person obtaining a copy of this software and associated\ndocumentation files (the \"Software\"), to deal in the Software without restriction, including without limitation the\nrights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit\npersons to whom the Software is furnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all copies or substantial portions of the\nSoftware.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE\nWARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR\nCOPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR\nOTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n" }, { "path": "README.md", "content": "\n\n[![Documentation](https://img.shields.io/badge/documentation-reference-brightgreen.svg)](https://spotty.cloud)\n[![PyPI](https://img.shields.io/pypi/v/spotty.svg)](https://pypi.org/project/spotty/)\n![PyPI - Python Version](https://img.shields.io/pypi/pyversions/spotty.svg)\n![PyPI - License](https://img.shields.io/pypi/l/spotty.svg)\n\nSpotty drastically simplifies training of deep learning models on [AWS](https://aws.amazon.com/) \nand [GCP](https://cloud.google.com/):\n\n- it makes training on GPU instances as simple as training on your local machine\n- it automatically manages all necessary cloud resources including images, volumes, snapshots and SSH keys\n- it makes your model trainable in the cloud by everyone with a couple of commands\n- it uses [tmux](https://en.wikipedia.org/wiki/Tmux) to easily detach remote processes from their terminals\n- it saves you up to 70% of the costs by using [AWS Spot Instances](https://aws.amazon.com/ec2/spot/) \nand [GCP Preemtible VMs](https://cloud.google.com/preemptible-vms/)\n\n## Documentation\n\n- See the [documentation page](https://spotty.cloud).\n- Read [this](https://medium.com/@apls/how-to-train-deep-learning-models-on-aws-spot-instances-using-spotty-8d9e0543d365) \narticle on Medium for a real-world example.\n\n## Installation\n\nRequirements:\n * Python >=3.6\n * AWS CLI (see [Installing the AWS Command Line Interface](http://docs.aws.amazon.com/cli/latest/userguide/installing.html)) \n if you're using AWS\n * Google Cloud SDK (see [Installing Google Cloud SDK](https://cloud.google.com/sdk/install)) \n if you're using GCP\n\nUse [pip](http://www.pip-installer.org/en/latest/) to install or upgrade Spotty:\n\n $ pip install -U spotty\n\n## Get Started\n\n1. Prepare a `spotty.yaml` file and put it to the root directory of your project:\n\n - See the file specification [here](https://spotty.cloud/docs/user-guide/configuration-file.html).\n - Read [this](https://medium.com/@apls/how-to-train-deep-learning-models-on-aws-spot-instances-using-spotty-8d9e0543d365) \n article for a real-world example.\n\n2. Start an instance:\n\n ```bash\n $ spotty start\n ```\n\n It will run a Spot Instance, restore snapshots if any, synchronize the project with the running instance \n and start the Docker container with the environment.\n\n3. Train a model or run notebooks.\n\n To connect to the running container via SSH, use the following command:\n\n ```bash\n $ spotty sh\n ```\n\n It runs a [tmux](https://github.com/tmux/tmux/wiki) session, so you can always detach this session using\n __`Ctrl + b`__, then __`d`__ combination of keys. To be attached to that session later, just use the\n `spotty sh` command again.\n\n Also, you can run your custom scripts inside the Docker container using the `spotty run ` command. Read more\n about custom scripts in the documentation: \n [Configuration: \"scripts\" section](https://spotty.cloud/docs/configuration-file/#scripts-section-optional).\n\n## Contributions\n\nAny feedback or contributions are welcome! Please check out the [guidelines](CONTRIBUTING.md).\n\n## License\n\n[MIT License](LICENSE)\n" }, { "path": "bin/spotty", "content": "#!/usr/bin/env python\n\nimport sys\nimport logging\nimport spotty\nfrom spotty.cli import get_parser\nfrom spotty.commands.writers.output_writrer import OutputWriter\n\n\nparser = get_parser()\n\nargs = sys.argv[1:]\noutput = OutputWriter()\n\n# display the version\nif '-V' in args:\n output.write(spotty.__version__)\n sys.exit(0)\n\n# separate Spotty arguments from custom arguments\ncustom_args = []\nif '--' in args:\n dd_idx = args.index('--')\n custom_args = args[(dd_idx + 1):]\n args = args[:dd_idx]\n\n# parse arguments\nargs = parser.parse_args(args)\nargs.custom_args = custom_args\n\n# logging\nlogging_level = logging.DEBUG if 'debug' in args and args.debug else logging.WARNING\nlogging.basicConfig(level=logging_level, format='[%(levelname)s] %(message)s')\n\nif 'command' not in args:\n parser.print_help()\n sys.exit(1)\n\n# run a command\ntry:\n args.command.run(args, output)\nexcept Exception as e:\n output.write('Error:\\n'\n '------\\n'\n '%s' % str(e))\n sys.exit(1)\n" }, { "path": "docs/Makefile", "content": "# Minimal makefile for Sphinx documentation\n#\n\n# You can set these variables from the command line.\nSPHINXOPTS =\nSPHINXBUILD = python -msphinx\nSPHINXPROJ = spotty\nSOURCEDIR = source\nBUILDDIR = build\n\n# Put it first so that \"make\" without argument is like \"make help\".\nhelp:\n\t@$(SPHINXBUILD) -M help \"$(SOURCEDIR)\" \"$(BUILDDIR)\" $(SPHINXOPTS) $(O)\n\n.PHONY: help Makefile\n\n# Catch-all target: route all unknown targets to Sphinx using the new\n# \"make mode\" option. $(O) is meant as a shortcut for $(SPHINXOPTS).\n%: Makefile\n\t@$(SPHINXBUILD) -M $@ \"$(SOURCEDIR)\" \"$(BUILDDIR)\" $(SPHINXOPTS) $(O)" }, { "path": "docs/make.bat", "content": "@ECHO OFF\n\npushd %~dp0\n\nREM Command file for Sphinx documentation\n\nif \"%SPHINXBUILD%\" == \"\" (\n\tset SPHINXBUILD=python -msphinx\n)\nset SOURCEDIR=source\nset BUILDDIR=build\nset SPHINXPROJ=spotty\n\nif \"%1\" == \"\" goto help\n\n%SPHINXBUILD% >NUL 2>NUL\nif errorlevel 9009 (\n\techo.\n\techo.The Sphinx module was not found. Make sure you have Sphinx installed,\n\techo.then set the SPHINXBUILD environment variable to point to the full\n\techo.path of the 'sphinx-build' executable. Alternatively you may add the\n\techo.Sphinx directory to PATH.\n\techo.\n\techo.If you don't have Sphinx installed, grab it from\n\techo.http://sphinx-doc.org/\n\texit /b 1\n)\n\n%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%\ngoto end\n\n:help\n%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%\n\n:end\npopd\n" }, { "path": "docs/requirements.txt", "content": "sphinx==3.1.2\nrecommonmark==0.6.0\nsphinx-argparse==0.2.5\nsphinx-rtd-theme==0.5.0\nPyYAML\nschema\nchevron\nboto3\n" }, { "path": "docs/source/_static/favicon/browserconfig.xml", "content": "\n\n \n \n \n #00aba9\n \n \n\n" }, { "path": "docs/source/_static/favicon/site.webmanifest", "content": "{\n \"name\": \"\",\n \"short_name\": \"\",\n \"icons\": [\n {\n \"src\": \"/android-chrome-192x192.png\",\n \"sizes\": \"192x192\",\n \"type\": \"image/png\"\n },\n {\n \"src\": \"/android-chrome-512x512.png\",\n \"sizes\": \"512x512\",\n \"type\": \"image/png\"\n }\n ],\n \"theme_color\": \"#ffffff\",\n \"background_color\": \"#ffffff\",\n \"display\": \"standalone\"\n}\n" }, { "path": "docs/source/_static/scripts.js", "content": "window.onload = function() {\n var links = document.querySelectorAll('a.external');\n\n for(var i = 0; i < links.length; i++) {\n links[i].target = '_blank';\n }\n}\n" }, { "path": "docs/source/_static/styles.css", "content": ".wy-nav-content {\n max-width: 1280px;\n}\n\n.wy-side-nav-search {\n background: none;\n}\n\n.wy-menu-vertical header, .wy-menu-vertical p.caption {\n color: #e44859;\n}\n\n.wy-side-nav-search > a img.logo, .wy-side-nav-search .wy-dropdown > a img.logo {\n max-width: 75%;\n}\n\n.wy-breadcrumbs a.icon-home {\n color: #e44859;\n}\n\n.section#welcome-to-spotty-documentation {\n display: none;\n}\n" }, { "path": "docs/source/conf.py", "content": "#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n#\n# spotty documentation build configuration file, created by\n# sphinx-quickstart on Fri Jul 17 16:00:08 2020.\n#\n# This file is execfile()d with the current directory set to its\n# containing dir.\n#\n# Note that not all possible configuration values are present in this\n# autogenerated file.\n#\n# All configuration values have a default; values that are commented out\n# serve to show the default.\n\n# If extensions (or modules to document with autodoc) are in another directory,\n# add these directories to sys.path here. If the directory is relative to the\n# documentation root, use os.path.abspath to make it absolute, like shown here.\n#\n# import os\n# import sys\n# sys.path.insert(0, os.path.abspath('.'))\n\nimport os\nimport sys\n\n\nsys.path.insert(0, os.path.abspath('../..'))\n\nimport spotty\n\n# -- General configuration ------------------------------------------------\n\n# If your documentation needs a minimal Sphinx version, state it here.\n#\n# needs_sphinx = '1.0'\n\n# Add any Sphinx extension module names here, as strings. They can be\n# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom\n# ones.\nextensions = [\n 'recommonmark',\n 'sphinxarg.ext',\n 'sphinx_rtd_theme',\n 'sphinx.ext.autosectionlabel',\n]\n\n# Add any paths that contain templates here, relative to this directory.\ntemplates_path = ['_templates']\n\n# The suffix(es) of source filenames.\n# You can specify multiple suffix as a list of string:\n#\n# source_suffix = ['.rst', '.md']\nsource_suffix = {\n '.rst': 'restructuredtext',\n '.md': 'markdown',\n}\n\n# The master toctree document.\nmaster_doc = 'index'\n\n# General information about the project.\nproject = 'spotty'\ncopyright = '2020, Oleg Polosin'\nauthor = 'Oleg Polosin'\n\n# The version info for the project you're documenting, acts as replacement for\n# |version| and |release|, also used in various other places throughout the\n# built documents.\n#\n# The short X.Y version.\nversion = spotty.__version__\n# The full version, including alpha/beta/rc tags.\nrelease = spotty.__version__\n\n# The language for content autogenerated by Sphinx. Refer to documentation\n# for a list of supported languages.\n#\n# This is also used if you do content translation via gettext catalogs.\n# Usually you set \"language\" from the command line for these cases.\nlanguage = None\n\n# List of patterns, relative to source directory, that match files and\n# directories to ignore when looking for source files.\n# This patterns also effect to html_static_path and html_extra_path\nexclude_patterns = []\n\n# The name of the Pygments (syntax highlighting) style to use.\npygments_style = 'sphinx'\n\n# If true, `todo` and `todoList` produce output, else they produce nothing.\ntodo_include_todos = False\n\n# to display double-dash (--) in epilogs of some Spotty commands\nsmartquotes = False\n\n# Prefix document path to section labels, otherwise autogenerated labels would look like 'heading'\n# rather than 'path/to/file:heading'\nautosectionlabel_prefix_document = True\n\n# -- Options for HTML output ----------------------------------------------\n\n# The theme to use for HTML and HTML Help pages. See the documentation for\n# a list of builtin themes.\n#\nhtml_theme = 'sphinx_rtd_theme'\n\n# Theme options are theme-specific and customize the look and feel of a theme\n# further. For a list of options available for each theme, see the\n# documentation.\n#\nhtml_theme_options = {\n 'logo_only': True,\n 'style_external_links': True,\n 'collapse_navigation': False,\n 'titles_only': True,\n}\n\n# Add any paths that contain custom static files (such as style sheets) here,\n# relative to this directory. They are copied after the builtin static files,\n# so a file named \"default.css\" will overwrite the builtin \"default.css\".\nhtml_static_path = ['_static']\n\n# Custom sidebar templates, must be a dictionary that maps document names\n# to template names.\n#\n# This is required for the alabaster theme\n# refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars\nhtml_sidebars = {\n '**': [\n 'about.html',\n 'navigation.html',\n 'relations.html', # needs 'show_related': True theme option to display\n 'searchbox.html',\n 'donate.html',\n ]\n}\n\nhtml_show_copyright = False\nhtml_show_sphinx = False\nhtml_show_sourcelink = False\n\nhtml_logo = '_static/images/logo_400x130_grey.png'\nhtml_favicon = '_static/favicon/favicon.ico'\n\nhtml_css_files = [\n 'styles.css',\n]\n\nhtml_js_files = [\n 'scripts.js',\n]\n\n# -- Options for HTMLHelp output ------------------------------------------\n\n# Output file base name for HTML help builder.\nhtmlhelp_basename = 'spottydoc'\n\n\n# -- Options for LaTeX output ---------------------------------------------\n\nlatex_elements = {\n # The paper size ('letterpaper' or 'a4paper').\n #\n # 'papersize': 'letterpaper',\n\n # The font size ('10pt', '11pt' or '12pt').\n #\n # 'pointsize': '10pt',\n\n # Additional stuff for the LaTeX preamble.\n #\n # 'preamble': '',\n\n # Latex figure (float) alignment\n #\n # 'figure_align': 'htbp',\n}\n\n# Grouping the document tree into LaTeX files. List of tuples\n# (source start file, target name, title,\n# author, documentclass [howto, manual, or own class]).\nlatex_documents = [\n (master_doc, 'spotty.tex', 'spotty Documentation',\n 'Oleg Polosin', 'manual'),\n]\n\n\n# -- Options for manual page output ---------------------------------------\n\n# One entry per manual page. List of tuples\n# (source start file, name, description, authors, manual section).\nman_pages = [\n (master_doc, 'spotty', 'spotty Documentation',\n [author], 1)\n]\n\n\n# -- Options for Texinfo output -------------------------------------------\n\n# Grouping the document tree into Texinfo files. List of tuples\n# (source start file, target name, title, author,\n# dir menu entry, description, category)\ntexinfo_documents = [\n (master_doc, 'spotty', 'spotty Documentation',\n author, 'spotty', 'One line description of project.',\n 'Miscellaneous'),\n]\n" }, { "path": "docs/source/docs/cli/spotty-aws.rst", "content": "spotty aws\n==========\n\n.. argparse::\n :nodefaultconst:\n :ref: spotty.cli.get_parser\n :prog: spotty\n :path: aws\n" }, { "path": "docs/source/docs/cli/spotty-download.rst", "content": "spotty download\n===============\n\n.. argparse::\n :nodefaultconst:\n :ref: spotty.cli.get_parser\n :prog: spotty\n :path: download\n" }, { "path": "docs/source/docs/cli/spotty-exec.rst", "content": "spotty exec\n===========\n\n.. argparse::\n :nodefaultconst:\n :ref: spotty.cli.get_parser\n :prog: spotty\n :path: exec\n" }, { "path": "docs/source/docs/cli/spotty-run.rst", "content": "spotty run\n==========\n\n.. argparse::\n :nodefaultconst:\n :ref: spotty.cli.get_parser\n :prog: spotty\n :path: run\n" }, { "path": "docs/source/docs/cli/spotty-sh.rst", "content": "spotty sh\n=========\n\n.. argparse::\n :nodefaultconst:\n :ref: spotty.cli.get_parser\n :prog: spotty\n :path: sh\n" }, { "path": "docs/source/docs/cli/spotty-start.rst", "content": "spotty start\n============\n\n.. argparse::\n :nodefaultconst:\n :ref: spotty.cli.get_parser\n :prog: spotty\n :path: start\n" }, { "path": "docs/source/docs/cli/spotty-stop.rst", "content": "spotty stop\n===========\n\n.. argparse::\n :nodefaultconst:\n :ref: spotty.cli.get_parser\n :prog: spotty\n :path: stop\n" }, { "path": "docs/source/docs/cli/spotty-sync.rst", "content": "spotty sync\n===========\n\n.. argparse::\n :nodefaultconst:\n :ref: spotty.cli.get_parser\n :prog: spotty\n :path: sync\n" }, { "path": "docs/source/docs/cli/spotty.rst", "content": "Spotty Command-line Interface\n=============================\n\n.. argparse::\n :nosubcommands:\n :nodefaultconst:\n :noepilog:\n :nodescription:\n :ref: spotty.cli.get_parser\n :prog: spotty\n\n.. toctree::\n :maxdepth: 1\n :caption: Sub-commands\n\n spotty-start\n spotty-stop\n spotty-sh\n spotty-sync\n spotty-download\n spotty-run\n spotty-exec\n\n.. toctree::\n :maxdepth: 1\n :caption: Custom provider sub-commands\n\n spotty-aws\n" }, { "path": "docs/source/docs/providers/aws/caching-docker-image-on-an-ebs-volume.md", "content": "# Caching Docker Image on an EBS Volume\n\nYou can cache images that you've built or downloaded from the internet on an EBS volume or in a snapshot.\n\nA configuration file has the \"__dockerDataRoot__\" parameter. It's a directory on the host OS where the Docker \ndaemon will save all the images.\n\nSpecify the `mountDir` directory for one of the instance volumes and set the `dockerDataRoot` parameter\nto the same value (or to a subdirectory of the `mountDir` directory). Also, consider changing a deletion policy\nfor that volume to \"__retain__\", then the volume with the cache will be retained and the next time it just will be \nattached to the instance.\n\nExample:\n```yaml\n# ...\n\ninstances:\n - name: aws-1\n provider: aws\n parameters:\n # ...\n dockerDataRoot: /docker\n volumes:\n # ...\n - name: docker\n parameters:\n size: 10\n mountDir: /docker\n```\n" }, { "path": "docs/source/docs/providers/aws/ebs-volumes-and-deletion-policies.md", "content": "# EBS Volumes and Deletion Policies\n\nBy default, EBS volumes have names in the following format: `--`.\nBut you can specify a custom name using the `volumeName` parameter. \n\nWhen you're starting an instance:\n1. Spotty is looking for existing EBS volumes using their names. If a volume exists, it will be attached to the \ninstance.\n2. If not - Spotty will be looking for a snapshot with the same name. If the snapshot exists, the volume will be \nrestored from that snapshot.\n3. If neither snapshot nor volume with this name exists, new EBS volume will be created. \n\nWhen you're stopping the instance Spotty applies deletion policies for the volumes. There are 4 deletion policies that \ncan be specified using the `deletionPolicy` parameter:\n\n- __Retain__: this is the default deletion policy. The volume will retain, a snapshot won't be created.\n\n- __CreateSnapshot__: Spotty will create a new snapshot every time you're stopping an instance, the old snapshot \nwill be renamed. AWS uses incremental snapshots, so each new snapshot keeps only the data that was changed since \nthe last snapshot made (see: \n[How Incremental Snapshots Work](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSSnapshots.html#how_snapshots_work)).\n\n- __UpdateSnapshot__: a new snapshot will be created and the old one will be deleted.\n\n- __Delete__: the volume will be deleted without creating a snapshot. All data on this volume will be lost.\n" }, { "path": "docs/source/docs/providers/aws/faq.md", "content": "# FAQ\n\n## How does Spotty choose the AWS Availability Zone where to run the instance?\n\n1. If the AZ is specified in the configuration file, this AZ will be used to run the instance.\n2. If the instance already has some EBS volumes created, Spotty will pick up the volumes' AZ.\n3. Otherwise Spotty will let AWS choose an AZ. Automatically chosen AZ might not have the \nlowest Spot price, but in practice, it usually does.\n\nSpotty will raise an error if the AZ in the configuration file doesn't match AZs of the volumes \nor AZs of the volumes are different.\n\n\n## Why an instance is launching too long?\n\nMost likely the instance cannot be launched because you're trying to launch a Spot instance\nand it cannot be fulfilled. You can try to change the region or availability zone, choose another\ntype of the instance, or run an On-demand Instance by removing the `spotInstance` parameter or setting it to `false`.\n\n\n## The instance is failed to start. Where can I find the logs?\n\n1. If the CloudFormation stack failed when it was launching the instance itself, then you need to log in to\nyour AWS Console and check CloudFormation logs there.\n\n2. If the stack is failed after the instance is launched, then most likely the container is\nfailed to start because of the startup commands. In this case, Spotty usually automatically downloads necessary \nlogs to your local machine and shows where to find them. If that didn't happen, you can connect to the \nhost OS using the following command:\n\n ```bash\n spotty sh -H\n ```\n \n Then you can check the `cfn-init` logs to find out why the container is failed:\n ```bash\n sudo tail /var/log/cfn-init-cmd.log\n ```\n\n## How to ssh to a Spotty instance from a different machine?\n\nWhen you start an instance, Spotty creates an EC2 Key Pair and downloads a private key to the \n`~/.spotty/keys/aws` directory. If you want to have access to the instance from a different machine using \nthe `spotty sh` or the `spotty run` commands, you need to copy the private key to that machine to the same directory.\n\n__Note:__ if you already have an EC2 Key Pair created for the project and the private key was \nsaved on the machine A (where from an instance was launched the first time) and then you're running\nan instance for the same project from the machine B that doesn't have a private key in the `~/.spotty/keys/aws` \ndirectory, then the EC2 Key Pair will be recreated and the machine A will not be able to connect to instances \nbecause its private key doesn't match the EC2 Key Pair anymore.\n" }, { "path": "docs/source/docs/providers/aws/instance-parameters.md", "content": "# Instance Parameters\n\nInstance parameters are part of the [configuration file], but for each provider they are different. \nHere you can find parameters for an AWS instance:\n\n- __`containerName`__ _(optional)_ - a name of the container from the `containers` section.\nDefault value: `default`.\n\n- __`region`__ - AWS region where to run an instance (you can use command `spotty aws spot-prices` to find the \ncheapest region).\n\n- __`availabilityZone`__ _(optional)_ - AWS availability zone where to run an instance. If a zone is not specified, it \nwill be chosen automatically.\n\n- __`subnetId`__ _(optional)_ - AWS subnet ID. If this parameter is set, the \"availabilityZone\" parameter should be \nset as well. If it's not specified, a default subnet will be used.\n\n- __`instanceType`__ - a type of the instance to run. You can find more information about \ntypes of GPU instances here: \n[Recommended GPU Instances](https://docs.aws.amazon.com/dlami/latest/devguide/gpu.html).\n\n- __`spotInstance`__ _(optional)_ - if set to `true`, runs a Spot instance instead of an On-demand instance,\n\n- __`amiName`__ _(optional)_ - a name of the AMI with NVIDIA Docker (default value is \"SpottyAMI\"). Use the \n`spotty aws create-ami` command to create it. This AMI will be used to run your application inside the Docker container.\n\n- __`amiId`__ _(optional)_ - ID of the AMI with NVIDIA Docker. This parameter can be used to run an instance using a \nshared Spotty AMI.\n\n- __`maxPrice`__ _(optional)_ - the maximum price per hour that you are willing to pay for a Spot Instance. By default, \nit's the On-demand price for the chosen instance type. Read more here: \n[Spot Instances](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-spot-instances.html).\n\n- __`rootVolumeSize`__ _(optional)_ - size of the root volume in GB. The root volume will be destroyed once \nthe instance is terminated. Use attached volumes to store the data you need to keep (see \"volumes\" parameter below).\n\n- __`dockerDataRoot`__ _(optional)_ - directory where Docker will store all downloaded and built images. \nRead more: [Caching Docker Image on an EBS Volume].\n\n- __`volumes`__ _(optional)_ - the list of volumes to attach to the instance:\n - __`name`__ - a name of the volume. This name should match one of the container's `volumeMounts` to have this \n volume attached to the container's filesystem.\n\n - __`parameters`__ _(optional)_ - parameters of the volume:\n - __`type`__ _(optional)_ - the volume type. Supported types: \"__gp2__\", \"__sc1__\", \"__st1__\" \n and \"__standard__\". The default value is \"gp2\". Read more here: \n [Amazon EBS Volume Types](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSVolumeTypes.html).\n \n - __`size`__ _(optional)_ - size of the volume in GB. Size of the volume cannot be less than the size of \n the existing snapshot but can be increased.\n\n - __`deletionPolicy`__ _(optional)_ - what to do with the volume once the instance is terminated using the \n `spotty stop` command. Possible values include: \"__Retain__\" _(value by default)_, \"__CreateSnapshot__\", \n \"__UpdateSnapshot__\" and \"__Delete__\". Read more here: [EBS Volumes and Deletion Policies].\n\n - __`volumeName`__ _(optional)_ - name of the EBS volume. The default name is \n \"{project_name}-{instance_name}-{volume_name}\".\n\n - __`mountDir`__ _(optional)_ - directory where the volume will be mounted on the instance. The default \n directory is \"/mnt/{ebs_volume_name}\".\n\n- __`ports`__ _(optional)_ - list of ports to open on the instance. For example:\n ```yaml\n ports: [6006, 8888]\n ```\n It will open ports 6006 for TensorBoard and 8888 for Jupyter Notebook. \n\n- __`localSshPort`__ _(optional)_ - if this parameter is set, all the Spotty commands will create SSH connections \nwith the instance using the IP address __127.0.0.1__ and the specified port. This can be useful in case when an \ninstance doesn't have a public IP address and a jump-server is used for tunneling.\n\n- __`managedPolicyArns`__ _(optional)_ - a list of Amazon Resource Names (ARNs) of the IAM managed policies that \nyou want to attach to the instance role. Read more about Managed Policies \n[here](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_managed-vs-inline.html).\n\n- __`instanceProfileArn`__ _(optional)_ - an Amazon Resource Name (ARN) of the IAM Instance Profile that you'd like\nto attach to the instance. Read more about Instance Profiles\n[here](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2_instance-profiles.html).\n\n- __`commands`__ _(optional)_ - commands that should be run on the host OS before the container is started. \nFor example, you could login to Amazon ECR to pull a Docker image from there \n([Deep Learning Containers Images](https://docs.aws.amazon.com/dlami/latest/devguide/deep-learning-containers-images.html)):\n ```yaml\n commands: |\n $(aws ecr get-login --no-include-email --region us-east-2 --registry-ids 763104351884)\n ```\n\n\n[configuration file]: \n[Caching Docker Image on an EBS Volume]: \n[EBS Volumes and Deletion Policies]: \n" }, { "path": "docs/source/docs/providers/aws/overview.rst", "content": "AWS Provider Overview\n=====================\n\n.. toctree::\n\n instance-parameters\n ebs-volumes-and-deletion-policies\n caching-docker-image-on-an-ebs-volume\n faq\n" }, { "path": "docs/source/docs/providers/gcp/account-preparation.md", "content": "# GCP Account Preparation\n\n1. [Create a project](https://console.cloud.google.com/projectcreate) \nif you don't have one already.\n2. Enable the [Deployment Manager API](https://console.cloud.google.com/apis/library/deploymentmanager.googleapis.com) \nfor the created project.\n3. Enable the [Runtime Configuration API](https://console.developers.google.com/apis/library/runtimeconfig.googleapis.com) \nfor the created project.\n4. [Create a service account](https://console.cloud.google.com/iam-admin/serviceaccounts/create).\n5. Go to the [IAM page](https://console.cloud.google.com/iam-admin/iam) and add the following \nroles to the created service account:\n 1. _Compute Admin_\n 2. _Storage Admin_\n 3. _Deployment Manager Editor_\n 4. _Cloud RuntimeConfig Admin_\n6. Make sure you have a quota to run GPU instances:\n 1. Go to the quotas page in \"IAM & admin\" and filter the list of services by setting the Metric \n field to __\"GPUs (all regions)\"__: [https://console.cloud.google.com/iam-admin/quotas?metric=GPUs%20(all%20regions)](https://console.cloud.google.com/iam-admin/quotas?metric=GPUs%20(all%20regions)).\n 2. Check the limit for the __\"Compute Engine API\"__ service. If it's a zero, select the service and \n click the __\"[+] EDIT QUOTAS\"__ button at the top of the page.\n 3. Set a new quota limit to 1 or more and submit the request.\n7. [Install Google Cloud SDK](https://cloud.google.com/sdk/install).\n8. Before using Spotty commands like `spotty start`, `spotty run` and others, make sure that the \n`GOOGLE_APPLICATION_CREDENTIALS` environmental variable is set up and contains the path to your service \naccount key file:\n ```bash\n export GOOGLE_APPLICATION_CREDENTIALS=\"/path/to/the/service/account/key/file.json\"\n ```\n" }, { "path": "docs/source/docs/providers/gcp/caching-docker-image-on-a-disk.md", "content": "# Caching Docker Image on a Disk\n\nYou can cache images that you've built or downloaded from the internet on a disk that you attach to the instance.\n\nA configuration file has the \"__dockerDataRoot__\" parameter. It's a directory on the host OS where the Docker \ndaemon will save all the images.\n\nSpecify the `moundDir` directory for one of the instance volumes and set the `dockerDataRoot` parameter\nto the same value (or to a subdirectory of the `moundDir` directory).\n\nExample:\n```yaml\n# ...\n\ninstances:\n - name: gcp-1\n provider: gcp\n parameters:\n # ...\n dockerDataRoot: /docker\n volumes:\n # ...\n - name: docker\n parameters:\n size: 10\n mountDir: /docker\n```\n" }, { "path": "docs/source/docs/providers/gcp/disks-and-deletion-policies.md", "content": "# Disks and Deletion Policies\n\nBy default, disks have names in the following format: `--`.\nBut you can specify a custom name using the `diskName` parameter. \n\nWhen you're starting an instance:\n1. Spotty is looking for existing disks using their names. If a disk exists, it will be attached to the \ninstance.\n2. If not - Spotty will be looking for a snapshot with the same name. If the snapshot exists, the disk will be \nrestored from that snapshot.\n3. If neither snapshot nor disk with this name exists, a new disk will be created. \n\n__Note:__ Deletion Policies for the GCP provider are not implemented yet, so, regardless of the `deletionPolicy` \nparameter value, created disks will retain when the instance is terminated.\n" }, { "path": "docs/source/docs/providers/gcp/instance-parameters.md", "content": "# Instance Parameters\n\nInstance parameters are part of the [configuration file], but for each provider they are different. \nHere you can find parameters for a GCP instance:\n\n- __`containerName`__ _(optional)_ - a name of the container from the `containers` section.\nDefault value: `default`.\n\n- __`zone`__ - GCP zone where to run an instance.\n\n- __`machineType`__ - a type of the instance to run. You can find a list of predefined machine types\nhere: [Machine Types](https://cloud.google.com/compute/docs/machine-types). If you in doubt what to use,\njust go for `n1-standard-1`. To attach GPUs to the selected machine type, use the `gpu` parameter (see \nthe details below).\n\n- __`gpu`__ _(optional)_ - _a dictionary with keys `type` and `count`_:\n - __`type`__ - a type of GPU to attach to the instance. Read more about GPUs and their availabily\n in different zones here: [GPUs on Compute Engine](https://cloud.google.com/compute/docs/gpus/).\n - __`count`__ _(optional)_ - a number of GPUs that should be attached to the instance. The default\n value is 1. See here a number of GPUs that you can attach to different machine types: \n [Valid numbers of GPUs for each machine type](https://cloud.google.com/ml-engine/docs/tensorflow/using-gpus#gpu-compatibility-table).\n\n- __`preemptibleInstance`__ _(optional)_ - if set to `true`, runs a preemptible instance instead of an on-demand \ninstance. __Note:__ be aware that GCP terminates preemptible instances in 24 hours. Read more about Preemptible VMs \n[here](https://cloud.google.com/compute/docs/instances/preemptible).\n\n- __`imageName`__ _(optional)_ - a name of the image with NVIDIA Docker in the current GCP project. You can use \nthe `spotty gcp create-image` command to create it. By default, the command will create an image with the name \n\"spotty\". This image will be used to run your application inside the Docker container. If you didn't create your own \nimage, see the behaviour of the `imageUrl` parameter.\n\n- __`imageUrl`__ _(optional)_ - a URL of the image with NVIDIA Docker. You can use this parameter to work with an image\nfrom another GCP project. If this parameter is not specified and you didn't create your own image (see the `imageName` \nparameter), Spotty will be using the `projects/spotty-cloud/global/images/family/spotty` image provided by the Spotty \nproject.\n\n- __`bootDiskSize`__ _(optional)_ - size of the root volume in GB. The root volume will be destroyed once \nthe instance is terminated. Use attached volumes to store the data that you need to keep (see the `volumes` \nparameter below).\n\n- __`dockerDataRoot`__ _(optional)_ - directory where Docker will store all downloaded and built images. \nRead more: [Caching Docker Image on a Disk].\n\n- __`volumes`__ _(optional)_ - the list of volumes to attach to the instance:\n - __`name`__ - a name of the volume. This name should match one of the container's `volumeMounts` to have this \n volume attached to the container's filesystem.\n\n - __`parameters`__ _(optional)_ - parameters of the volume:\n - __`size`__ _(optional)_ - size of the disk in GB. Size of the disk cannot be less than the size of \n the existing snapshot but can be increased.\n\n - __`deletionPolicy`__ _(optional)_ - what to do with the disk once the instance is terminated using the \n `spotty stop` command. Possible values include: \"__Retain__\" _(value by default)_, \"__CreateSnapshot__\", \n \"__UpdateSnapshot__\" and \"__Delete__\". Read more: [Disks and Deletion Policies].\n \n __(!) Note:__ Deletion Policies are not implemented yet, so created disks will always retain.\n\n - __`diskName`__ _(optional)_ - name of the disk. The default name is \n \"{project_name}-{instance_name}-{volume_name}\".\n\n - __`mountDir`__ _(optional)_ - directory where the disk will be mounted on the instance. The default \n directory is \"/mnt/{disk_name}\".\n\n- __`ports`__ _(optional)_ - list of ports to open on the instance. For example:\n ```yaml\n ports: [6006, 8888]\n ```\n It will open ports 6006 for TensorBoard and 8888 for Jupyter Notebook. \n\n- __`localSshPort`__ _(optional)_ - if this parameter is set, all the Spotty commands will create SSH connections \nwith the instance using the IP address __127.0.0.1__ and the specified port. This can be useful in case when an \ninstance doesn't have a public IP address and a jump-server is used for tunneling.\n\n- __`commands`__ _(optional)_ - commands that should be run on the host OS before the container is started.\n\n\n[configuration file]: \n[Caching Docker Image on a Disk]: \n[Disks and Deletion Policies]: \n" }, { "path": "docs/source/docs/providers/gcp/overview.rst", "content": "GCP Provider Overview\n=====================\n\n.. toctree::\n\n account-preparation\n instance-parameters\n disks-and-deletion-policies\n caching-docker-image-on-a-disk\n" }, { "path": "docs/source/docs/providers/local/instance-parameters.md", "content": "# Instance Parameters\n\nInstance parameters are part of the [configuration file], but for each provider they are different. \nHere you can find parameters for a local instance:\n\n- __`containerName`__ _(optional)_ - a name of the container from the `containers` section.\nDefault value: `default`.\n\n- __`volumes`__ _(optional)_ - the list of volumes to attach to the instance:\n - __`name`__ - a name of the volume. This name should match one of the container's `volumeMounts` to have this \n volume attached to the container's filesystem.\n\n - __`parameters`__ _(optional)_ - parameters of the volume:\n - __`path`__ _(optional)_ - a path on a local instance that should be mounted to the container.\n\n[configuration file]: \n" }, { "path": "docs/source/docs/providers/local/overview.rst", "content": "Local Provider Overview\n=======================\n\n.. toctree::\n\n instance-parameters\n" }, { "path": "docs/source/docs/providers/remote/instance-parameters.md", "content": "# Instance Parameters\n\nInstance parameters are part of the [configuration file], but for each provider they are different. \nHere you can find parameters for a remote instance:\n\n- __`containerName`__ _(optional)_ - a name of the container from the `containers` section.\nDefault value: `default`.\n\n- __`volumes`__ _(optional)_ - the list of volumes to attach to the instance:\n - __`name`__ - a name of the volume. This name should match one of the container's `volumeMounts` to have this \n volume attached to the container's filesystem.\n\n - __`parameters`__ _(optional)_ - parameters of the volume:\n - __`path`__ _(optional)_ - a path on a remote instance that should be mounted to the container.\n\n[configuration file]: \n" }, { "path": "docs/source/docs/providers/remote/overview.rst", "content": "Remote Provider Overview\n========================\n\n.. toctree::\n\n instance-parameters\n" }, { "path": "docs/source/docs/user-guide/configuration-file.md", "content": "# Spotty Configuration File\n\nBy default, Spotty is looking for a `spotty.yaml` file in the root directory of the project. This file describes \nparameters of a remote instance and an environment for the project. Here is a basic example of such file for AWS:\n\n```yaml\nproject:\n name: my-project-name\n syncFilters:\n - exclude:\n - .git/*\n - .idea/*\n - '*/__pycache__/*'\n\ncontainers:\n - projectDir: /workspace/project\n image: tensorflow/tensorflow:latest-gpu-py3-jupyter\n env:\n PYTHONPATH: /workspace/project\n ports:\n # TensorBoard\n - containerPort: 6006\n hostPort: 6006\n # Jupyter\n - containerPort: 8888\n hostPort: 8888\n volumeMounts:\n - name: workspace\n mountPath: /workspace\n\ninstances:\n - name: aws-1\n provider: aws\n parameters:\n region: eu-west-1\n instanceType: p2.xlarge\n ports: [6006, 8888]\n volumes:\n - name: workspace\n parameters:\n size: 50\n\nscripts:\n tensorboard: |\n tensorboard --bind_all --port 6006 --logdir /workspace/project/training\n jupyter: |\n jupyter notebook --allow-root --ip 0.0.0.0 --notebook-dir=/workspace/project\n```\n\nInstance parameters are different for each provider:\n\n- [Local Provider Instance Parameters]\n- [Remote Provider Instance Parameters]\n- [AWS Provider Instance Parameters]\n- [GCP Provider Instance Parameters]\n\n\n## Available Parameters\n\nConfiguration file consists of 4 sections: `project`, `containers`, `instances` and `scripts`.\n\n### __`project`__ section\n\nThe `project` section contains the following parameters:\n\n- __`name`__ - the name of your project. It will be used to create an S3 bucket and a CloudFormation stack to run \nan instance.\n\n- __`syncFilters`__ _(optional)_ - filters to skip some directories or files during synchronization. By default, all \nproject files will be synced with the instance. Example:\n ```yaml\n syncFilters:\n - exclude:\n - .idea/*\n - .git/*\n - data/*\n - include:\n - data/test/*\n - exclude:\n - data/test/dump.json\n ```\n \n It will skip \".idea/\", \".git/\" and \"data/\" directories except the \"data/test/\" directory. All files from \n the \"data/test/\" directory will be synced with the instance except the \"data/test/dump.json\" file.\n \n You can read more about filters \n here: [Use of Exclude and Include Filter](https://docs.aws.amazon.com/cli/latest/reference/s3/index.html#use-of-exclude-and-include-filters). \n\n### __`containers`__ section\n\nThe `containers` section contains a list of containers where each container is described \nwith the following parameters:\n\n- __`name`__ - a name of the container. You can associate containers with the instances using the `containerName`\nparameter in the instance configuration. Default value: `default`.\n\n- __`projectDir`__ - a directory inside the container where the local project will be copied. If\nit's a subdirectory of a container volume, the project will be located on that volume, \notherwise, the data will be lost once the instance is terminated.\n\n- __`image`__ _(optional)_ - the name of the Docker image that contains the environment for your project. For example, \nyou could use [TensorFlow image for GPU](https://hub.docker.com/r/tensorflow/tensorflow/) \n(`tensorflow/tensorflow:latest-gpu-py3-jupyter`). It already contains NumPy, SciPy, scikit-learn, pandas, Jupyter Notebook and \nTensorFlow itself. If you need to use your own image, you can specify the path to your Dockerfile in the \n__`file`__ parameter (see below), or push your image to the [Docker Hub](https://hub.docker.com/).\n\n- __`file`__ _(optional)_ - relative path to your custom Dockerfile.\n \n __Note:__ Spotty uses the directory with the Dockerfile as its build context, so make sure it doesn't contain \n gigabytes of irrelevant data (keep the Dockerfile in a separate directory or use the `.dockerignore` file). \n Otherwise, you may get an out-of-space error because Docker copies the entire build context to the Docker daemon \n during the build. Read more here: [\"docker build\" command](https://docs.docker.com/engine/reference/commandline/build/).\n\n __Example:__ if you use TensorFlow and need to download your dataset from S3, you could install \n [AWS CLI](https://github.com/aws/aws-cli) on top of the original TensorFlow image. Just create a \n `Dockerfile` in the `docker/` directory of your project:\n ```dockerfile\n FROM tensorflow/tensorflow:latest-gpu-py3-jupyter\n\n RUN pip install awscli\n ```\n\n Then set the `file` parameter to `docker/Dockerfile`.\n\n- __`runAsHostUser`__ _(optional)_ - if set to `true`, the container will be run with the host user ID and group ID,\n\n- __`volumeMounts`__ _(optional)_ - where to mount instance volumes into the container's filesystem. Each element \nof a list has the following parameters:\n - __`name`__ - this must match the name of an instance volume.\n - __`mountPath`__ - a path within the container at which the volume should be mounted.\n\n- __`workingDir`__ _(optional)_ - working directory for your custom scripts (see \"scripts\" section below),\n\n- __`env`__ _(optional)_ - a dictionary with environmental variables that will be available in the container,\n\n- __`hostNetwork`__ _(optional)_ - if set to `true`, the Docker container will be run with the host network,\n\n- __`ports`__ _(optional)_ - container ports that should be published to the host. Each element of a list \ncontains the following parameters:\n - __`containerPort`__ - a container port,\n - __`hostPort`__ _(optional)_ - a host port. By default, the container port will be published on a random \n host port.\n\n- __`commands`__ _(optional)_ - commands which should be performed once your container is started. For example, you \ncould download your datasets from an S3 bucket to the project directory (see \"project\" section):\n ```yaml\n commands: |\n aws s3 sync s3://my-bucket/datasets/my-dataset /workspace/project/data\n ```\n\n- __`runtimeParameters`__ _(optional)_ - a list of additional parameters for the container runtime. For example:\n ```yaml\n runtimeParameters: ['--privileged', '--shm-size', '2G']\n ```\n\n### __`instances`__ section\n\nThe `instances` section contains a list of instances where each instance is described \nwith the following parameters:\n\n- __`name`__ - a name of the instance. Use this name to manage the instance with the commands like \n\"spotty start\" or \"spotty stop\". Also Spotty uses this name in the names of AWS and GCP resources.\n\n- __`provider`__ - a provider for the instance. At the moment Spotty supports 4 providers:\n - \"__local__\" - runs containers using the Docker installed on the local machine,\n - \"__remote__\" - runs containers on a remote machine through SSH,\n - \"__aws__\" - Amazon Web Services EC2 instances,\n - \"__gcp__\" - Google Cloud Platform VMs.\n\n- __`parameters`__ - parameters of the instance. These parameters are different for each provider:\n - [Local Provider Instance Parameters]\n - [Remote Provider Instance Parameters]\n - [AWS Provider Instance Parameters]\n - [GCP Provider Instance Parameters]\n\n### __`scripts`__ section\n\nThe `scripts` section contains custom scripts which can be run with the `spotty run ` \ncommand. The following example defines 2 scripts: `jupyter` - to run Jupyter server and `train` - \nto start training a model:\n\n```yaml\nscripts:\n jupyter: |\n jupyter notebook --allow-root --ip 0.0.0.0 --notebook-dir=/workspace/project\n\n train: |\n if [ -n \"{{MODEL}}\" ]; then\n python /workspace/project/model/train.py --model-name {{MODEL}}\n else\n echo \"The MODEL parameter is required.\"\n fi\n```\n\nTo start Jupyter simply run:\n```bash\nspotty run jupyter\n```\n\nIt will start Jupyter server on the remote instance inside a tmux session. Jupyter will be available on the port\nspecified in the container configuration (see the example on top of the page).\n\nCopy an authentication token from the command output and use the __`Ctrl + b`__, then __`d`__ combination of keys \nto detach the tmux session - Jupyter will keep running.\n\nYou also can write parametrized scripts. For example, the `train` script contains the `MODEL` parameter. So you\ncould run your training script with different model names:\n\n```bash\nspotty run train -p MODEL=my-model\n```\n\nUse the __`Ctrl + b`__, then __`d`__ combination of keys to detached tmux session - the script will keep running. \n\nYou can come back to the running script the following ways:\n- either use the same command again - you will be reattached to the existing tmux session,\n- or connect to the instance using the `spotty sh` command and then use the __`Ctrl + b`__, \nthen __`s`__ combination of keys to switch into the right tmux session.\n\n__Note:__ don't forget to use the \"|\" character for multi-line scripts, otherwise the YAML parser\nwill merge multiple lines together.\n\n\n[Local Provider Instance Parameters]: \n[Remote Provider Instance Parameters]: \n[AWS Provider Instance Parameters]: \n[GCP Provider Instance Parameters]: \n" }, { "path": "docs/source/docs/user-guide/getting-started.md", "content": "# Getting Started\n\n## Installation\n\nUse [pip](http://www.pip-installer.org/en/latest/) to install or upgrade Spotty:\n\n```bash\npip install -U spotty\n```\n\nPython >=3.6 is required.\n\nAlso, depending on the use case, some additional software is needed:\n\n* __Docker__ if you want to run containers locally: [Get Docker](https://docs.docker.com/get-docker/)\n* __AWS CLI__ if you're going to use AWS: [Installing the AWS Command Line Interface](http://docs.aws.amazon.com/cli/latest/userguide/installing.html)\n* __Google Cloud SDK__ if you're going to use GCP: [Installing Google Cloud SDK](https://cloud.google.com/sdk/install)\n\n\n## Prepare a configuration file\n\nPrepare a `spotty.yaml` file and put it to the root directory of your project:\n\n - See the file specification and an example here: [Spotty Configuration File].\n - Read [this](https://medium.com/@apls/how-to-train-deep-learning-models-on-aws-spot-instances-using-spotty-8d9e0543d365) \n article for a real-world example.\n\n## Start an instance\n\nUse the following command to launch an instance with the Docker container:\n \n```bash\nspotty start\n```\n\nIf you're using AWS, it will create EBS volumes if needed, start an instance, upload project files and start \nthe Docker container with the environment for your project.\n\n## Train your models or run notebooks\n\nTo connect to the running container via SSH, use the following command:\n\n```bash\nspotty sh\n```\n\nIt runs a [tmux](https://github.com/tmux/tmux/wiki) session, so you can always detach this session using\n__`Ctrl + b`__, then __`d`__ combination of keys. To be attached to that session later, just use the\n`spotty sh` command again.\n\nAlso, you can run custom scripts inside the Docker container using the `spotty run ` command. Read more\nabout custom scripts in the documentation: [Configuration File: \"scripts\" section].\n\n\n[Spotty Configuration File]: \n[Configuration File: \"scripts\" section]: \n" }, { "path": "docs/source/docs/user-guide/installation.md", "content": "# Installation\n\nUse [pip](http://www.pip-installer.org/en/latest/) to install or upgrade Spotty:\n\n```bash\npip install -U spotty\n```\n\nPython >=3.6 is required.\n\nAlso, depending on the use case, some additional software is needed:\n\n* __Docker__ if you want to run containers locally: [Get Docker](https://docs.docker.com/get-docker/)\n* __AWS CLI__ if you're going to use AWS: [Installing the AWS Command Line Interface](http://docs.aws.amazon.com/cli/latest/userguide/installing.html)\n* __Google Cloud SDK__ if you're going to use GCP: [Installing Google Cloud SDK](https://cloud.google.com/sdk/install)\n" }, { "path": "docs/source/index.rst", "content": ".. raw:: html\n :file: main.html\n\nWelcome to Spotty Documentation\n===============================\n\n.. toctree::\n :hidden:\n :maxdepth: 2\n :caption: User Guide\n\n docs/user-guide/getting-started\n docs/user-guide/installation\n docs/user-guide/configuration-file\n docs/cli/spotty\n\n.. toctree::\n :hidden:\n :maxdepth: 2\n :caption: Providers\n\n Local Provider \n Remote Provider \n AWS Provider \n GCP Provider \n" }, { "path": "docs/source/main.html", "content": "\n\n
\n
\n \n
\n\n

\n An Open-source Tool for Training
Deep Learning Models in the Cloud\n

\n\n
\n \n Get started now\n \n \n View it on GitHub\n \n
\n\n
\n \n \n \n \n \n \n \n \n \n
\n \n \n \n \n \n
\n \n \n
Run Training on AWS and GCP Instances
\n
Spotty makes training of deep learning models on AWS and GCP instances as simple as training on your local machine.
\n
\n 		\n \n \n \n \n \n
\n \n \n
Reduce Training Costs
\n
Spotty can save you up to 70% of the costs by using AWS Spot Instances\n or GCP Preemtible VMs.
\n
\n
\n \n \n \n \n \n
\n \n \n
Share Your Model with Everyone
\n
Spotty makes your model trainable locally or in the cloud by everyone with a couple of commands.
\n
\n 		\n \n \n \n \n \n
\n \n \n
Develop with Docker
\n
Spotty helps you to develop your model locally using a Docker container, so the environment can be set up by anyone and anywhere with a single command.
\n
\n
\n
\n
\n" }, { "path": "setup.cfg", "content": "[metadata]\ndescription-file = README.md\n" }, { "path": "setup.py", "content": "#!/usr/bin/env python\n\nimport os\nimport re\nfrom setuptools import setup, find_packages\n\n\ndef get_version():\n root_dir = os.path.abspath(os.path.dirname(__file__))\n with open(os.path.join(root_dir, 'spotty', '__init__.py')) as f:\n content = f.read()\n\n version_match = re.search(r'^__version__ = [\\'\"]([^\\'\"]*)[\\'\"]', content, re.M)\n if not version_match:\n raise RuntimeError('Unable to find version string.')\n\n return version_match.group(1)\n\n\ndef get_description():\n readme_path = os.path.abspath(os.path.join(os.path.dirname(__file__), 'README.md'))\n with open(readme_path, encoding='utf-8') as f:\n description = f.read()\n\n return description\n\n\nsetup(name='spotty',\n version=get_version(),\n description='Training deep learning models on AWS and GCP instances',\n url='https://github.com/spotty-cloud/spotty',\n author='Oleg Polosin',\n author_email='apls777@gmail.com',\n license='MIT',\n long_description=get_description(),\n long_description_content_type='text/markdown',\n packages=find_packages(exclude=['tests*']),\n package_data={\n 'spotty.deployment.container.docker.scripts': ['data/*', 'data/*/*'],\n 'spotty.providers.aws.cfn_templates.instance': ['data/*', 'data/*/*'],\n 'spotty.providers.aws.cfn_templates.instance_profile': ['data/*', 'data/*/*'],\n 'spotty.providers.gcp.dm_templates.instance': ['data/*', 'data/*/*'],\n },\n scripts=['bin/spotty'],\n install_requires=[\n 'boto3>=1.9.0',\n 'google-api-python-client>=1.7.8',\n 'google-cloud-storage>=1.15.0',\n 'cfn_flip', # to work with CloudFormation templates\n 'schema',\n 'chevron',\n ],\n tests_require=['moto'],\n test_suite='tests',\n classifiers=[\n 'Development Status :: 5 - Production/Stable',\n 'Intended Audience :: Science/Research',\n 'Intended Audience :: Developers',\n 'Intended Audience :: System Administrators',\n 'Natural Language :: English',\n 'License :: OSI Approved :: MIT License',\n 'Programming Language :: Python',\n 'Programming Language :: Python :: 3',\n 'Programming Language :: Python :: 3.6',\n 'Programming Language :: Python :: 3.7',\n ])\n" }, { "path": "spotty/__init__.py", "content": "__version__ = '1.3.4'\n" }, { "path": "spotty/cli.py", "content": "import argparse\nfrom typing import List, Type\nimport pkg_resources\nfrom spotty.commands.abstract_command import AbstractCommand\nfrom spotty.commands.aws import AwsCommand\nfrom spotty.commands.download import DownloadCommand\nfrom spotty.commands.exec import ExecCommand\nfrom spotty.commands.run import RunCommand\nfrom spotty.commands.sh import ShCommand\nfrom spotty.commands.start import StartCommand\nfrom spotty.commands.status import StatusCommand\nfrom spotty.commands.stop import StopCommand\nfrom spotty.commands.sync import SyncCommand\n\n\ndef get_parser() -> argparse.ArgumentParser:\n parser = argparse.ArgumentParser()\n parser.add_argument('-V', '--version', action='store_true', help='Display the version of the Spotty')\n\n command_classes = [\n StartCommand,\n StopCommand,\n StatusCommand,\n ShCommand,\n RunCommand,\n ExecCommand,\n SyncCommand,\n DownloadCommand,\n AwsCommand,\n ] + _get_custom_commands()\n\n # add commands to the parser\n add_subparsers(parser, command_classes)\n\n return parser\n\n\ndef add_subparsers(parser: argparse.ArgumentParser, command_classes: List[Type[AbstractCommand]]):\n \"\"\"Adds commands to the parser.\"\"\"\n subparsers = parser.add_subparsers()\n for command_class in command_classes:\n command = command_class()\n subparser = subparsers.add_parser(command.name, help=command.description, description=command.description)\n subparser.set_defaults(command=command, parser=subparser)\n command.configure(subparser)\n\n\ndef _get_custom_commands() -> List[Type[AbstractCommand]]:\n \"\"\"Returns custom commands that integrated through entry points.\"\"\"\n return [entry_point.load() for entry_point in pkg_resources.iter_entry_points('spotty.commands')]\n" }, { "path": "spotty/commands/__init__.py", "content": "" }, { "path": "spotty/commands/abstract_command.py", "content": "from abc import ABC, abstractmethod\nfrom argparse import Namespace, ArgumentParser\nfrom spotty.commands.writers.abstract_output_writrer import AbstractOutputWriter\n\n\nclass AbstractCommand(ABC):\n \"\"\"Abstract class for a Spotty sub-command.\"\"\"\n\n @property\n @abstractmethod\n def name(self) -> str:\n \"\"\"The sub-command name.\"\"\"\n raise NotImplementedError\n\n @property\n def description(self) -> str:\n \"\"\"The sub-command description. It will be displayed in the help text.\"\"\"\n return ''\n\n def configure(self, parser: ArgumentParser):\n \"\"\"Adds arguments for the sub-command.\"\"\"\n parser.add_argument('-d', '--debug', action='store_true', help='Show debug messages')\n\n @abstractmethod\n def run(self, args: Namespace, output: AbstractOutputWriter):\n \"\"\"Runs the sub-command.\n\n Args:\n args: Arguments provided by argparse.\n output: Output writer.\n Raises:\n ValueError: If command's arguments can't be processed.\n \"\"\"\n raise NotImplementedError\n" }, { "path": "spotty/commands/abstract_config_command.py", "content": "from abc import abstractmethod\nfrom typing import List\nfrom argparse import Namespace, ArgumentParser\nfrom spotty.config.config_utils import load_config\nfrom spotty.deployment.abstract_instance_manager import AbstractInstanceManager\nfrom spotty.providers.instance_manager_factory import InstanceManagerFactory\nfrom spotty.commands.abstract_command import AbstractCommand\nfrom spotty.commands.writers.abstract_output_writrer import AbstractOutputWriter\n\n\nclass AbstractConfigCommand(AbstractCommand):\n \"\"\"Abstract class for a Spotty sub-command that needs to use a project's configuration.\"\"\"\n\n @abstractmethod\n def _run(self, instance_manager: AbstractInstanceManager, args: Namespace, output: AbstractOutputWriter):\n raise NotImplementedError\n\n def configure(self, parser: ArgumentParser):\n super().configure(parser)\n parser.add_argument('-c', '--config', type=str, default=None, help='Path to the configuration file')\n parser.add_argument('instance_name', metavar='INSTANCE_NAME', nargs='?', type=str, help='Instance name')\n\n def run(self, args: Namespace, output: AbstractOutputWriter):\n # get project configuration\n project_config = load_config(args.config)\n\n # get instance configuration\n instance_id = self._get_instance_id(project_config.instances, args.instance_name, output)\n instance_config = project_config.instances[instance_id]\n\n # create an instance manger\n instance_manager = InstanceManagerFactory.get_instance(project_config, instance_config)\n\n # run the command\n self._run(instance_manager, args, output)\n\n @staticmethod\n def _get_instance_id(instances: List[dict], instance_name: str, output: AbstractOutputWriter):\n if not instance_name:\n if len(instances) > 1:\n # ask user to choose the instance\n output.write('Select the instance:\\n')\n with output.prefix(' '):\n for i, instance_config in enumerate(instances):\n output.write('[%d] %s' % (i + 1, instance_config['name']))\n output.write()\n\n try:\n num = int(input('Enter number: '))\n output.write()\n except ValueError:\n num = 0\n\n if num < 1 or num > len(instances):\n raise ValueError('The value from 1 to %d was expected.' % len(instances))\n\n instance_id = num - 1\n else:\n instance_id = 0\n else:\n # get instance ID by name\n instance_ids = [i for i, instance in enumerate(instances) if instance['name'] == instance_name]\n if not instance_ids:\n raise ValueError('Instance \"%s\" not found in the configuration file' % instance_name)\n\n instance_id = instance_ids[0]\n\n return instance_id\n" }, { "path": "spotty/commands/abstract_provider_command.py", "content": "from abc import abstractmethod\nfrom argparse import Namespace, ArgumentParser\nimport sys\nfrom spotty.commands.abstract_command import AbstractCommand\nfrom spotty.commands.writers.abstract_output_writrer import AbstractOutputWriter\n\n\nclass AbstractProviderCommand(AbstractCommand):\n \"\"\"Abstract class for a provider sub-command.\"\"\"\n\n @property\n @abstractmethod\n def commands(self) -> list:\n \"\"\"Returns a list of the provider sub-commands.\"\"\"\n raise NotImplementedError\n\n def configure(self, parser: ArgumentParser):\n from spotty.cli import add_subparsers\n add_subparsers(parser, self.commands)\n\n def run(self, args: Namespace, output: AbstractOutputWriter):\n \"\"\"If the command is called, it just displays a list of available sub-commands.\"\"\"\n args.parser.print_help()\n sys.exit(1)\n" }, { "path": "spotty/commands/aws.py", "content": "from spotty.commands.abstract_provider_command import AbstractProviderCommand\nfrom spotty.providers.aws.commands.clean_logs import CleanLogsCommand\nfrom spotty.providers.aws.commands.spot_prices import SpotPricesCommand\n\n\nclass AwsCommand(AbstractProviderCommand):\n\n name = 'aws'\n description = 'AWS commands'\n commands = [\n SpotPricesCommand,\n CleanLogsCommand,\n ]\n" }, { "path": "spotty/commands/download.py", "content": "from argparse import Namespace, ArgumentParser\nfrom spotty.commands.abstract_config_command import AbstractConfigCommand\nfrom spotty.commands.writers.abstract_output_writrer import AbstractOutputWriter\nfrom spotty.errors.instance_not_running import InstanceNotRunningError\nfrom spotty.errors.nothing_to_do import NothingToDoError\nfrom spotty.deployment.abstract_instance_manager import AbstractInstanceManager\n\n\nclass DownloadCommand(AbstractConfigCommand):\n\n name = 'download'\n description = 'Download files from the running instance'\n\n def configure(self, parser: ArgumentParser):\n super().configure(parser)\n parser.add_argument('-i', '--include', metavar='PATTERN', action='append', type=str, required=True,\n help='Download all files that matches the specified pattern (see Include Filters '\n 'for the \"aws s3 sync\" command). Paths must be relative to your project directory, '\n 'they cannot be absolute.')\n parser.add_argument('--dry-run', action='store_true', help='Show files to be downloaded')\n\n def _run(self, instance_manager: AbstractInstanceManager, args: Namespace, output: AbstractOutputWriter):\n # check that the instance is started\n if not instance_manager.is_running():\n raise InstanceNotRunningError(instance_manager.instance_config.name)\n\n filters = [\n {'exclude': ['*']},\n {'include': args.include}\n ]\n\n dry_run = args.dry_run\n with output.prefix('[dry-run] ' if dry_run else ''):\n try:\n instance_manager.download(filters, output, dry_run)\n except NothingToDoError as e:\n output.write(str(e))\n return\n\n output.write('Done')\n" }, { "path": "spotty/commands/exec.py", "content": "import sys\nfrom argparse import ArgumentParser, Namespace\nfrom spotty.commands.abstract_config_command import AbstractConfigCommand\nfrom spotty.commands.writers.abstract_output_writrer import AbstractOutputWriter\nfrom spotty.deployment.utils.cli import shlex_join\nfrom spotty.errors.instance_not_running import InstanceNotRunningError\nfrom spotty.errors.nothing_to_do import NothingToDoError\nfrom spotty.deployment.abstract_instance_manager import AbstractInstanceManager\n\n\nclass ExecCommand(AbstractConfigCommand):\n\n name = 'exec'\n description = 'Execute a command in the container'\n\n def configure(self, parser: ArgumentParser):\n super().configure(parser)\n parser.add_argument('-i', '--interactive', action='store_true', help='Pass STDIN to the container')\n parser.add_argument('-t', '--tty', action='store_true', help='Allocate a pseudo-TTY')\n parser.add_argument('-u', '--user', type=str, default=None,\n help='Container username or UID (format: [:])')\n parser.add_argument('--no-sync', action='store_true', help='Don\\'t sync the project before running the script')\n\n # add the \"double-dash\" argument to the usage message\n parser.prog = 'spotty exec'\n parser.usage = parser.format_usage()[7:-1] + ' -- COMMAND [args...]\\n'\n parser.epilog = 'The double dash (--) separates the command that you want to execute inside the container ' \\\n 'from the Spotty arguments.'\n\n def _run(self, instance_manager: AbstractInstanceManager, args: Namespace, output: AbstractOutputWriter):\n # check that the command is provided\n if not args.custom_args:\n raise ValueError('Use the double-dash (\"--\") to split Spotty arguments from the command that should be '\n 'executed inside the container.')\n\n # check that the instance is started\n if not instance_manager.is_running():\n raise InstanceNotRunningError(instance_manager.instance_config.name)\n\n # sync the project with the instance\n if not args.no_sync:\n try:\n instance_manager.sync(output)\n except NothingToDoError:\n pass\n\n # generate a \"docker exec\" command\n command = shlex_join(args.custom_args)\n command = instance_manager.container_commands.exec(command, interactive=args.interactive, tty=args.tty,\n user=args.user)\n\n # execute the command on the host OS\n exit_code = instance_manager.exec(command, tty=args.tty)\n sys.exit(exit_code)\n" }, { "path": "spotty/commands/run.py", "content": "from argparse import ArgumentParser, Namespace\nfrom spotty.commands.abstract_config_command import AbstractConfigCommand\nfrom spotty.commands.writers.abstract_output_writrer import AbstractOutputWriter\nfrom spotty.deployment.utils.commands import get_script_command, get_log_command, get_tmux_session_command, get_bash_command\nfrom spotty.errors.instance_not_running import InstanceNotRunningError\nfrom spotty.errors.nothing_to_do import NothingToDoError\nfrom spotty.deployment.utils.user_scripts import parse_script_parameters, render_script\nfrom spotty.deployment.abstract_instance_manager import AbstractInstanceManager\n\n\nclass RunCommand(AbstractConfigCommand):\n\n name = 'run'\n description = 'Run a custom script from the configuration file inside the container'\n\n def configure(self, parser: ArgumentParser):\n super().configure(parser)\n parser.add_argument('script_name', metavar='SCRIPT_NAME', type=str, help='Script name')\n parser.add_argument('-u', '--user', type=str, default=None,\n help='Container username or UID (format: [:])')\n parser.add_argument('-s', '--session-name', type=str, default=None, help='tmux session name')\n parser.add_argument('-l', '--logging', action='store_true', help='Log the script outputs to a file')\n parser.add_argument('-p', '--parameter', metavar='PARAMETER=VALUE', action='append', type=str, default=[],\n help='Set a value for the script parameter (format: PARAMETER=VALUE). This '\n 'argument can be used multiple times to set several parameters. Parameters can be '\n 'used in the script as Mustache variables (for example: {{PARAMETER}}).')\n parser.add_argument('--no-sync', action='store_true', help='Don\\'t sync the project before running the script')\n\n # add the \"double-dash\" argument to the usage message\n parser.prog = 'spotty run'\n parser.usage = parser.format_usage()[7:-1] + ' [-- args...]\\n'\n parser.epilog = 'The double dash (--) separates custom arguments that you can pass to the script ' \\\n 'from the Spotty arguments.'\n\n def _run(self, instance_manager: AbstractInstanceManager, args: Namespace, output: AbstractOutputWriter):\n # check that the script exists\n script_name = args.script_name\n scripts = instance_manager.project_config.scripts\n if script_name not in scripts:\n raise ValueError('Script \"%s\" is not defined in the configuration file.' % script_name)\n\n # replace script parameters\n params = parse_script_parameters(args.parameter)\n script_content = render_script(scripts[script_name], params)\n\n # check that the instance is started\n if not instance_manager.is_running():\n raise InstanceNotRunningError(instance_manager.instance_config.name)\n\n # sync the project with the instance\n if not args.no_sync:\n try:\n instance_manager.sync(output)\n except NothingToDoError:\n pass\n\n # get a command to run the script with \"docker exec\"\n script_command = get_script_command(script_name, script_content, script_args=args.custom_args,\n logging=args.logging)\n command = instance_manager.container_commands.exec(script_command, interactive=True, tty=True,\n user=args.user)\n\n # wrap the command with the tmux session\n if instance_manager.use_tmux:\n session_name = args.session_name if args.session_name else 'spotty-script-%s' % script_name\n default_command = instance_manager.container_commands.exec(get_bash_command(), interactive=True, tty=True,\n user=args.user)\n command = get_tmux_session_command(command, session_name, script_name, default_command=default_command,\n keep_pane=True)\n\n # execute command on the host OS\n instance_manager.exec(command)\n" }, { "path": "spotty/commands/sh.py", "content": "from argparse import ArgumentParser, Namespace\nfrom spotty.commands.abstract_config_command import AbstractConfigCommand\nfrom spotty.commands.writers.abstract_output_writrer import AbstractOutputWriter\nfrom spotty.deployment.utils.commands import get_bash_command, get_tmux_session_command\nfrom spotty.errors.instance_not_running import InstanceNotRunningError\nfrom spotty.deployment.abstract_instance_manager import AbstractInstanceManager\n\n\nclass ShCommand(AbstractConfigCommand):\n\n name = 'sh'\n description = 'Get a shell to the container or to the instance itself'\n\n def configure(self, parser: ArgumentParser):\n super().configure(parser)\n parser.add_argument('-u', '--user', type=str, default=None,\n help='Container username or UID (format: [:])')\n parser.add_argument('-H', '--host-os', action='store_true', help='Connect to the host OS instead of the Docker '\n 'container')\n parser.add_argument('-s', '--session-name', type=str, default=None, help='tmux session name')\n parser.add_argument('-l', '--list-sessions', action='store_true', help='List all tmux sessions managed by the '\n 'instance')\n\n def _run(self, instance_manager: AbstractInstanceManager, args: Namespace, output: AbstractOutputWriter):\n # check that the instance is started\n if not instance_manager.is_running():\n raise InstanceNotRunningError(instance_manager.instance_config.name)\n\n if args.list_sessions:\n if not instance_manager.use_tmux:\n raise ValueError('The \"%s\" provider doesn\\'t support tmux.'\n % instance_manager.instance_config.provider_name)\n\n # a command to list existing tmux session on the host OS\n command = 'tmux ls; echo \"\"'\n else:\n if args.host_os:\n # get a command to open a login shell on the host OS\n session_name = args.session_name if args.session_name else 'spotty-sh-host'\n shell_command = '$SHELL'\n command = get_tmux_session_command(shell_command, session_name, keep_pane=False) \\\n if instance_manager.use_tmux else shell_command\n else:\n # get a command to run bash inside the docker container\n command = instance_manager.container_commands.exec(get_bash_command(), interactive=True, tty=True,\n user=args.user)\n\n # wrap the command with the tmux session\n if instance_manager.use_tmux:\n session_name = args.session_name if args.session_name else 'spotty-sh-container'\n command = get_tmux_session_command(command, session_name, default_command=command, keep_pane=False)\n\n # execute command on the host OS\n instance_manager.exec(command)\n" }, { "path": "spotty/commands/start.py", "content": "from argparse import Namespace, ArgumentParser\nfrom spotty.commands.abstract_config_command import AbstractConfigCommand\nfrom spotty.commands.writers.abstract_output_writrer import AbstractOutputWriter\nfrom spotty.errors.instance_not_running import InstanceNotRunningError\nfrom spotty.deployment.abstract_instance_manager import AbstractInstanceManager\n\n\nclass StartCommand(AbstractConfigCommand):\n\n name = 'start'\n description = 'Start an instance with a container'\n\n def configure(self, parser: ArgumentParser):\n super().configure(parser)\n parser.add_argument('-C', '--container', action='store_true', help='Starts or restarts container on the '\n 'running instance')\n parser.add_argument('--dry-run', action='store_true', help='Displays the steps that would be performed '\n 'using the specified command without actually '\n 'running them')\n\n def _run(self, instance_manager: AbstractInstanceManager, args: Namespace, output: AbstractOutputWriter):\n dry_run = args.dry_run\n\n if args.container:\n # check that the instance is started\n if not instance_manager.is_running():\n raise InstanceNotRunningError(instance_manager.instance_config.name)\n\n # start a container on the running instance\n instance_manager.start_container(output, dry_run=dry_run)\n\n if not dry_run:\n instance_name = ''\n if len(instance_manager.project_config.instances) > 1:\n instance_name = ' ' + instance_manager.instance_config.name\n\n output.write('\\nContainer was successfully started.\\n'\n 'Use the \"spotty sh%s\" command to connect to the container.\\n'\n % instance_name)\n else:\n # start the instance\n with output.prefix('[dry-run] ' if dry_run else ''):\n instance_manager.start(output, dry_run)\n\n if not dry_run:\n instance_name = ''\n if len(instance_manager.project_config.instances) > 1:\n instance_name = ' ' + instance_manager.instance_config.name\n\n output.write('\\n%s\\n'\n '\\nUse the \"spotty sh%s\" command to connect to the container.\\n'\n % (instance_manager.get_status_text(), instance_name))\n" }, { "path": "spotty/commands/status.py", "content": "from argparse import Namespace\nfrom spotty.commands.abstract_config_command import AbstractConfigCommand\nfrom spotty.commands.writers.abstract_output_writrer import AbstractOutputWriter\nfrom spotty.deployment.abstract_instance_manager import AbstractInstanceManager\n\n\nclass StatusCommand(AbstractConfigCommand):\n\n name = 'status'\n description = 'Print information about the instance'\n\n def _run(self, instance_manager: AbstractInstanceManager, args: Namespace, output: AbstractOutputWriter):\n output.write(instance_manager.get_status_text())\n" }, { "path": "spotty/commands/stop.py", "content": "from argparse import Namespace\nfrom spotty.commands.abstract_config_command import AbstractConfigCommand\nfrom spotty.commands.writers.abstract_output_writrer import AbstractOutputWriter\nfrom spotty.deployment.abstract_instance_manager import AbstractInstanceManager\n\n\nclass StopCommand(AbstractConfigCommand):\n\n name = 'stop'\n description = 'Terminate running instance and apply deletion policies for the volumes'\n\n # TODO: the \"spotty start\" command should restart the instance and the container if the instance was shutdown\n # def configure(self, parser: ArgumentParser):\n # super().configure(parser)\n # parser.add_argument('-s', '--shutdown', action='store_true',\n # help='Shutdown the instance without terminating it. Deletion policies for the volumes '\n # 'won\\'t be applied.')\n\n def _run(self, instance_manager: AbstractInstanceManager, args: Namespace, output: AbstractOutputWriter):\n\n instance_manager.stop(only_shutdown=False, output=output)\n" }, { "path": "spotty/commands/sync.py", "content": "from argparse import Namespace, ArgumentParser\nfrom spotty.commands.abstract_config_command import AbstractConfigCommand\nfrom spotty.commands.writers.abstract_output_writrer import AbstractOutputWriter\nfrom spotty.errors.instance_not_running import InstanceNotRunningError\nfrom spotty.errors.nothing_to_do import NothingToDoError\nfrom spotty.deployment.abstract_instance_manager import AbstractInstanceManager\n\n\nclass SyncCommand(AbstractConfigCommand):\n\n name = 'sync'\n description = 'Synchronize the project with the running instance'\n\n def configure(self, parser: ArgumentParser):\n super().configure(parser)\n parser.add_argument('--dry-run', action='store_true', help='Show files to be synced')\n\n def _run(self, instance_manager: AbstractInstanceManager, args: Namespace, output: AbstractOutputWriter):\n # check that the instance is started\n if not instance_manager.is_running():\n raise InstanceNotRunningError(instance_manager.instance_config.name)\n\n dry_run = args.dry_run\n with output.prefix('[dry-run] ' if dry_run else ''):\n try:\n instance_manager.sync(output, dry_run)\n except NothingToDoError as e:\n output.write(str(e))\n return\n\n output.write('Done')\n" }, { "path": "spotty/commands/writers/__init__.py", "content": "" }, { "path": "spotty/commands/writers/abstract_output_writrer.py", "content": "from abc import ABC, abstractmethod\nfrom contextlib import contextmanager\n\n\nclass AbstractOutputWriter(ABC):\n\n def __init__(self):\n self._prefix = ''\n self._ignore_prefix = False\n\n @abstractmethod\n def _write(self, msg: str, newline: bool = True):\n raise NotImplementedError\n\n def write(self, msg: str = '', newline: bool = True):\n if not self._ignore_prefix:\n msg = '\\n'.join([self._prefix + line for line in msg.split('\\n')])\n\n self._write(msg, newline=newline)\n self._ignore_prefix = not newline\n\n @contextmanager\n def prefix(self, prefix):\n self._prefix += prefix\n yield\n self._prefix = self._prefix[:-len(prefix)]\n" }, { "path": "spotty/commands/writers/null_output_writrer.py", "content": "from spotty.commands.writers.abstract_output_writrer import AbstractOutputWriter\n\n\nclass NullOutputWriter(AbstractOutputWriter):\n\n def _write(self, msg: str, newline: bool = True):\n \"\"\"Does nothing.\"\"\"\n pass\n" }, { "path": "spotty/commands/writers/output_writrer.py", "content": "from spotty.commands.writers.abstract_output_writrer import AbstractOutputWriter\n\n\nclass OutputWriter(AbstractOutputWriter):\n\n def _write(self, msg: str, newline: bool = True):\n \"\"\"Prints messages to STDOUT.\"\"\"\n print(msg, end=('\\n' if newline else ''), flush=True)\n" }, { "path": "spotty/config/__init__.py", "content": "" }, { "path": "spotty/config/abstract_instance_config.py", "content": "import os\nfrom abc import ABC, abstractmethod\nfrom collections import OrderedDict, namedtuple\nfrom typing import List\nfrom spotty.config.container_config import ContainerConfig\nfrom spotty.config.project_config import ProjectConfig\nfrom spotty.config.tmp_dir_volume import TmpDirVolume\nfrom spotty.config.validation import DEFAULT_CONTAINER_NAME, is_subdir\nfrom spotty.config.abstract_instance_volume import AbstractInstanceVolume\nfrom spotty.deployment.abstract_cloud_instance.file_structure import INSTANCE_SPOTTY_TMP_DIR, CONTAINERS_TMP_DIR\nfrom spotty.utils import filter_list\n\n\nVolumeMount = namedtuple('VolumeMount', ['name', 'host_path', 'mount_path', 'mode', 'hidden'])\n\n\nclass AbstractInstanceConfig(ABC):\n\n def __init__(self, instance_config: dict, project_config: ProjectConfig):\n self._project_config = project_config\n\n # set instance parameters\n self._name = instance_config['name']\n self._provider_name = instance_config['provider']\n self._params = self._validate_instance_params(instance_config['parameters'])\n\n # get container config\n container_configs = filter_list(project_config.containers, 'name', self.container_name)\n if not container_configs:\n raise ValueError('Container configuration with the name \"%s\" not found.' % self.container_name)\n\n self._container_config = ContainerConfig(container_configs[0])\n\n # get volumes\n self._volumes = self._get_volumes()\n\n # get container volume mounts\n self._volume_mounts = self._get_volume_mounts(self._volumes)\n\n # get the host project directory\n self._host_project_dir = self._get_host_project_dir(self._volume_mounts)\n\n @abstractmethod\n def _validate_instance_params(self, params: dict) -> dict:\n \"\"\"Validates instance parameters and fill missing ones with the default values.\"\"\"\n raise NotImplementedError\n\n @abstractmethod\n def _get_instance_volumes(self) -> List[AbstractInstanceVolume]:\n \"\"\"Returns specific to the provider volumes that should be mounted on the host OS.\"\"\"\n raise NotImplementedError\n\n @property\n def project_config(self) -> ProjectConfig:\n return self._project_config\n\n @property\n def container_config(self) -> ContainerConfig:\n return self._container_config\n\n @property\n @abstractmethod\n def user(self) -> str:\n raise NotImplementedError\n\n @property\n def name(self) -> str:\n \"\"\"Name of the instance.\"\"\"\n return self._name\n\n @property\n def provider_name(self):\n \"\"\"Provider name.\"\"\"\n return self._provider_name\n\n @property\n def container_name(self) -> str:\n return self._params['containerName'] if self._params['containerName'] else DEFAULT_CONTAINER_NAME\n\n @property\n def full_container_name(self) -> str:\n \"\"\"A container name that is used in the \"docker run\" command.\"\"\"\n return ('spotty-%s-%s-%s' % (self.project_config.project_name, self.name, self.container_name)).lower()\n\n @property\n def docker_data_root(self) -> str:\n \"\"\"Data root directory for Docker daemon.\"\"\"\n return self._params['dockerDataRoot']\n\n @property\n def local_ssh_port(self) -> int:\n \"\"\"Local SSH port to connect to the instance (in case of a tunnel).\"\"\"\n return self._params['localSshPort']\n\n @property\n def commands(self) -> str:\n \"\"\"Commands that should be run once an instance is started.\"\"\"\n return self._params['commands']\n\n @property\n def host_project_dir(self):\n \"\"\"Project directory on the host OS.\"\"\"\n return self._host_project_dir\n\n @property\n def volumes(self) -> List[AbstractInstanceVolume]:\n return self._volumes\n\n @property\n def volume_mounts(self) -> List[VolumeMount]:\n return self._volume_mounts\n\n @property\n def dockerfile_path(self):\n \"\"\"Dockerfile path on the host OS.\"\"\"\n dockerfile_path = self.container_config.file\n if dockerfile_path:\n dockerfile_path = self.host_project_dir + '/' + dockerfile_path\n\n return dockerfile_path\n\n @property\n def docker_context_path(self):\n \"\"\"Docker build's context path on the host OS.\"\"\"\n dockerfile_path = self.dockerfile_path\n if not dockerfile_path:\n return ''\n\n return os.path.dirname(dockerfile_path)\n\n @property\n def host_container_dir(self):\n \"\"\"A temporary directory on the host OS that contains container-related files and directories.\"\"\"\n return '%s/%s' % (CONTAINERS_TMP_DIR, self.full_container_name)\n\n @property\n def host_logs_dir(self):\n \"\"\"A directory mainly for the \"spotty run\" command logs.\"\"\"\n return self.host_container_dir + '/logs'\n\n @property\n def host_volumes_dir(self):\n \"\"\"A directory with temporary volumes. If there is a Volume Mount in the configuration file\n that doesn't have a corresponding instance volume, a temporary directory will be created\n and attached to the container.\n \"\"\"\n return self.host_container_dir + '/volumes'\n\n def _get_volumes(self) -> List[AbstractInstanceVolume]:\n \"\"\"Returns volumes that should be mounted on the host OS.\"\"\"\n volumes = self._get_instance_volumes()\n\n # create temporary volumes for the volume mounts that don't have corresponding\n # volumes in the instance configuration\n instance_volume_names = set(volume.name for volume in volumes)\n for container_volume in self.container_config.volume_mounts:\n if container_volume['name'] not in instance_volume_names:\n volumes.append(TmpDirVolume(volume_config={\n 'name': container_volume['name'],\n 'parameters': {'path': '%s/%s' % (self.host_volumes_dir, container_volume['name'])}\n }))\n\n return volumes\n\n def _get_volume_mounts(self, volumes: List[AbstractInstanceVolume]) \\\n -> List[VolumeMount]:\n \"\"\"Returns container volume mounts and a path to the project directory on the host OS.\"\"\"\n # get mount directories for the volumes\n host_paths = OrderedDict([(volume.name, volume.host_path) for volume in volumes])\n\n # get container volumes mapping\n volume_mounts = []\n for container_volume in self.container_config.volume_mounts:\n volume_mounts.append(VolumeMount(\n name=container_volume['name'],\n host_path=host_paths[container_volume['name']],\n mount_path=container_volume['mountPath'],\n mode='rw',\n hidden=False,\n ))\n\n return volume_mounts\n\n def _get_host_project_dir(self, volume_mounts: List[VolumeMount]) -> str:\n \"\"\"Returns the host project directory.\"\"\"\n host_project_dir = None\n for volume_mount in sorted(volume_mounts, key=lambda x: len(x.mount_path), reverse=True):\n if is_subdir(self.container_config.project_dir, volume_mount.mount_path):\n # the project directory is a subdirectory of a Volume Mount directory\n project_subdir = os.path.relpath(self.container_config.project_dir, volume_mount.mount_path)\n host_project_dir = os.path.normpath(volume_mount.host_path + '/' + project_subdir)\n break\n\n # this should not be the case as the volume mount for the project directory should be added automatically\n # if it doesn't exist in the configuration\n assert host_project_dir is not None, 'A volume mount that contains the project directory not found.'\n\n return host_project_dir\n" }, { "path": "spotty/config/abstract_instance_volume.py", "content": "from abc import ABC, abstractmethod\n\n\nclass AbstractInstanceVolume(ABC):\n\n def __init__(self, volume_config: dict):\n self._name = volume_config['name']\n self._params = self._validate_volume_parameters(volume_config['parameters'])\n\n @abstractmethod\n def _validate_volume_parameters(self, params: dict) -> dict:\n raise NotImplementedError\n\n @property\n def name(self) -> str:\n \"\"\"Unique name of the volume that will be used for the deployment.\"\"\"\n return self._name\n\n @property\n @abstractmethod\n def host_path(self) -> str:\n \"\"\"A path on the host OS that will be mounted to the container.\"\"\"\n raise NotImplementedError\n\n @property\n @abstractmethod\n def title(self) -> str:\n \"\"\"A title for the volume type.\n It will be used to display information about the volumes during the deployment.\n \"\"\"\n raise NotImplementedError\n\n @property\n def deletion_policy_title(self) -> str:\n \"\"\"A title for the volume's deletion policy.\n It will be used to display information about the volumes during the deployment.\n \"\"\"\n return ''\n" }, { "path": "spotty/config/config_utils.py", "content": "import os\nfrom collections import namedtuple\nimport yaml\nfrom spotty.config.project_config import ProjectConfig\nfrom spotty.config.validation import DEFAULT_CONTAINER_NAME\n\n\nDEFAULT_CONFIG_FILENAME = 'spotty.yaml'\nOVERRIDE_CONFIG_FILENAME = 'spotty.override.yaml'\n\n\ndef load_config(config_path: str = None) -> ProjectConfig:\n # get project directory\n if not config_path:\n config_path = DEFAULT_CONFIG_FILENAME\n\n if os.path.isabs(config_path):\n config_abs_path = config_path\n else:\n config_abs_path = os.path.abspath(os.path.join(os.getcwd(), config_path))\n\n if not os.path.exists(config_abs_path):\n raise ValueError('Configuration file \"%s\" not found.' % config_path)\n\n # get the project directory\n project_dir = os.path.dirname(config_abs_path)\n\n # read the config\n config = _read_yaml(config_abs_path)\n\n # update the config if an override config exists\n if os.path.basename(config_abs_path) == DEFAULT_CONFIG_FILENAME:\n override_config_abs_path = os.path.join(project_dir, OVERRIDE_CONFIG_FILENAME)\n if os.path.isfile(override_config_abs_path):\n override_config = _read_yaml(override_config_abs_path)\n config = _merge_configs(config, override_config)\n\n # get project configuration\n project_config = ProjectConfig(config, project_dir)\n\n return project_config\n\n\ndef _read_yaml(file_path: str):\n \"\"\"Returns content of the YAML file.\"\"\"\n with open(file_path, 'r') as f:\n res = yaml.safe_load(f)\n\n return res\n\n\ndef _merge_configs(orig_config, override_config):\n \"\"\"Merges original config with the override config.\"\"\"\n\n MergeRule = namedtuple('MergeRule', ['key', 'merge_key', 'default_value', 'has_default_value'])\n\n merge_rules = [MergeRule(\n key='containers',\n merge_key='name',\n default_value=DEFAULT_CONTAINER_NAME,\n has_default_value=True,\n ), MergeRule(\n key='instances',\n merge_key='name',\n default_value=None,\n has_default_value=False,\n )]\n\n # validate and merge lists by keys\n for rule in merge_rules:\n if override_config and (rule.key in orig_config) and (rule.key in override_config):\n if not isinstance(orig_config[rule.key], list):\n raise ValueError('The \"%s\" key in the config must be a list.' % rule.key)\n\n if not isinstance(override_config[rule.key], list):\n raise ValueError('The \"%s\" key in the override config must be a list.' % rule.key)\n\n # convert lists to dictionaries\n dicts_to_merge = []\n for list_to_merge in [orig_config[rule.key], override_config[rule.key]]:\n dict_to_merge = {}\n for item in list_to_merge:\n if not isinstance(item, dict):\n raise ValueError('Each item of the \"%s\" list must be a dictionary.' % rule.key)\n\n if rule.merge_key in item:\n merge_value = item[rule.merge_key]\n elif rule.has_default_value:\n merge_value = rule.default_value\n else:\n raise ValueError('Each item of the \"%s\" list must contain the \"%s\" field.'\n % (rule.key, rule.merge_key))\n\n if merge_value in dict_to_merge:\n raise ValueError('Each item of the \"%s\" list must have a unique \"%s\" value.'\n % (rule.key, rule.merge_key))\n\n dict_to_merge[merge_value] = item\n\n dicts_to_merge.append(dict_to_merge)\n\n # merge lists\n merged_dict = _update_dict(*dicts_to_merge)\n orig_config[rule.key] = [{**item, rule.merge_key: key} for key, item in merged_dict.items()]\n del override_config[rule.key]\n\n # merge the rest of the override config\n config = _update_dict(orig_config, override_config)\n\n return config\n\n\ndef _update_dict(d, u):\n if not isinstance(u, dict):\n return d\n\n if not isinstance(d, dict):\n return u\n\n for k, v in u.items():\n if isinstance(d, dict):\n if isinstance(v, dict):\n d[k] = _update_dict(d.get(k, {}), v)\n else:\n d[k] = u[k]\n else:\n d = {k: u[k]}\n\n return d\n" }, { "path": "spotty/config/container_config.py", "content": "from typing import List\nfrom spotty.config.validation import is_subdir\n\n\nPROJECT_VOLUME_MOUNT_NAME = '.project'\n\n\nclass ContainerConfig(object):\n\n def __init__(self, container_config: dict):\n self._config = container_config\n self._volume_mounts = self._get_volume_mounts()\n\n @property\n def name(self) -> str:\n return self._config['name']\n\n @property\n def project_dir(self) -> str:\n return self._config['projectDir']\n\n @property\n def image(self) -> str:\n return self._config['image']\n\n @property\n def file(self) -> str:\n return self._config['file']\n\n @property\n def run_as_host_user(self) -> str:\n return self._config['runAsHostUser']\n\n @property\n def volume_mounts(self) -> list:\n return self._volume_mounts\n\n @property\n def commands(self) -> str:\n return self._config['commands']\n\n @property\n def working_dir(self) -> str:\n \"\"\"Working directory for the Docker container.\"\"\"\n working_dir = self._config['workingDir']\n if not working_dir:\n working_dir = self._config['projectDir']\n\n return working_dir\n\n @property\n def env(self) -> dict:\n return self._config['env']\n\n @property\n def host_network(self) -> bool:\n return self._config['hostNetwork']\n\n @property\n def ports(self) -> List[dict]:\n return self._config['ports']\n\n @property\n def runtime_parameters(self) -> list:\n return self._config['runtimeParameters']\n\n def _get_volume_mounts(self):\n \"\"\"Returns container volume mounts from the configuration and\n adds the project volume mount if necessary.\"\"\"\n\n volume_mounts = self._config['volumeMounts']\n\n # check if the project directory is a sub-directory of one of the volume mounts\n project_has_volume = False\n for volume_mount in volume_mounts:\n if is_subdir(self.project_dir, volume_mount['mountPath']):\n project_has_volume = True\n break\n\n # if it's not, then add new volume mount\n if not project_has_volume:\n volume_mounts.insert(0, {\n 'name': PROJECT_VOLUME_MOUNT_NAME,\n 'mountPath': self.project_dir,\n })\n\n return volume_mounts\n" }, { "path": "spotty/config/host_path_volume.py", "content": "import os\nfrom spotty.config.abstract_instance_volume import AbstractInstanceVolume\nfrom spotty.config.validation import validate_host_path_volume_parameters\n\n\nclass HostPathVolume(AbstractInstanceVolume):\n\n TYPE_NAME = 'HostPath'\n\n def __init__(self, volume_config: dict, base_dir: str = None):\n super().__init__(volume_config)\n\n self._base_dir = base_dir\n\n def _validate_volume_parameters(self, params: dict) -> dict:\n return validate_host_path_volume_parameters(params)\n\n @property\n def title(self):\n return 'HostPath volume'\n\n @property\n def name(self):\n return self._name\n\n @property\n def deletion_policy_title(self) -> str:\n return ''\n\n @property\n def host_path(self) -> str:\n \"\"\"A path on the host OS that will be mounted to the container.\"\"\"\n path = os.path.expanduser(self._params['path'])\n if not os.path.isabs(path):\n if self._base_dir is not None:\n path = os.path.join(self._base_dir, path)\n else:\n raise ValueError('Use absolute path for the \"%s\" volume.' % self.name)\n\n return path\n" }, { "path": "spotty/config/project_config.py", "content": "from spotty.config.validation import validate_basic_config\n\n\nclass ProjectConfig(object):\n\n def __init__(self, config: dict, project_dir: str):\n # validate the config\n config = validate_basic_config(config)\n\n self._project_dir = project_dir\n self._config = config\n\n @property\n def project_dir(self) -> str:\n return self._project_dir\n\n @property\n def project_name(self) -> str:\n return self._config['project']['name']\n\n @property\n def sync_filters(self) -> list:\n return self._config['project']['syncFilters']\n\n @property\n def containers(self) -> list:\n return self._config['containers']\n\n @property\n def instances(self) -> list:\n return self._config['instances']\n\n @property\n def scripts(self) -> dict:\n return self._config['scripts']\n" }, { "path": "spotty/config/tmp_dir_volume.py", "content": "from spotty.config.host_path_volume import HostPathVolume\n\n\nclass TmpDirVolume(HostPathVolume):\n\n @property\n def title(self):\n return 'temporary directory'\n\n @property\n def deletion_policy_title(self) -> str:\n return ''\n" }, { "path": "spotty/config/validation.py", "content": "import os\nfrom typing import List\nfrom schema import Schema, And, Use, Optional, Or, Regex, Hook, SchemaError, SchemaForbiddenKeyError\n\n\nDEFAULT_CONTAINER_NAME = 'default'\n\n\ndef validate_basic_config(data):\n\n container = And(\n {\n Optional('name', default=DEFAULT_CONTAINER_NAME): And(str, Regex(r'^[\\w-]+$')),\n 'projectDir': And(str,\n And(os.path.isabs,\n error='Use an absolute path when specifying the project directory'),\n Use(lambda x: x.rstrip('/'))\n ),\n Optional('image', default=''): And(str, len),\n Optional('file', default=''): And(str, # TODO: a proper regex that the filename is valid\n Regex(r'^[\\w\\.\\/@-]*$',\n error='Invalid name for a Dockerfile'),\n And(lambda x: not x.endswith('/'),\n error='Invalid name for a Dockerfile'),\n And(lambda x: not os.path.isabs(x),\n error='Path to the Dockerfile should be relative to the '\n 'project\\'s root directory.'),\n ),\n Optional('runAsHostUser', default=False): bool,\n Optional('volumeMounts', default=[]): (And(\n [{\n 'name': And(Or(int, str), Use(str), Regex(r'^[\\w-]+$')),\n 'mountPath': And(\n str,\n And(os.path.isabs, error='Use an absolute path when specifying a mount directory'),\n Use(lambda x: x.rstrip('/')),\n ),\n }],\n And(lambda x: is_unique_value(x, 'name'),\n error='Each volume mount must have a unique name.'),\n And(lambda x: not has_prefix([(volume['mountPath'] + '/') for volume in x]),\n error='Volume mount paths cannot be prefixes for each other.'),\n )),\n Optional('workingDir', default=''): And(str,\n And(os.path.isabs,\n error='Use an absolute path when specifying a '\n 'working directory'),\n Use(lambda x: x.rstrip('/'))\n ),\n Optional('env', default={}): {\n And(str, Regex(r'^[a-zA-Z_]+[a-zA-Z0-9_]*$')): str,\n },\n Optional('hostNetwork', default=False): bool,\n Optional('ports', default=[]): [{\n 'containerPort': And(int, lambda x: 0 < x < 65536),\n Optional('hostPort', default=None): And(int, lambda x: 0 < x < 65536),\n }],\n Optional('commands', default=''): str,\n # TODO: allow to use only certain runtime parameters\n Optional('runtimeParameters', default=[]): And([str], Use(lambda x: [p.strip() for p in x])),\n },\n And(lambda x: x['image'] or x['file'], error='Either \"image\" or \"file\" should be specified.'),\n And(lambda x: not (x['image'] and x['file']), error='\"image\" and \"file\" cannot be specified together.'),\n And(lambda x: not (x['hostNetwork'] and x['ports']),\n error='Published ports and the host network mode cannot be used together.'),\n )\n\n schema = Schema({\n 'project': {\n 'name': And(str, Regex(r'^[a-zA-Z0-9][a-zA-Z0-9-]{,26}[a-zA-Z0-9]$')),\n Optional('syncFilters', default=[]): And(\n [And(\n {\n Optional('exclude'): [And(str, len, And(lambda x: '**' not in x,\n error='Use single asterisks (\"*\") in sync filters'))],\n Optional('include'): [And(str, len, And(lambda x: '**' not in x,\n error='Use single asterisks (\"*\") in sync filters'))],\n },\n And(lambda x: x, error='Either \"exclude\" or \"include\" filter should be specified.'),\n And(lambda x: not ('exclude' in x and 'include' in x),\n error='\"exclude\" and \"include\" filters should be specified as different list items.'),\n )],\n error='\"project.syncFilters\" field must be a list.',\n )\n },\n WrongKey('container', error='Use \"containers\" field instead of \"container\".'): object,\n Optional('containers', default=[]): And(\n [container],\n And(lambda x: is_unique_value(x, 'name'), error='Each container must have a unique name.'),\n error='\"containers\" field must be a list.',\n ),\n WrongKey('instance', error='Use \"instances\" field instead of \"instance\".'): object,\n 'instances': And(\n [{\n 'name': And(Or(int, str), Use(str), Regex(r'^[\\w-]+$')),\n 'provider': str,\n Optional('parameters', default={}): {\n And(str, Regex(r'^[\\w]+$')): object,\n }\n }],\n And(lambda x: len(x), error='At least one instance must be specified in the configuration file.'),\n And(lambda x: is_unique_value(x, 'name'), error='Each instance must have a unique name.'),\n ),\n Optional('scripts', default={}): {\n And(str, Regex(r'^[\\w-]+$')): And(str, len),\n },\n })\n\n return validate_config(schema, data)\n\n\ndef validate_host_path_volume_parameters(params: dict):\n schema = Schema({\n 'path': And(str, Use(lambda x: x.rstrip('/'))),\n })\n\n return validate_config(schema, params)\n\n\ndef get_instance_parameters_schema(instance_parameters: dict, default_volume_type: str,\n instance_checks: list = None, volumes_checks: list = None):\n if not instance_checks:\n instance_checks = []\n\n if not volumes_checks:\n volumes_checks = []\n\n schema = Schema(And(\n {\n **instance_parameters,\n Optional('containerName', default=None): And(str, Regex(r'^[\\w-]+$')),\n Optional('dockerDataRoot', default=''): And(\n str,\n And(os.path.isabs, error='Use an absolute path when specifying a Docker data root directory'),\n Use(lambda x: x.rstrip('/')),\n ),\n Optional('volumes', default=[]): And(\n [{\n 'name': And(Or(int, str), Use(str), Regex(r'^[\\w-]+$')),\n Optional('type', default=default_volume_type): str,\n Optional('parameters', default={}): {\n And(str, Regex(r'^[\\w]+$')): object,\n },\n }],\n And(lambda x: is_unique_value(x, 'name'), error='Each instance volume must have a unique name.'),\n *volumes_checks,\n ),\n Optional('localSshPort', default=None): Or(None, And(int, lambda x: 0 < x < 65536)),\n Optional('commands', default=''): str,\n },\n And(lambda x: not x['dockerDataRoot'] or any([True for v in x['volumes'] if v['parameters'].get('mountDir') and\n is_subdir(x['dockerDataRoot'], v['parameters']['mountDir'])]),\n error='The \"mountDir\" of one of the volumes must be a prefix for the \"dockerDataRoot\" path.'),\n *instance_checks\n ))\n\n return schema\n\n\ndef is_unique_value(x: List[dict], key):\n \"\"\"Returns \"True\" if all values of the key in the list of dictionaries are unique.\"\"\"\n return len(x) == len(set([v[key] for v in x]))\n\n\ndef has_prefix(x: list):\n \"\"\"Returns \"True\" if some value in the list is a prefix for another value in this list.\"\"\"\n for val in x:\n if len(list(filter(val.startswith, x))) > 1:\n return True\n\n return False\n\n\ndef is_subdir(subdir_path, dir_path):\n \"\"\"Returns \"True\" if it's the second path parameter is a subdirectory of the first path parameter.\"\"\"\n return (subdir_path.rstrip('/') + '/').startswith(dir_path.rstrip('/') + '/')\n\n\ndef validate_config(schema: Schema, config):\n try:\n validated = schema.validate(config)\n except SchemaError as e:\n raise ValueError('Validation error: ' + (e.errors[-1] if e.errors[-1] else e.autos[-1]))\n\n return validated\n\n\nclass WrongKey(Hook):\n def __init__(self, *args, **kwargs):\n super().__init__(*args, **kwargs, handler=self.raise_error)\n\n def raise_error(self, key, *args):\n raise SchemaForbiddenKeyError(self._error)\n" }, { "path": "spotty/configuration.py", "content": "import os\n\n\ndef get_spotty_config_dir():\n \"\"\"Spotty configuration directory.\"\"\"\n path = os.path.join(os.path.expanduser('~'), '.spotty')\n if not os.path.isdir(path):\n os.makedirs(path, mode=0o755, exist_ok=True)\n\n return path\n\n\ndef get_spotty_keys_dir(provider_name: str):\n \"\"\"\"Spotty keys directory.\"\"\"\n path = os.path.join(get_spotty_config_dir(), 'keys', provider_name)\n if not os.path.isdir(path):\n os.makedirs(path, mode=0o755, exist_ok=True)\n\n return path\n" }, { "path": "spotty/deployment/__init__.py", "content": "" }, { "path": "spotty/deployment/abstract_cloud_instance/__init__.py", "content": "" }, { "path": "spotty/deployment/abstract_cloud_instance/abstract_bucket_manager.py", "content": "from abc import ABC\nfrom spotty.deployment.abstract_cloud_instance.resources.abstract_bucket import AbstractBucket\n\n\nclass AbstractBucketManager(ABC):\n\n def __init__(self, project_name: str):\n self._project_name = project_name\n\n @property\n def project_name(self) -> str:\n return self._project_name\n\n def get_bucket(self) -> AbstractBucket:\n raise NotImplementedError\n\n def create_bucket(self) -> AbstractBucket:\n raise NotImplementedError\n" }, { "path": "spotty/deployment/abstract_cloud_instance/abstract_cloud_instance_manager.py", "content": "import logging\nfrom abc import ABC, abstractmethod\nfrom spotty.commands.writers.abstract_output_writrer import AbstractOutputWriter\nfrom spotty.config.project_config import ProjectConfig\nfrom spotty.deployment.abstract_cloud_instance.abstract_data_transfer import AbstractDataTransfer\nfrom spotty.deployment.abstract_cloud_instance.abstract_instance_deployment import AbstractInstanceDeployment\nfrom spotty.deployment.abstract_cloud_instance.abstract_bucket_manager import AbstractBucketManager\nfrom spotty.deployment.abstract_cloud_instance.errors.bucket_not_found import BucketNotFoundError\nfrom spotty.errors.instance_not_running import InstanceNotRunningError\nfrom spotty.deployment.abstract_ssh_instance_manager import AbstractSshInstanceManager\n\n\nclass AbstractCloudInstanceManager(AbstractSshInstanceManager, ABC):\n\n def __init__(self, project_config: ProjectConfig, instance_config: dict):\n super().__init__(project_config, instance_config)\n\n self._bucket_manager = self._get_bucket_manager()\n self._data_transfer = self._get_data_transfer()\n self._instance_deployment = self._get_instance_deployment()\n\n @abstractmethod\n def _get_bucket_manager(self) -> AbstractBucketManager:\n \"\"\"Returns an bucket manager.\"\"\"\n raise NotImplementedError\n\n @abstractmethod\n def _get_data_transfer(self) -> AbstractDataTransfer:\n \"\"\"Returns a data transfer object.\"\"\"\n raise NotImplementedError\n\n @abstractmethod\n def _get_instance_deployment(self) -> AbstractInstanceDeployment:\n \"\"\"Returns an instance deployment manager.\"\"\"\n raise NotImplementedError\n\n @property\n def bucket_manager(self) -> AbstractBucketManager:\n \"\"\"Returns a bucket manager.\"\"\"\n return self._bucket_manager\n\n @property\n def data_transfer(self) -> AbstractDataTransfer:\n \"\"\"Returns a data transfer object.\"\"\"\n return self._data_transfer\n\n @property\n def instance_deployment(self) -> AbstractInstanceDeployment:\n \"\"\"Returns an instance deployment manager.\"\"\"\n return self._instance_deployment\n\n def is_running(self) -> bool:\n \"\"\"Checks if the instance is running.\"\"\"\n instance = self.instance_deployment.get_instance()\n return instance and instance.is_running\n\n def start(self, output: AbstractOutputWriter, dry_run=False):\n # make sure the Dockerfile exists\n self._check_dockerfile_exists()\n\n if not dry_run:\n # check if the instance is already running\n instance = self.instance_deployment.get_instance()\n if instance:\n if instance.is_running:\n print('Instance is already running. Are you sure you want to restart it?')\n res = input('Type \"y\" to confirm: ')\n if res != 'y':\n raise ValueError('The operation was cancelled.')\n\n # terminating the instance to make EBS volumes available (the stack will be deleted later)\n output.write('Terminating the instance... ', newline=False)\n instance.terminate()\n output.write('DONE')\n\n elif instance.is_stopped:\n # TODO: restart the instance if it stopped\n pass\n\n # create or get existing bucket for the project\n bucket_name = None\n try:\n bucket_name = self.bucket_manager.get_bucket().name\n except BucketNotFoundError:\n if not dry_run:\n bucket_name = self.bucket_manager.create_bucket().name\n output.write('Bucket \"%s\" was created.' % bucket_name)\n\n # deploy the instance\n self.instance_deployment.deploy(\n container_commands=self.container_commands,\n bucket_name=bucket_name,\n data_transfer=self.data_transfer,\n output=output,\n dry_run=dry_run,\n )\n\n def stop(self, only_shutdown: bool, output: AbstractOutputWriter):\n if only_shutdown:\n output.write('Shutting down the instance... ', newline=False)\n self.instance_deployment.get_instance().stop()\n output.write('DONE')\n else:\n # delete the stack and apply deletion policies\n self.instance_deployment.delete(output)\n\n def clean(self, output: AbstractOutputWriter):\n pass\n\n def sync(self, output: AbstractOutputWriter, dry_run=False):\n # get the project bucket name\n bucket_name = self.bucket_manager.get_bucket().name\n\n # sync the project with the S3 bucket\n output.write('Syncing the project with the bucket...')\n self.data_transfer.upload_local_to_bucket(bucket_name, dry_run=dry_run)\n\n if not dry_run:\n # sync the S3 bucket with the instance\n output.write('Syncing the bucket with the instance...')\n remote_cmd = self.data_transfer.get_download_bucket_to_instance_command(\n bucket_name=bucket_name,\n use_sudo=(not self.instance_config.container_config.run_as_host_user),\n )\n logging.debug('Remote sync command: ' + remote_cmd)\n\n # execute the command on the host OS\n exit_code = self.exec(remote_cmd)\n if exit_code != 0:\n raise ValueError('Failed to download files from the bucket to the instance')\n\n def download(self, download_filters: list, output: AbstractOutputWriter, dry_run=False):\n # get the project bucket name\n bucket_name = self.bucket_manager.get_bucket().name\n\n # sync files from the instance to a temporary S3 directory\n output.write('Uploading files from the instance to the bucket...')\n remote_cmd = self.data_transfer.get_upload_instance_to_bucket_command(\n bucket_name=bucket_name,\n download_filters=download_filters,\n use_sudo=(not self.instance_config.container_config.run_as_host_user),\n dry_run=dry_run,\n )\n logging.debug('Remote sync command: ' + remote_cmd)\n\n # execute the command on the host OS\n exit_code = self.exec(remote_cmd)\n if exit_code != 0:\n raise ValueError('Failed to upload files from the instance to the bucket')\n\n if not dry_run:\n # sync the project with the S3 bucket\n output.write('Downloading files from the bucket to local...')\n self.data_transfer.download_bucket_to_local(bucket_name=bucket_name, download_filters=download_filters)\n\n @property\n def ssh_host(self):\n \"\"\"Returns an IP address that will be used for SSH connections.\"\"\"\n if self._instance_config.local_ssh_port:\n return '127.0.0.1'\n\n # get a public IP address of the running instance\n instance = self.instance_deployment.get_instance()\n if not instance or not instance.is_running:\n raise InstanceNotRunningError(self.instance_config.name)\n\n instance_ip_address = instance.public_ip_address if instance.public_ip_address else instance.private_ip_address\n if not instance_ip_address:\n raise ValueError('Instance IP address not found')\n\n return instance_ip_address\n\n @property\n def ssh_port(self) -> int:\n if self._instance_config.local_ssh_port:\n return self._instance_config.local_ssh_port\n\n return 22\n\n @property\n def use_tmux(self) -> bool:\n return True\n" }, { "path": "spotty/deployment/abstract_cloud_instance/abstract_data_transfer.py", "content": "from abc import ABC, abstractmethod\n\n\nclass AbstractDataTransfer(ABC):\n\n def __init__(self, local_project_dir: str, host_project_dir: str, sync_filters: list, instance_name: str):\n self._instance_name = instance_name\n self._local_project_dir = local_project_dir\n self._host_project_dir = host_project_dir\n self._sync_filters = sync_filters\n\n @property\n def instance_name(self):\n return self._instance_name\n\n @property\n @abstractmethod\n def scheme_name(self) -> str:\n raise NotImplementedError\n\n def _get_bucket_project_path(self, bucket_name: str) -> str:\n \"\"\"A bucket path where the project files are located.\"\"\"\n return '%s://%s/project' % (self.scheme_name, bucket_name)\n\n def _get_bucket_downloads_path(self, bucket_name: str) -> str:\n \"\"\"A bucket path where the downloaded files are located.\"\"\"\n return '%s://%s/download/instance-%s' % (self.scheme_name, bucket_name, self.instance_name)\n\n @abstractmethod\n def upload_local_to_bucket(self, bucket_name: str, dry_run: bool = False):\n \"\"\"Uploads files from local to the bucket.\"\"\"\n raise NotImplementedError\n\n @abstractmethod\n def download_bucket_to_local(self, bucket_name: str, download_filters: list):\n \"\"\"Downloads files from the bucket to local.\"\"\"\n raise NotImplementedError\n\n @abstractmethod\n def get_download_bucket_to_instance_command(self, bucket_name: str, use_sudo: bool = False) -> str:\n \"\"\"A remote command to download files from the bucket to the instance.\"\"\"\n raise NotImplementedError\n\n @abstractmethod\n def get_upload_instance_to_bucket_command(self, bucket_name: str, download_filters: list, use_sudo: bool = False,\n dry_run: bool = False) -> str:\n \"\"\"A remote command to upload files from the instance to the bucket.\"\"\"\n raise NotImplementedError\n" }, { "path": "spotty/deployment/abstract_cloud_instance/abstract_instance_deployment.py", "content": "from abc import abstractmethod, ABC\nfrom spotty.commands.writers.abstract_output_writrer import AbstractOutputWriter\nfrom spotty.config.abstract_instance_config import AbstractInstanceConfig\nfrom spotty.deployment.abstract_cloud_instance.abstract_data_transfer import AbstractDataTransfer\nfrom spotty.deployment.abstract_cloud_instance.resources.abstract_instance import AbstractInstance\nfrom spotty.deployment.container.abstract_container_commands import AbstractContainerCommands\n\n\nclass AbstractInstanceDeployment(ABC):\n\n def __init__(self, instance_config: AbstractInstanceConfig):\n self._instance_config = instance_config\n\n @property\n def instance_config(self) -> AbstractInstanceConfig:\n return self._instance_config\n\n @abstractmethod\n def get_instance(self) -> AbstractInstance:\n \"\"\"Returns information about the instance it it exists.\"\"\"\n raise NotImplementedError\n\n @abstractmethod\n def deploy(self, container_commands: AbstractContainerCommands, bucket_name: str,\n data_transfer: AbstractDataTransfer, output: AbstractOutputWriter, dry_run: bool = False):\n \"\"\"Deploys or redeploys the instance.\"\"\"\n raise NotImplementedError\n\n @abstractmethod\n def delete(self, output: AbstractOutputWriter):\n \"\"\"Deletes the stack with the instance and applies deletion policies for the volumes.\"\"\"\n raise NotImplementedError\n" }, { "path": "spotty/deployment/abstract_cloud_instance/errors/__init__.py", "content": "" }, { "path": "spotty/deployment/abstract_cloud_instance/errors/bucket_not_found.py", "content": "class BucketNotFoundError(Exception):\n def __init__(self):\n super().__init__('Bucket for the project not found.')\n" }, { "path": "spotty/deployment/abstract_cloud_instance/file_structure.py", "content": "\"\"\"\nINSTANCE FILE STRUCTURE\n\"\"\"\n\n# a base temporary directory on an instance\nINSTANCE_SPOTTY_TMP_DIR = '/tmp/spotty'\n\n# a base directory for container-related files and directories\nCONTAINERS_TMP_DIR = INSTANCE_SPOTTY_TMP_DIR + '/containers'\n\n# a base directory for instance-related files and directories\nINSTANCE_DIR = INSTANCE_SPOTTY_TMP_DIR + '/instance'\n\n# helper scripts\nINSTANCE_SCRIPTS_DIR = INSTANCE_DIR + '/scripts'\n\n# instance startup scripts\nINSTANCE_STARTUP_SCRIPTS_DIR = INSTANCE_SCRIPTS_DIR + '/startup'\n\n# a path to the script that attaches user to the container\nCONTAINER_BASH_SCRIPT_PATH = INSTANCE_SCRIPTS_DIR + '/container_bash.sh'\n" }, { "path": "spotty/deployment/abstract_cloud_instance/resources/__init__.py", "content": "" }, { "path": "spotty/deployment/abstract_cloud_instance/resources/abstract_bucket.py", "content": "from abc import ABC\n\n\nclass AbstractBucket(ABC):\n\n @property\n def name(self):\n raise NotImplementedError\n" }, { "path": "spotty/deployment/abstract_cloud_instance/resources/abstract_instance.py", "content": "from abc import ABC\n\n\nclass AbstractInstance(ABC):\n\n @property\n def public_ip_address(self):\n raise NotImplementedError\n\n @property\n def private_ip_address(self):\n raise NotImplementedError\n\n @property\n def is_running(self):\n \"\"\"Returns true if the instance is running.\"\"\"\n raise NotImplementedError\n\n @property\n def is_stopped(self):\n \"\"\"Returns true if the instance is stopped, so it can be restarted.\"\"\"\n raise NotImplementedError\n\n def terminate(self, wait: bool = True):\n raise NotImplementedError\n\n def stop(self, wait: bool = True):\n raise NotImplementedError\n" }, { "path": "spotty/deployment/abstract_docker_instance_manager.py", "content": "import os\nfrom abc import ABC\nfrom spotty.commands.writers.abstract_output_writrer import AbstractOutputWriter\nfrom spotty.deployment.utils.commands import get_script_command\nfrom spotty.deployment.container.docker.docker_commands import DockerCommands\nfrom spotty.deployment.container.docker.scripts.start_container_script import StartContainerScript\nfrom spotty.deployment.container.docker.scripts.stop_container_script import StopContainerScript\nfrom spotty.deployment.abstract_instance_manager import AbstractInstanceManager\nfrom spotty.errors.nothing_to_do import NothingToDoError\nfrom spotty.utils import render_table\n\n\nclass AbstractDockerInstanceManager(AbstractInstanceManager, ABC):\n\n @property\n def container_commands(self) -> DockerCommands:\n \"\"\"A collection of commands to manage a container from the host OS.\"\"\"\n return DockerCommands(self.instance_config)\n\n def is_container_running(self) -> bool:\n \"\"\"Checks if the container is running.\"\"\"\n is_running_cmd = self.container_commands.is_created(is_running=True)\n exit_code = self.exec(is_running_cmd)\n\n return exit_code == 0\n\n def start_container(self, output: AbstractOutputWriter, dry_run=False):\n \"\"\"Starts or restarts container on the host OS.\"\"\"\n # make sure the Dockerfile exists\n self._check_dockerfile_exists()\n\n # sync the project with the instance\n try:\n self.sync(output, dry_run=dry_run)\n except NothingToDoError:\n pass\n\n # generate a script that starts container\n start_container_script = StartContainerScript(self.container_commands).render()\n start_container_command = get_script_command('start-container', start_container_script)\n\n # start the container\n exit_code = self.exec(start_container_command)\n if exit_code != 0:\n raise ValueError('Failed to start the container')\n\n def start(self, output: AbstractOutputWriter, dry_run=False):\n # start or restart container\n self.start_container(output, dry_run=dry_run)\n\n def stop(self, only_shutdown: bool, output: AbstractOutputWriter):\n # stop container\n stop_container_script = StopContainerScript(self.container_commands).render()\n stop_container_command = get_script_command('stop-container', stop_container_script)\n\n exit_code = self.exec(stop_container_command)\n if exit_code != 0:\n raise ValueError('Failed to stop the container')\n\n def get_status_text(self):\n if self.is_container_running():\n msg = 'Container is running.'\n else:\n msg = 'Container is not running.'\n\n return render_table([(msg,)])\n\n def _check_dockerfile_exists(self):\n \"\"\"Raises an error if a Dockerfile specified in the configuration file but doesn't exist.\"\"\"\n if self.instance_config.container_config.file:\n dockerfile_path = os.path.join(self.project_config.project_dir, self.instance_config.container_config.file)\n if not os.path.isfile(dockerfile_path):\n raise FileNotFoundError('A Dockerfile specified in the container configuration doesn\\'t exist:\\n ' +\n dockerfile_path)\n" }, { "path": "spotty/deployment/abstract_instance_manager.py", "content": "import subprocess\nfrom abc import ABC, abstractmethod\nfrom spotty.commands.writers.abstract_output_writrer import AbstractOutputWriter\nfrom spotty.config.abstract_instance_config import AbstractInstanceConfig\nfrom spotty.config.project_config import ProjectConfig\nfrom spotty.deployment.container.abstract_container_commands import AbstractContainerCommands\n\n\nclass AbstractInstanceManager(ABC):\n\n def __init__(self, project_config: ProjectConfig, instance_config: dict):\n self._project_config = project_config\n self._instance_config = self._get_instance_config(instance_config)\n\n @property\n def project_config(self) -> ProjectConfig:\n return self._project_config\n\n @property\n def instance_config(self) -> AbstractInstanceConfig:\n return self._instance_config\n\n @abstractmethod\n def _get_instance_config(self, instance_config: dict) -> AbstractInstanceConfig:\n \"\"\"A factory method to create a provider's instance config.\"\"\"\n raise NotImplementedError\n\n @property\n @abstractmethod\n def container_commands(self) -> AbstractContainerCommands:\n \"\"\"A collection of commands to manage a container from the host OS.\"\"\"\n raise NotImplementedError\n\n @abstractmethod\n def is_running(self) -> bool:\n \"\"\"Checks if the instance is running.\"\"\"\n raise NotImplementedError\n\n @abstractmethod\n def start(self, output: AbstractOutputWriter, dry_run=False):\n \"\"\"Creates a stack with the instance.\"\"\"\n raise NotImplementedError\n\n @abstractmethod\n def start_container(self, output: AbstractOutputWriter, dry_run=False):\n \"\"\"Starts or restarts container on the host OS.\"\"\"\n raise NotImplementedError\n\n @abstractmethod\n def stop(self, only_shutdown: bool, output: AbstractOutputWriter):\n \"\"\"Deletes the stack.\"\"\"\n raise NotImplementedError\n\n def exec(self, command: str, tty: bool = True) -> int:\n \"\"\"Executes a command on the host OS.\"\"\"\n return subprocess.call(command, shell=True)\n\n @abstractmethod\n def clean(self, output: AbstractOutputWriter):\n \"\"\"Deletes the stack.\"\"\"\n raise NotImplementedError\n\n @abstractmethod\n def sync(self, output: AbstractOutputWriter, dry_run=False):\n \"\"\"Synchronizes the project code with the instance.\"\"\"\n raise NotImplementedError\n\n @abstractmethod\n def download(self, download_filters: list, output: AbstractOutputWriter, dry_run=False):\n \"\"\"Downloads files from the instance.\"\"\"\n raise NotImplementedError\n\n @abstractmethod\n def get_status_text(self) -> str:\n \"\"\"Returns information about the started instance.\n\n It will be shown to the user once the instance is started and by using the \"status\" command.\n \"\"\"\n raise NotImplementedError\n\n @property\n def use_tmux(self) -> bool:\n \"\"\"Use tmux when running a custom script or connecting to the instance.\"\"\"\n return False\n" }, { "path": "spotty/deployment/abstract_ssh_instance_manager.py", "content": "import logging\nimport os\nfrom abc import abstractmethod\nfrom spotty.deployment.utils.commands import get_ssh_command\nfrom spotty.deployment.abstract_docker_instance_manager import AbstractDockerInstanceManager\n\n\nclass AbstractSshInstanceManager(AbstractDockerInstanceManager):\n\n def exec(self, command: str, tty: bool = True) -> int:\n \"\"\"Executes a command on the host OS.\"\"\"\n if not os.path.isfile(self.ssh_key_path):\n raise ValueError('SSH key doesn\\'t exist: ' + self.ssh_key_path)\n\n ssh_command = get_ssh_command(self.ssh_host, self.ssh_port, self.ssh_user, self.ssh_key_path,\n command, env_vars=self.ssh_env_vars, tty=tty)\n logging.debug('SSH command: ' + ssh_command)\n\n return super().exec(ssh_command)\n\n @property\n @abstractmethod\n def ssh_host(self):\n raise NotImplementedError\n\n @property\n @abstractmethod\n def ssh_port(self) -> int:\n raise NotImplementedError\n\n @property\n @abstractmethod\n def ssh_key_path(self) -> str:\n raise NotImplementedError\n\n @property\n def ssh_user(self) -> str:\n return self.instance_config.user\n\n @property\n def ssh_env_vars(self) -> dict:\n \"\"\"Environmental variables that will be set when ssh to the instance.\"\"\"\n return {\n 'SPOTTY_CONTAINER_NAME': self.instance_config.full_container_name,\n 'SPOTTY_CONTAINER_WORKING_DIR': self.instance_config.container_config.working_dir,\n }\n\n @property\n def use_tmux(self) -> bool:\n \"\"\"Use tmux when running a custom script or connecting to the instance.\"\"\"\n return True\n" }, { "path": "spotty/deployment/container/__init__.py", "content": "" }, { "path": "spotty/deployment/container/abstract_container_commands.py", "content": "from abc import ABC, abstractmethod\nfrom spotty.config.abstract_instance_config import AbstractInstanceConfig\n\n\nclass AbstractContainerCommands(ABC):\n\n def __init__(self, instance_config: AbstractInstanceConfig):\n self._instance_config = instance_config\n\n @property\n def instance_config(self) -> AbstractInstanceConfig:\n return self._instance_config\n\n @abstractmethod\n def exec(self, command: str, interactive: bool = False, tty: bool = False, user: str = None,\n container_name: str = None, working_dir: str = None) -> str:\n raise NotImplementedError\n" }, { "path": "spotty/deployment/container/abstract_container_script.py", "content": "from abc import ABC, abstractmethod\nfrom spotty.deployment.container.abstract_container_commands import AbstractContainerCommands\n\n\nclass AbstractContainerScript(ABC):\n\n def __init__(self, container_commands: AbstractContainerCommands):\n self._commands = container_commands\n\n @property\n def commands(self) -> AbstractContainerCommands:\n return self._commands\n\n @abstractmethod\n def render(self) -> str:\n raise NotImplementedError\n" }, { "path": "spotty/deployment/container/docker/__init__.py", "content": "" }, { "path": "spotty/deployment/container/docker/docker_commands.py", "content": "import shlex\nfrom spotty.deployment.container.abstract_container_commands import AbstractContainerCommands\nfrom spotty.deployment.utils.cli import shlex_join\n\n\nclass DockerCommands(AbstractContainerCommands):\n\n def build(self, image_name: str) -> str:\n if not self._instance_config.dockerfile_path:\n raise ValueError('Cannot generate the \"build\" command as Dockerfile path is not specified')\n\n if not self._instance_config.docker_context_path:\n raise ValueError('Cannot generate the \"build\" command as Docker context path is not set')\n\n build_cmd = 'docker build -t %s -f %s %s' % (image_name, shlex.quote(self._instance_config.dockerfile_path),\n shlex.quote(self._instance_config.docker_context_path))\n\n if self._instance_config.container_config.run_as_host_user:\n build_cmd += ' --build-arg USER_ID=$(id -u %s) --build-arg GROUP_ID=$(id -g %s)' \\\n % (self._instance_config.user, self._instance_config.user)\n\n return build_cmd\n\n def pull(self) -> str:\n return 'docker pull ' + self._instance_config.container_config.image\n\n def run(self, image_name: str = None) -> str:\n image_name = image_name if image_name else self._instance_config.container_config.image\n\n # prepare \"docker run\" arguments\n args = ['-td'] + self._instance_config.container_config.runtime_parameters\n\n if self._instance_config.container_config.host_network:\n args += ['--net=host']\n\n for port in self._instance_config.container_config.ports:\n host_port = port['hostPort']\n container_port = port['containerPort']\n args += ['-p', ('%d:%d' % (host_port, container_port)) if host_port else str(container_port)]\n\n for volume_mount in self._instance_config.volume_mounts:\n args += ['-v', '%s:%s:%s' % (volume_mount.host_path, volume_mount.mount_path, volume_mount.mode)]\n\n for env_name, env_value in self._instance_config.container_config.env.items():\n args += ['-e', '%s=%s' % (env_name, env_value)]\n\n args += ['--name', self._instance_config.full_container_name]\n\n run_cmd = 'docker run $(nvidia-smi &> /dev/null && echo \"--gpus all\")'\n\n if self._instance_config.container_config.run_as_host_user:\n run_cmd += ' -u $(id -u %s):$(id -g %s) -e HOST_USER_ID=$(id -u %s) -e HOST_GROUP_ID=$(id -g %s)' \\\n % tuple([self._instance_config.user] * 4)\n\n run_cmd += ' %s %s /bin/sh > /dev/null' % (shlex_join(args), image_name)\n\n return run_cmd\n\n def is_created(self, container_name: str = None, is_running: bool = False) -> str:\n container_name = container_name if container_name else self._instance_config.full_container_name\n show_all = '' if is_running else 'a'\n\n test_cmd = '[ $(docker ps -q%s --filter name=\"%s\" | wc -c) -ne 0 ]' % (show_all, container_name)\n\n return test_cmd\n\n def remove(self):\n return 'docker rm -f \"%s\" > /dev/null' % self._instance_config.full_container_name\n\n def exec(self, command: str, interactive: bool = False, tty: bool = False, user: str = None,\n container_name: str = None, working_dir: str = None) -> str:\n container_name = container_name if container_name else self._instance_config.full_container_name\n working_dir = working_dir if working_dir else self._instance_config.container_config.working_dir\n\n exec_cmd = 'docker exec'\n\n if interactive:\n exec_cmd += ' -i'\n\n if tty:\n exec_cmd += ' -t'\n\n if user:\n exec_cmd += ' -u ' + shlex.quote(user)\n\n if working_dir:\n # no quoting, it can be environmental variable\n exec_cmd += ' -w ' + working_dir\n\n exec_cmd += ' %s %s' % (container_name, command)\n\n # run \"exec\" command only if the container is running\n test_cmd = self.is_created(container_name, is_running=True)\n error_msg = 'Container is not running.\\\\nUse the \"spotty start -C\" command to start it.\\\\n'\n cond_exec_cmd = 'if %s; then %s; else printf %s; exit 1; fi' % (test_cmd, exec_cmd, shlex.quote(error_msg))\n\n return cond_exec_cmd\n" }, { "path": "spotty/deployment/container/docker/scripts/__init__.py", "content": "" }, { "path": "spotty/deployment/container/docker/scripts/abstract_docker_script.py", "content": "from abc import ABC\nfrom spotty.deployment.container.docker.docker_commands import DockerCommands\nfrom spotty.deployment.container.abstract_container_script import AbstractContainerScript\n\n\nclass AbstractDockerScript(AbstractContainerScript, ABC):\n\n @property\n def commands(self) -> DockerCommands:\n return self._commands\n" }, { "path": "spotty/deployment/container/docker/scripts/container_bash_script.py", "content": "import os\nimport chevron\nfrom spotty.deployment.utils.commands import get_bash_command\nfrom spotty.deployment.container.docker.scripts.abstract_docker_script import AbstractDockerScript\n\n\nclass ContainerBashScript(AbstractDockerScript):\n\n def render(self) -> str:\n # read template file\n template_path = os.path.join(os.path.dirname(__file__), 'data', 'container_bash.sh.tpl')\n with open(template_path) as f:\n template = f.read()\n\n # render the script\n content = chevron.render(template, data={\n 'docker_exec_bash': self.commands.exec(get_bash_command(), interactive=True, tty=True,\n container_name='$SPOTTY_CONTAINER_NAME',\n working_dir='$SPOTTY_CONTAINER_WORKING_DIR'),\n })\n\n return content\n" }, { "path": "spotty/deployment/container/docker/scripts/data/container_bash.sh.tpl", "content": "#!/bin/bash -e\n\nif [ -z \"$SPOTTY_CONTAINER_NAME\" ]; then\n echo -e \"\\nSPOTTY_CONTAINER_NAME environmental variable is not set.\\n\"\n exit 1\nfi\n\nSPOTTY_CONTAINER_WORKING_DIR=${SPOTTY_CONTAINER_WORKING_DIR:-/}\n\n{{{docker_exec_bash}}}\n" }, { "path": "spotty/deployment/container/docker/scripts/data/start_container.sh.tpl", "content": "#!/usr/bin/env bash\n\n{{bash_flags}}\n\nif {{{is_created_cmd}}}; then\n printf 'Removing existing container... '\n {{{remove_cmd}}}\n echo 'DONE'\nfi\n\n{{> before_image_build}}\n\n{{#build_image_cmd}}\necho 'Building Docker image...'\n{{{build_image_cmd}}}\n{{/build_image_cmd}}\n\n{{> before_container_run}}\n\n{{#pull_image_cmd}}\ni=0\nuntil [ \"$i\" -ge 3 ]\ndo\n if [ \"$i\" -ne 0 ]; then\n echo \"Retrying to pull the image $i...\"\n fi\n\n PULL_EXIT_CODE=0\n {{{pull_image_cmd}}} || PULL_EXIT_CODE=$?\n\n if [ \"$PULL_EXIT_CODE\" -ne 125 ]; then\n break\n fi\n\n i=$((i+1))\n sleep 10\ndone\n\nif [ \"$PULL_EXIT_CODE\" -ne 0 ]; then\n exit $PULL_EXIT_CODE\nfi\n{{/pull_image_cmd}}\n\nprintf 'Starting container... '\n{{{start_container_cmd}}}\necho 'DONE'\n\n{{> before_startup_commands}}\n\n{{#docker_exec_startup_script_cmd}}\necho 'Running startup commands...'\n{{{docker_exec_startup_script_cmd}}}\n{{/docker_exec_startup_script_cmd}}\n" }, { "path": "spotty/deployment/container/docker/scripts/data/stop_container.sh.tpl", "content": "#!/usr/bin/env bash\n\nset -e\n\nif {{{is_created_cmd}}}; then\n printf 'Removing the container... '\n {{{remove_cmd}}}\n echo 'DONE'\nelse\n echo 'Container is not running.'\nfi\n" }, { "path": "spotty/deployment/container/docker/scripts/start_container_script.py", "content": "import os\nimport time\nimport chevron\nfrom spotty.deployment.utils.commands import get_script_command\nfrom spotty.deployment.container.docker.scripts.abstract_docker_script import AbstractDockerScript\n\n\nclass StartContainerScript(AbstractDockerScript):\n\n def _partials(self) -> dict:\n return {\n 'before_image_build': '',\n 'before_container_run': '',\n 'before_startup_commands': '',\n }\n\n def render(self, print_trace: bool = False) -> str:\n # read template file\n template_path = os.path.join(os.path.dirname(__file__), 'data', 'start_container.sh.tpl')\n with open(template_path) as f:\n template = f.read()\n\n # generate \"docker build\" command if necessary\n if self.commands.instance_config.dockerfile_path:\n image_name = '%s:%d' % (self.commands.instance_config.full_container_name, time.time())\n build_image_cmd = self.commands.build(image_name)\n pull_image_cmd = ''\n else:\n image_name = self.commands.instance_config.container_config.image\n build_image_cmd = ''\n pull_image_cmd = self.commands.pull()\n\n # generate a command to run the startup script\n exec_script_cmd = ''\n if self.commands.instance_config.container_config.commands:\n startup_script_cmd = get_script_command('container-startup-commands',\n self.commands.instance_config.container_config.commands)\n exec_script_cmd = self.commands.exec(startup_script_cmd, user='root')\n\n # render the script\n content = chevron.render(template, data={\n 'bash_flags': 'set -xe' if print_trace else 'set -e',\n 'is_created_cmd': self.commands.is_created(),\n 'remove_cmd': self.commands.remove(),\n 'build_image_cmd': build_image_cmd,\n 'pull_image_cmd': pull_image_cmd,\n 'tmp_container_dir': self.commands.instance_config.host_container_dir,\n 'start_container_cmd': self.commands.run(image_name),\n 'docker_exec_startup_script_cmd': exec_script_cmd,\n }, partials_dict=self._partials())\n\n return content\n" }, { "path": "spotty/deployment/container/docker/scripts/stop_container_script.py", "content": "import os\nimport chevron\nfrom spotty.deployment.container.docker.scripts.abstract_docker_script import AbstractDockerScript\n\n\nclass StopContainerScript(AbstractDockerScript):\n\n def render(self) -> str:\n # read template file\n template_path = os.path.join(os.path.dirname(__file__), 'data', 'stop_container.sh.tpl')\n with open(template_path) as f:\n template = f.read()\n\n # render the script\n content = chevron.render(template, data={\n 'is_created_cmd': self.commands.is_created(),\n 'remove_cmd': self.commands.remove(),\n })\n\n return content\n" }, { "path": "spotty/deployment/utils/__init__.py", "content": "" }, { "path": "spotty/deployment/utils/cli.py", "content": "import shlex\n\n\ndef shlex_join(split_command: list):\n \"\"\"Return a shell-escaped string from *split_command*.\n Copy-pasted from the Python 3.8 code.\n \"\"\"\n return ' '.join(shlex.quote(arg) for arg in split_command)\n" }, { "path": "spotty/deployment/utils/commands.py", "content": "import base64\nimport os\nimport shlex\nimport time\nfrom spotty.deployment.utils.cli import shlex_join\n\n\ndef get_bash_command() -> str:\n return '/usr/bin/env bash'\n\n\ndef get_script_command(script_name: str, script_content: str, script_args: list = None,\n logging: bool = False) -> str:\n \"\"\"Encodes a multi-line script into base64 and returns a one-line command\n that unpacks the script to a temporary file and runs it.\"\"\"\n\n # encode the script content to base64\n script_base64 = base64.b64encode(script_content.encode('utf-8')).decode('utf-8')\n\n # command to decode the script, save it to a temporary file and run inside the container\n script_args = shlex_join(script_args) if script_args else ''\n\n script_cmd = ' && '.join([\n 'TMPDIR=${TMPDIR%/}',\n 'TMP_SCRIPT_PATH=$(mktemp ${TMPDIR:-/tmp}/spotty-%s.XXXXXXXX)' % script_name,\n 'chmod +x $TMP_SCRIPT_PATH',\n 'echo %s | base64 -d > $TMP_SCRIPT_PATH' % script_base64,\n '$TMP_SCRIPT_PATH ' + script_args,\n ])\n\n # log the command output to a file\n if logging:\n log_file_path = '/var/log/spotty/run/%s-%d.log' % (script_name, time.time())\n script_cmd = get_log_command(script_cmd, log_file_path)\n\n # execute the command with bash\n script_cmd = '%s -c %s' % (get_bash_command(), shlex.quote(script_cmd))\n\n return script_cmd\n\n\ndef get_log_command(command: str, log_file_path: str) -> str:\n # log the command outputs to a file on the host OS\n log_dir = os.path.dirname(log_file_path)\n log_cmd = '; '.join([\n 'set -o pipefail',\n ' && '.join([\n 'mkdir -pm 777 ' + shlex.quote(log_dir),\n '(%s) 2>&1 | tee %s' % (command, shlex.quote(log_file_path)),\n ]),\n ])\n\n return log_cmd\n\n\ndef get_tmux_session_command(command: str, session_name: str, window_name: str = None, default_command: str = None,\n keep_pane: bool = False) -> str:\n session_cmd = 'tmux new -A -s ' + session_name\n if window_name:\n session_cmd += ' -n ' + window_name\n\n if command:\n # keep the pane alive when the script is finished\n keep_pane_cmd = 'tmux set -w remain-on-exit on; ' if keep_pane else ''\n\n # set the default command (to automatically run bash inside the container when a new window is created)\n default_command_cmd = ('tmux set default-command %s; ' % shlex.quote(default_command)) \\\n if default_command else ''\n\n # keep the pane alive if the script is failed\n tmux_cmd = '%s%s(%s) || tmux set -w remain-on-exit on' % (keep_pane_cmd, default_command_cmd, command)\n\n # run the command inside the tmux session\n session_cmd += ' ' + shlex.quote(tmux_cmd)\n\n # use tmux only if it's installed\n session_cmd = 'if command -v tmux &> /dev/null; then %s; else %s; fi' % (session_cmd, command)\n\n return session_cmd\n\n\ndef get_ssh_command(host: str, port: int, user: str, key_path: str, command: str, env_vars: dict = None,\n tty: bool = True, quiet: bool = False) -> str:\n\n ssh_command = 'ssh -i %s -o StrictHostKeyChecking=no -o ConnectTimeout=10' % shlex.quote(key_path)\n\n if tty:\n ssh_command += ' -t'\n\n if port != 22:\n ssh_command += ' -p %d' % port\n\n if quiet:\n ssh_command += ' -q'\n\n # export environmental variables\n if env_vars:\n export_cmd = '; '.join(['export %s=%s' % (name, shlex.quote(val)) for name, val in env_vars.items()])\n command = '%s; %s' % (export_cmd, command)\n\n # final SSH command\n ssh_command += ' %s@%s %s' % (user, host, shlex.quote(command))\n\n return ssh_command\n" }, { "path": "spotty/deployment/utils/print_info.py", "content": "from typing import List\nfrom spotty.config.abstract_instance_config import VolumeMount\nfrom spotty.config.abstract_instance_volume import AbstractInstanceVolume\nfrom spotty.config.container_config import PROJECT_VOLUME_MOUNT_NAME\nfrom spotty.utils import render_table\n\n\ndef render_volumes_info_table(volume_mounts: List[VolumeMount], volumes: List[AbstractInstanceVolume]):\n table = [('Name', 'Mount Path', 'Type', 'Deletion Policy')]\n\n # add volume mounts to the info table\n volumes_dict = {volume.name: volume for volume in volumes}\n for volume_mount in volume_mounts:\n if not volume_mount.hidden:\n # the volume will be mounted to the container\n volume = volumes_dict[volume_mount.name]\n vol_mount_name = '-' if volume_mount.name == PROJECT_VOLUME_MOUNT_NAME else volume_mount.name\n deletion_policy = volume.deletion_policy_title if volume.deletion_policy_title else '-'\n table.append((vol_mount_name, volume_mount.mount_path, volume.title, deletion_policy))\n\n # add volumes that were not mounted to the container to the info table\n volume_mounts_dict = {volume_mount.name for volume_mount in volume_mounts}\n for volume in volumes:\n if volume.name not in volume_mounts_dict:\n deletion_policy = volume.deletion_policy_title if volume.deletion_policy_title else '-'\n table.append((volume.name, '-', volume.title, deletion_policy))\n\n return render_table(table, separate_title=True)\n" }, { "path": "spotty/deployment/utils/user_scripts.py", "content": "import re\nimport chevron\nfrom spotty.deployment.utils.commands import get_bash_command\n\n\ndef parse_script_parameters(script_params: str):\n \"\"\"Parses script parameters.\"\"\"\n params = {}\n for param in script_params:\n match = re.match('(\\\\w+)=(.*)', param)\n if not match:\n raise ValueError('Invalid format for the script parameter: \"%s\" (the \"PARAMETER=VALUE\" format is expected).'\n % param)\n\n param_name, param_value = match.groups()\n if param_name in params:\n raise ValueError('Parameter \"%s\" was defined twice.' % param_name)\n\n params[param_name] = param_value\n\n return params\n\n\ndef render_script(template: str, params: dict):\n \"\"\"Renders a script template.\n\n It based on the Mustache templates, but only\n variables and delimiter changes are allowed.\n\n Raises an exception if one of the provided parameters doesn't\n exist in the template.\n \"\"\"\n tokens = list(chevron.tokenizer.tokenize(template))\n template_keys = set()\n for tag, key in tokens:\n if tag not in ['literal', 'no escape', 'variable', 'set delimiter']:\n raise ValueError('Script templates support only variables and delimiter changes.')\n\n template_keys.add(key)\n\n # check that the script contains keys for all provided parameters\n for key in params:\n if key not in template_keys:\n raise ValueError('Parameter \"%s\" doesn\\'t exist in the script.' % key)\n\n content = chevron.render(tokens, params)\n\n if content[:2] != '#!':\n content = ('#!%s\\n\\nset -xe\\n\\n' % get_bash_command()) + content\n\n return content\n" }, { "path": "spotty/errors/__init__.py", "content": "" }, { "path": "spotty/errors/instance_not_running.py", "content": "class InstanceNotRunningError(Exception):\n def __init__(self, instance_name: str):\n super().__init__('Instance \"%s\" is not running.\\n'\n 'Use the \"spotty start %s\" command to start the instance.'\n % (instance_name, instance_name))\n" }, { "path": "spotty/errors/nothing_to_do.py", "content": "class NothingToDoError(Exception):\n pass\n" }, { "path": "spotty/providers/__init__.py", "content": "" }, { "path": "spotty/providers/aws/__init__.py", "content": "" }, { "path": "spotty/providers/aws/cfn_templates/__init__.py", "content": "" }, { "path": "spotty/providers/aws/cfn_templates/instance/__init__.py", "content": "" }, { "path": "spotty/providers/aws/cfn_templates/instance/data/files/tmux.conf", "content": "bind-key x kill-pane\n" }, { "path": "spotty/providers/aws/cfn_templates/instance/data/startup_scripts/01_prepare_instance.sh", "content": "#!/bin/bash -xe\n\ncfn-signal -e 0 --stack ${AWS::StackName} --region ${AWS::Region} --resource PreparingInstanceSignal\n\n# install AWS CLI\nupdate-locale LANG=en_US.UTF-8 LC_ALL=en_US.UTF-8\napt-get update && apt-get install -y python3-pip\npip3 install -U awscli\naws configure set default.region ${AWS::Region}\n\n# install jq\napt-get install -y jq\n\n# create an alias to connect to the docker container\nCONTAINER_BASH_ALIAS=container\necho \"alias $CONTAINER_BASH_ALIAS=\\\"{{CONTAINER_BASH_SCRIPT_PATH}}\\\"\" >> /home/ubuntu/.bashrc\necho \"alias $CONTAINER_BASH_ALIAS=\\\"{{CONTAINER_BASH_SCRIPT_PATH}}\\\"\" >> /root/.bashrc\n\n# create common temporary directories\nmkdir -pm 777 '{{SPOTTY_TMP_DIR}}'\nmkdir -pm 777 '{{CONTAINERS_TMP_DIR}}'\n" }, { "path": "spotty/providers/aws/cfn_templates/instance/data/startup_scripts/02_mount_volumes.sh", "content": "#!/bin/bash -xe\n\ncfn-signal -e 0 --stack ${AWS::StackName} --region ${AWS::Region} --resource MountingVolumesSignal\n\n# mount volumes\nDEVICE_LETTERS=(f g h i j k l m n o p)\nMOUNT_DIRS=({{{MOUNT_DIRS}}})\n\nfor i in ${!!MOUNT_DIRS[*]}\ndo\n MOUNT_DIR=${!MOUNT_DIRS[$i]}\n DEVICE=/dev/xvd${!DEVICE_LETTERS[$i]}\n\n # NVMe EBS volume (see: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/nvme-ebs-volumes.html)\n if [ ! -b $DEVICE ]; then\n VOLUME_ID=$(cfn-get-metadata --stack ${AWS::StackName} --region ${AWS::Region} --resource VolumeAttachment${!DEVICE_LETTERS[$i]^} -k VolumeId)\n DEVICE=$(lsblk -o NAME,SERIAL -dpJ | jq -rc \".blockdevices[] | select(.serial == \\\"${!VOLUME_ID//-}\\\") | .name\")\n if [ -z \"$DEVICE\" ]; then\n echo \"Device for the volume $VOLUME_ID not found\"\n exit 1\n fi\n fi\n\n blkid -o value -s TYPE $DEVICE || mkfs -t ext4 $DEVICE\n mkdir -p $MOUNT_DIR\n mount $DEVICE $MOUNT_DIR\n chmod 777 $MOUNT_DIR\n resize2fs $DEVICE\ndone\n\n# create directories for temporary container volumes\n{{#TMP_VOLUME_DIRS}}\nmkdir -p {{PATH}}\nchmod 777 {{PATH}}\n{{/TMP_VOLUME_DIRS}}\n" }, { "path": "spotty/providers/aws/cfn_templates/instance/data/startup_scripts/03_set_docker_root.sh", "content": "#!/bin/bash -xe\n\ncfn-signal -e 0 --stack ${AWS::StackName} --region ${AWS::Region} --resource SettingDockerRootSignal\n\n# change docker data root directory\nif [ -n \"${DockerDataRootDirectory}\" ]; then\n jq '. + { \"data-root\": \"${DockerDataRootDirectory}\" }' /etc/docker/daemon.json > /tmp/docker_daemon.json \\\n && mv /tmp/docker_daemon.json /etc/docker/daemon.json\n service docker restart\nfi\n" }, { "path": "spotty/providers/aws/cfn_templates/instance/data/startup_scripts/04_sync_project.sh", "content": "#!/bin/bash -xe\n\ncfn-signal -e 0 --stack ${AWS::StackName} --region ${AWS::Region} --resource SyncingProjectSignal\n\n# create a project directory\nif [ -n \"${HostProjectDirectory}\" ]; then\n mkdir -p 777 ${HostProjectDirectory}\n chmod 777 ${HostProjectDirectory}\n\n if [ -d '${HostProjectDirectory}/lost+found' ]; then\n chmod 777 '${HostProjectDirectory}/lost+found'\n fi\nfi\n\n# sync project files from S3 bucket to the instance\n{{{SYNC_PROJECT_CMD}}}\n" }, { "path": "spotty/providers/aws/cfn_templates/instance/data/startup_scripts/05_run_instance_startup_commands.sh", "content": "#!/bin/bash -xe\n\ncfn-signal -e 0 --stack ${AWS::StackName} --region ${AWS::Region} --resource RunningInstanceStartupCommandsSignal\n\n/bin/bash -xe {{INSTANCE_STARTUP_SCRIPTS_DIR}}/instance_startup_commands.sh\n" }, { "path": "spotty/providers/aws/cfn_templates/instance/data/startup_scripts/user_data.sh", "content": "#!/bin/bash -x\n\ncd /root || exit 1\n\n# install CloudFormation tools if they are not installed yet\nif [ ! -e /usr/local/bin/cfn-init ]; then\n apt-get update\n apt-get install -y python-setuptools\n mkdir -p aws-cfn-bootstrap-latest\n curl https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz | tar xz -C aws-cfn-bootstrap-latest --strip-components 1\n python2 -m easy_install aws-cfn-bootstrap-latest\nfi\n\n# prepare the instance and run Docker container\ncfn-init \\\n --stack ${AWS::StackName} \\\n --region ${AWS::Region} \\\n --resource InstanceLaunchTemplate \\\n -c init \\\n -v\n\nSTACK_CREATED=$?\n\n# uplooad cfn-init logs to the bucket\nif [ $STACK_CREATED -ne 0 ]; then\n STACK_ID=${AWS::StackId}\n STACK_UUID=${!STACK_ID##*/}\n\n aws s3 cp /var/log/cfn-init-cmd.log ${LogsS3Path}/$STACK_UUID/cfn-init-cmd.log\nfi\n\n# send signal that the Docker container is ready or failed\ncfn-signal \\\n -e $STACK_CREATED \\\n --stack ${AWS::StackName} \\\n --region ${AWS::Region} \\\n --resource DockerReadyWaitCondition\n" }, { "path": "spotty/providers/aws/cfn_templates/instance/data/template.yaml", "content": "Description: Spotty EC2 Instance\nParameters:\n VpcId:\n Description: VPC ID\n Type: AWS::EC2::VPC::Id\n InstanceProfileArn:\n Description: Instance Profile ARN\n Type: String\n InstanceType:\n Description: Instance type\n Type: String\n KeyName:\n Description: EC2 Key Pair name\n Type: AWS::EC2::KeyPair::KeyName\n ImageId:\n Description: AMI ID\n Type: AWS::EC2::Image::Id\n RootVolumeSize:\n Description: Root volume size\n Type: String\n DockerDataRootDirectory:\n Description: Docker data root directory\n Type: String\n Default: ''\n InstanceNameTag:\n Description: Name for the instance\n Type: String\n Default: ''\n HostProjectDirectory:\n Description: Destination directory for the project\n Type: String\n Default: ''\n LogsS3Path:\n Description: An S3 path where logs will be uploaded in the case of a failure\n Type: String\n Default: ''\nResources:\n Instance:\n Type: AWS::EC2::Instance\n Properties:\n LaunchTemplate:\n LaunchTemplateId: !Ref InstanceLaunchTemplate\n Version: !GetAtt InstanceLaunchTemplate.LatestVersionNumber\n\n InstanceLaunchTemplate:\n Type: AWS::EC2::LaunchTemplate\n Properties:\n LaunchTemplateData:\n InstanceType: !Ref InstanceType\n ImageId: !Ref ImageId\n KeyName: !Ref KeyName\n EbsOptimized: 'false'\n TagSpecifications:\n - ResourceType: instance\n Tags:\n - Key: Name\n Value: !Ref InstanceNameTag\n IamInstanceProfile:\n Arn: !Ref InstanceProfileArn\n SecurityGroupIds:\n - !Ref InstanceSecurityGroup\n InstanceInitiatedShutdownBehavior: terminate\n InstanceMarketOptions:\n MarketType: spot\n SpotOptions:\n SpotInstanceType: one-time\n InstanceInterruptionBehavior: terminate\n BlockDeviceMappings:\n - DeviceName: /dev/sda1\n Ebs:\n VolumeSize: !Ref RootVolumeSize\n DeleteOnTermination: true\n UserData: ''\n Metadata:\n 'AWS::CloudFormation::Init': {}\n\n InstanceSecurityGroup:\n Type: AWS::EC2::SecurityGroup\n Properties:\n VpcId: !Ref VpcId\n GroupDescription: Spotty security group\n SecurityGroupEgress:\n - CidrIp: 0.0.0.0/0\n IpProtocol: -1\n FromPort: 0\n ToPort: 65535\n - CidrIpv6: ::/0\n IpProtocol: -1\n FromPort: 0\n ToPort: 65535\n SecurityGroupIngress:\n - CidrIp: 0.0.0.0/0\n IpProtocol: tcp\n FromPort: 22\n ToPort: 22\n - CidrIpv6: ::/0\n IpProtocol: tcp\n FromPort: 22\n ToPort: 22\n\n PreparingInstanceSignal:\n Type: AWS::CloudFormation::WaitCondition\n DependsOn: Instance\n CreationPolicy:\n ResourceSignal:\n Timeout: PT30M\n\n MountingVolumesSignal:\n Type: AWS::CloudFormation::WaitCondition\n DependsOn: Instance\n CreationPolicy:\n ResourceSignal:\n Timeout: PT30M\n\n SettingDockerRootSignal:\n Type: AWS::CloudFormation::WaitCondition\n DependsOn: Instance\n CreationPolicy:\n ResourceSignal:\n Timeout: PT60M\n\n SyncingProjectSignal:\n Type: AWS::CloudFormation::WaitCondition\n DependsOn: Instance\n CreationPolicy:\n ResourceSignal:\n Timeout: PT60M\n\n RunningInstanceStartupCommandsSignal:\n Type: AWS::CloudFormation::WaitCondition\n DependsOn: Instance\n CreationPolicy:\n ResourceSignal:\n Timeout: PT60M\n\n BuildingDockerImageSignal:\n Type: AWS::CloudFormation::WaitCondition\n DependsOn: Instance\n CreationPolicy:\n ResourceSignal:\n Timeout: PT60M\n\n StartingContainerSignal:\n Type: AWS::CloudFormation::WaitCondition\n DependsOn: Instance\n CreationPolicy:\n ResourceSignal:\n Timeout: PT60M\n\n RunningContainerStartupCommandsSignal:\n Type: AWS::CloudFormation::WaitCondition\n DependsOn: Instance\n CreationPolicy:\n ResourceSignal:\n Timeout: PT60M\n\n DockerReadyWaitCondition:\n Type: AWS::CloudFormation::WaitCondition\n DependsOn: Instance\n CreationPolicy:\n ResourceSignal:\n Timeout: PT30M\n\nOutputs:\n InstanceId:\n Value: !Ref Instance\n AvailabilityZone:\n Value: !GetAtt Instance.AvailabilityZone\n" }, { "path": "spotty/providers/aws/cfn_templates/instance/start_container_script.py", "content": "from spotty.deployment.container.docker.scripts.start_container_script import StartContainerScript\n\n\nclass StartContainerScriptWithCfnSignals(StartContainerScript):\n\n @staticmethod\n def _get_signal_command(resource_name: str):\n return 'cfn-signal -e 0 --stack $_{AWS::StackName} --region $_{AWS::Region} --resource ' + resource_name\n\n def _partials(self) -> dict:\n return {\n 'before_image_build': self._get_signal_command('BuildingDockerImageSignal'),\n 'before_container_run': self._get_signal_command('StartingContainerSignal'),\n 'before_startup_commands': self._get_signal_command('RunningContainerStartupCommandsSignal'),\n }\n\n def render(self, print_trace: bool = False) -> str:\n content = super().render(print_trace=print_trace)\n content = content.replace('${', '${!')\n content = content.replace('$_{', '${')\n\n return content\n" }, { "path": "spotty/providers/aws/cfn_templates/instance/template.py", "content": "from typing import List\nimport os\nimport chevron\nimport yaml\nfrom cfn_tools import CfnYamlLoader, CfnYamlDumper\nfrom spotty.commands.writers.abstract_output_writrer import AbstractOutputWriter\nfrom spotty.config.tmp_dir_volume import TmpDirVolume\nfrom spotty.config.validation import is_subdir\nfrom spotty.config.abstract_instance_volume import AbstractInstanceVolume\nfrom spotty.deployment.container.docker.docker_commands import DockerCommands\nfrom spotty.deployment.container.docker.scripts.container_bash_script import ContainerBashScript\nfrom spotty.deployment.abstract_cloud_instance.file_structure import INSTANCE_SPOTTY_TMP_DIR, \\\n CONTAINER_BASH_SCRIPT_PATH, \\\n INSTANCE_STARTUP_SCRIPTS_DIR, CONTAINERS_TMP_DIR\nfrom spotty.providers.aws.cfn_templates.instance.start_container_script import StartContainerScriptWithCfnSignals\nfrom spotty.providers.aws.helpers.ami import get_ami\nfrom spotty.providers.aws.helpers.vpc import get_vpc_id\nfrom spotty.providers.aws.resources.snapshot import Snapshot\nfrom spotty.providers.aws.resources.volume import Volume\nfrom spotty.providers.aws.config.instance_config import InstanceConfig\nfrom spotty.providers.aws.config.ebs_volume import EbsVolume\nfrom spotty.providers.aws.helpers.logs import get_logs_s3_path\n\n\ndef prepare_instance_template(ec2, instance_config: InstanceConfig, docker_commands: DockerCommands,\n availability_zone: str, sync_project_cmd: str, output: AbstractOutputWriter):\n \"\"\"Prepares CloudFormation template to run a Spot Instance.\"\"\"\n\n # read and update CF template\n with open(os.path.join(os.path.dirname(__file__), 'data', 'template.yaml')) as f:\n template = yaml.load(f, Loader=CfnYamlLoader)\n\n # get volume resources and updated availability zone\n volume_resources = _get_volume_resources(ec2, instance_config.volumes, output)\n\n # add volume resources to the template\n template['Resources'].update(volume_resources)\n\n # set availability zone\n if availability_zone:\n template['Resources']['InstanceLaunchTemplate']['Properties']['LaunchTemplateData']['Placement'] = {\n 'AvailabilityZone': availability_zone,\n }\n output.write('- availability zone: %s' % availability_zone)\n else:\n output.write('- availability zone: auto')\n\n # set subnet\n if instance_config.subnet_id:\n template['Resources']['InstanceLaunchTemplate']['Properties']['LaunchTemplateData']['NetworkInterfaces'] = [\n {\n 'SubnetId': instance_config.subnet_id,\n 'DeviceIndex': 0,\n 'Groups': template['Resources']['InstanceLaunchTemplate']['Properties']['LaunchTemplateData'][\n 'SecurityGroupIds'],\n }]\n del template['Resources']['InstanceLaunchTemplate']['Properties']['LaunchTemplateData']['SecurityGroupIds']\n\n # add ports to the security group\n for port in instance_config.ports:\n if port != 22:\n template['Resources']['InstanceSecurityGroup']['Properties']['SecurityGroupIngress'] += [{\n 'CidrIp': '0.0.0.0/0',\n 'IpProtocol': 'tcp',\n 'FromPort': port,\n 'ToPort': port,\n }, {\n 'CidrIpv6': '::/0',\n 'IpProtocol': 'tcp',\n 'FromPort': port,\n 'ToPort': port,\n }]\n\n if instance_config.is_spot_instance:\n # set maximum price\n if instance_config.max_price:\n template['Resources']['InstanceLaunchTemplate']['Properties']['LaunchTemplateData'] \\\n ['InstanceMarketOptions']['SpotOptions']['MaxPrice'] = instance_config.max_price\n\n output.write('- maximum Spot Instance price: %s'\n % (('%.04f' % instance_config.max_price) if instance_config.max_price else 'on-demand'))\n else:\n # run on-demand instance\n del template['Resources']['InstanceLaunchTemplate']['Properties']['LaunchTemplateData'][\n 'InstanceMarketOptions']\n output.write('- on-demand instance')\n\n # set the user data script\n template['Resources']['InstanceLaunchTemplate']['Properties']['LaunchTemplateData']['UserData'] = {\n 'Fn::Base64': {\n 'Fn::Sub': _read_template_file(os.path.join('startup_scripts', 'user_data.sh')),\n },\n }\n\n # run sync command as a non-root user\n if instance_config.container_config.run_as_host_user:\n sync_project_cmd = 'sudo -u %s %s' % (instance_config.user, sync_project_cmd)\n\n # get mount directories\n mount_dirs = [volume.mount_dir for volume in instance_config.volumes if isinstance(volume, EbsVolume)]\n\n # set CloudFormation configs\n cfn_init_configs = [\n {\n 'name': 'prepare_instance',\n 'files': {\n INSTANCE_STARTUP_SCRIPTS_DIR + '/01_prepare_instance.sh': {\n 'owner': 'ubuntu',\n 'group': 'ubuntu',\n 'mode': '000755',\n 'content': {\n 'Fn::Sub': _read_template_file(os.path.join('startup_scripts', '01_prepare_instance.sh'), {\n 'CONTAINER_BASH_SCRIPT_PATH': CONTAINER_BASH_SCRIPT_PATH,\n 'SPOTTY_TMP_DIR': INSTANCE_SPOTTY_TMP_DIR,\n 'CONTAINERS_TMP_DIR': CONTAINERS_TMP_DIR,\n }),\n },\n },\n CONTAINER_BASH_SCRIPT_PATH: {\n 'owner': 'ubuntu',\n 'group': 'ubuntu',\n 'mode': '000755',\n 'content': ContainerBashScript(docker_commands).render(),\n },\n '/home/ubuntu/.tmux.conf': {\n 'owner': 'ubuntu',\n 'group': 'ubuntu',\n 'mode': '000644',\n 'content': {\n 'Fn::Sub': _read_template_file(os.path.join('files', 'tmux.conf')),\n },\n },\n },\n 'command': INSTANCE_STARTUP_SCRIPTS_DIR + '/01_prepare_instance.sh',\n },\n {\n 'name': 'mount_volumes',\n 'files': {\n INSTANCE_STARTUP_SCRIPTS_DIR + '/02_mount_volumes.sh': {\n 'owner': 'ubuntu',\n 'group': 'ubuntu',\n 'mode': '000755',\n 'content': {\n 'Fn::Sub': _read_template_file(os.path.join('startup_scripts', '02_mount_volumes.sh'), {\n 'MOUNT_DIRS': ('\"%s\"' % '\" \"'.join(mount_dirs)) if mount_dirs else '',\n 'TMP_VOLUME_DIRS': [{'PATH': volume.host_path} for volume in instance_config.volumes\n if isinstance(volume, TmpDirVolume)],\n }),\n },\n },\n },\n 'command': INSTANCE_STARTUP_SCRIPTS_DIR + '/02_mount_volumes.sh',\n },\n {\n 'name': 'set_docker_root',\n 'files': {\n INSTANCE_STARTUP_SCRIPTS_DIR + '/03_set_docker_root.sh': {\n 'owner': 'ubuntu',\n 'group': 'ubuntu',\n 'mode': '000755',\n 'content': {\n 'Fn::Sub': _read_template_file(os.path.join('startup_scripts', '03_set_docker_root.sh')),\n },\n },\n },\n 'command': INSTANCE_STARTUP_SCRIPTS_DIR + '/03_set_docker_root.sh',\n },\n {\n 'name': 'sync_project',\n 'files': {\n INSTANCE_STARTUP_SCRIPTS_DIR + '/04_sync_project.sh': {\n 'owner': 'ubuntu',\n 'group': 'ubuntu',\n 'mode': '000755',\n 'content': {\n 'Fn::Sub': _read_template_file(os.path.join('startup_scripts', '04_sync_project.sh'), {\n 'SYNC_PROJECT_CMD': sync_project_cmd,\n }),\n },\n },\n },\n 'command': INSTANCE_STARTUP_SCRIPTS_DIR + '/04_sync_project.sh',\n },\n {\n 'name': 'run_instance_startup_commands',\n 'files': {\n INSTANCE_STARTUP_SCRIPTS_DIR + '/05_run_instance_startup_commands.sh': {\n 'owner': 'ubuntu',\n 'group': 'ubuntu',\n 'mode': '000755',\n 'content': {\n 'Fn::Sub': _read_template_file(\n os.path.join('startup_scripts', '05_run_instance_startup_commands.sh'), {\n 'INSTANCE_STARTUP_SCRIPTS_DIR': INSTANCE_STARTUP_SCRIPTS_DIR,\n }),\n },\n },\n INSTANCE_STARTUP_SCRIPTS_DIR + '/instance_startup_commands.sh': {\n 'owner': 'ubuntu',\n 'group': 'ubuntu',\n 'mode': '000644',\n 'content': instance_config.commands or '#',\n },\n },\n 'command': INSTANCE_STARTUP_SCRIPTS_DIR + '/05_run_instance_startup_commands.sh',\n },\n {\n 'name': 'start_container',\n 'files': {\n INSTANCE_STARTUP_SCRIPTS_DIR + '/06_start_container.sh': {\n 'owner': 'ubuntu',\n 'group': 'ubuntu',\n 'mode': '000755',\n 'content': {\n 'Fn::Sub': StartContainerScriptWithCfnSignals(docker_commands).render(print_trace=True),\n },\n },\n },\n 'command': INSTANCE_STARTUP_SCRIPTS_DIR + '/06_start_container.sh',\n },\n ]\n\n template['Resources']['InstanceLaunchTemplate']['Metadata']['AWS::CloudFormation::Init']['configSets'] = {\n 'init': [config['name'] for config in cfn_init_configs],\n }\n\n for config in cfn_init_configs:\n template['Resources']['InstanceLaunchTemplate']['Metadata']['AWS::CloudFormation::Init'][config['name']] = {\n 'files': config.get('files', {}),\n 'commands': {\n config['name']: {\n 'command': config['command'],\n }\n },\n }\n\n return yaml.dump(template, Dumper=CfnYamlDumper)\n\n\ndef _read_template_file(filename: str, params: dict = None):\n with open(os.path.join(os.path.dirname(__file__), 'data', filename)) as f:\n content = f.read()\n\n if params:\n content = chevron.render(content, params)\n\n return content\n\n\ndef _get_volume_attachment_resource(volume_id, device_name):\n attachment_resource = {\n 'Type': 'AWS::EC2::VolumeAttachment',\n 'Properties': {\n 'Device': device_name,\n 'InstanceId': {'Ref': 'Instance'},\n 'VolumeId': volume_id if isinstance(volume_id, str) else dict(volume_id), # avoid YAML aliases\n },\n 'Metadata': {\n 'Device': device_name,\n 'VolumeId': volume_id if isinstance(volume_id, str) else dict(volume_id), # avoid YAML aliases\n },\n }\n\n return attachment_resource\n\n\ndef _get_volume_resource(ec2, volume: EbsVolume, output: AbstractOutputWriter):\n # new volume will be created\n volume_resource = {\n 'Type': 'AWS::EC2::Volume',\n 'DeletionPolicy': 'Retain',\n 'Properties': {\n 'AvailabilityZone': {'Fn::GetAtt': ['Instance', 'AvailabilityZone']},\n 'Tags': [{\n 'Key': 'Name',\n 'Value': volume.ec2_volume_name,\n }],\n 'VolumeType': volume.type,\n },\n }\n\n # check if the snapshot exists and restore the volume from it\n snapshot = Snapshot.get_by_name(ec2, volume.ec2_volume_name)\n if snapshot:\n # volume will be restored from the snapshot\n # check size of the volume\n if volume.size and (volume.size < snapshot.size):\n raise ValueError('Specified size for the \"%s\" volume (%dGB) is less than size of the '\n 'snapshot (%dGB).'\n % (volume.name, volume.size, snapshot.size))\n\n # set snapshot ID\n volume_resource['Properties']['SnapshotId'] = snapshot.snapshot_id\n\n output.write('- volume \"%s\" will be restored from the snapshot' % volume.ec2_volume_name)\n\n else:\n # empty volume will be created, check that the size is specified\n if not volume.size:\n raise ValueError('Size for the new volume is required.')\n\n output.write('- volume \"%s\" will be created' % volume.ec2_volume_name)\n\n # set size of the volume\n if volume.size:\n volume_resource['Properties']['Size'] = volume.size\n\n # set a name for the new volume\n volume_resource['Properties']['Tags'] = [{'Key': 'Name', 'Value': volume.ec2_volume_name}]\n\n return volume_resource\n\n\ndef _get_volume_resources(ec2, volumes: List[AbstractInstanceVolume], output: AbstractOutputWriter):\n resources = {}\n\n # ending letters for the devices (see: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/device_naming.html)\n device_letters = 'fghijklmnop'\n\n # create and attach volumes\n for i, volume in enumerate(volumes):\n if isinstance(volume, EbsVolume):\n device_letter = device_letters[i]\n\n ec2_volume = Volume.get_by_name(ec2, volume.ec2_volume_name)\n if ec2_volume:\n # check if the volume is available\n if not ec2_volume.is_available():\n raise ValueError('EBS volume \"%s\" is not available (state: %s).'\n % (volume.ec2_volume_name, ec2_volume.state))\n\n # check size of the volume\n if volume.size and (volume.size != ec2_volume.size):\n raise ValueError('Specified size for the \"%s\" volume (%dGB) doesn\\'t match the size of the '\n 'existing volume (%dGB).' % (volume.name, volume.size, ec2_volume.size))\n\n output.write('- volume \"%s\" (%s) will be attached' % (ec2_volume.name, ec2_volume.volume_id))\n\n volume_id = ec2_volume.volume_id\n else:\n # create Volume resource\n vol_resource_name = 'Volume' + device_letter.upper()\n vol_resource = _get_volume_resource(ec2, volume, output)\n resources[vol_resource_name] = vol_resource\n\n volume_id = {'Ref': vol_resource_name}\n\n # create VolumeAttachment resource\n vol_attachment_resource_name = 'VolumeAttachment' + device_letter.upper()\n device_name = '/dev/sd' + device_letter\n vol_attachment_resource = _get_volume_attachment_resource(volume_id, device_name)\n resources[vol_attachment_resource_name] = vol_attachment_resource\n\n return resources\n\n\ndef get_template_parameters(ec2, instance_config: InstanceConfig, instance_profile_arn: str, bucket_name: str,\n key_pair_name: str, output: AbstractOutputWriter):\n # get AMI\n ami = get_ami(ec2, instance_config.ami_id, instance_config.ami_name)\n output.write('- AMI: \"%s\" (%s)' % (ami.name, ami.image_id))\n\n # check root volume size\n root_volume_size = instance_config.root_volume_size\n if root_volume_size and root_volume_size < ami.size:\n raise ValueError('Root volume size cannot be less than the size of AMI (%dGB).' % ami.size)\n elif not root_volume_size:\n # if a root volume size is not specified, make it 5GB larger than the AMI size\n root_volume_size = ami.size + 5\n\n # print info about the Docker data root\n ebs_volumes = [volume for volume in instance_config.volumes if isinstance(volume, EbsVolume)]\n if instance_config.docker_data_root:\n docker_data_volume_name = [volume.name for volume in ebs_volumes\n if is_subdir(instance_config.docker_data_root, volume.mount_dir)][0]\n output.write('- Docker data will be stored on the \"%s\" volume' % docker_data_volume_name)\n\n # create stack\n parameters = {\n 'VpcId': get_vpc_id(ec2, instance_config.subnet_id),\n 'InstanceProfileArn': instance_profile_arn,\n 'InstanceType': instance_config.instance_type,\n 'KeyName': key_pair_name,\n 'ImageId': ami.image_id,\n 'RootVolumeSize': str(root_volume_size),\n 'DockerDataRootDirectory': instance_config.docker_data_root,\n 'InstanceNameTag': instance_config.ec2_instance_name,\n 'HostProjectDirectory': instance_config.host_project_dir,\n 'LogsS3Path': get_logs_s3_path(bucket_name, instance_config.name),\n }\n\n return parameters\n" }, { "path": "spotty/providers/aws/cfn_templates/instance_profile/__init__.py", "content": "" }, { "path": "spotty/providers/aws/cfn_templates/instance_profile/data/template.yaml", "content": "Description: Spotty EC2 Instance Profile\nResources:\n InstanceProfile:\n Type: AWS::IAM::InstanceProfile\n Properties:\n Roles:\n - Ref: InstanceRole\n InstanceRole:\n Type: AWS::IAM::Role\n Properties:\n AssumeRolePolicyDocument:\n Version: '2012-10-17'\n Statement:\n - Effect: Allow\n Principal:\n Service:\n - ec2.amazonaws.com\n Action:\n - sts:AssumeRole\n {{#HAS_MANAGED_POLICIES}}\n ManagedPolicyArns:\n {{#MANAGED_POLICY_ARNS}}\n - {{MANAGED_POLICY_ARN}}\n {{/MANAGED_POLICY_ARNS}}\n {{/HAS_MANAGED_POLICIES}}\n Policies:\n - PolicyName: S3AccessPolicy\n PolicyDocument:\n Version: '2012-10-17'\n Statement:\n - Effect: Allow\n Action:\n - s3:ListAllMyBuckets\n - s3:GetBucketLocation\n - s3:ListBucket\n - s3:GetObject\n - s3:PutObject\n - s3:DeleteObject\n Resource:\n - arn:aws:s3:::*\nOutputs:\n ProfileArn:\n Value: !GetAtt InstanceProfile.Arn\n" }, { "path": "spotty/providers/aws/cfn_templates/instance_profile/template.py", "content": "import os\nimport chevron\n\n\ndef prepare_instance_profile_template(managed_policy_arns: list):\n with open(os.path.join(os.path.dirname(__file__), 'data', 'template.yaml')) as f:\n content = f.read()\n\n parameters = {\n 'HAS_MANAGED_POLICIES': len(managed_policy_arns),\n 'MANAGED_POLICY_ARNS': [{'MANAGED_POLICY_ARN': arn} for arn in managed_policy_arns]\n }\n\n template = chevron.render(content, parameters)\n\n return template\n" }, { "path": "spotty/providers/aws/commands/__init__.py", "content": "" }, { "path": "spotty/providers/aws/commands/clean_logs.py", "content": "from argparse import ArgumentParser, Namespace\nfrom time import time\nimport boto3\nfrom spotty.commands.abstract_command import AbstractCommand\nfrom spotty.commands.writers.abstract_output_writrer import AbstractOutputWriter\n\n\nclass CleanLogsCommand(AbstractCommand):\n\n name = 'clean-logs'\n description = 'Delete expired CloudFormation log groups with Spotty prefixes'\n\n def configure(self, parser: ArgumentParser):\n super().configure(parser)\n parser.add_argument('-r', '--region', type=str, required=True, help='AWS region')\n parser.add_argument('-a', '--delete-all', action='store_true', help='Delete all Spotty log groups, '\n 'not just expired ones')\n\n def run(self, args: Namespace, output: AbstractOutputWriter):\n region = args.region\n logs = boto3.client('logs', region_name=region)\n\n prefixes = ['spotty-', '/aws/lambda/spotty-']\n only_empty = not args.delete_all\n\n output.write('Deleting %s Spotty log groups...' % ('empty' if only_empty else 'all'))\n\n res = logs.describe_log_groups()\n self._delete_log_groups(logs, res['logGroups'], prefixes, only_empty, output)\n\n while 'nextToken' in res:\n res = logs.describe_log_groups(nextToken=res['nextToken'])\n self._delete_log_groups(logs, res['logGroups'], prefixes, only_empty, output)\n\n output.write('Done')\n\n @staticmethod\n def _delete_log_groups(logs, log_groups: list, prefixes: list, only_empty: bool, output: AbstractOutputWriter):\n for log_group in log_groups:\n for prefix in prefixes:\n if log_group['logGroupName'].startswith(prefix):\n delete = True\n if only_empty:\n delete = False\n days_passed = (int(time()) - log_group['creationTime'] // 1000) // 86400\n if ('retentionInDays' in log_group) and (days_passed >= log_group['retentionInDays']):\n delete = True\n\n if delete:\n output.write('[x] %s' % log_group['logGroupName'])\n logs.delete_log_group(logGroupName=log_group['logGroupName'])\n break\n" }, { "path": "spotty/providers/aws/commands/spot_prices.py", "content": "from argparse import ArgumentParser, Namespace\nimport boto3\nfrom spotty.commands.abstract_command import AbstractCommand\nfrom spotty.commands.writers.abstract_output_writrer import AbstractOutputWriter\nfrom spotty.providers.aws.helpers.instance_prices import get_spot_prices\n\n\nclass SpotPricesCommand(AbstractCommand):\n\n name = 'spot-prices'\n description = 'Get Spot Instance prices for an instance type across all AWS regions or within a specific region.'\n\n def configure(self, parser: ArgumentParser):\n super().configure(parser)\n parser.add_argument('-i', '--instance-type', type=str, required=True, help='Instance type')\n parser.add_argument('-r', '--region', type=str, help='AWS region')\n\n def run(self, args: Namespace, output: AbstractOutputWriter):\n # get all regions\n if not args.region:\n ec2 = boto3.client('ec2')\n res = ec2.describe_regions()\n regions = [row['RegionName'] for row in res['Regions']]\n else:\n regions = [args.region]\n\n instance_type = args.instance_type\n\n output.write('Getting spot instance prices for \"%s\"...\\n' % instance_type)\n\n prices = []\n for region in regions:\n ec2 = boto3.client('ec2', region_name=region)\n res = get_spot_prices(ec2, instance_type)\n prices += [(price, zone) for zone, price in res.items()]\n\n # sort availability zones by price\n prices.sort(key=lambda x: x[0])\n\n if prices:\n output.write('Price Zone')\n for price, zone in prices:\n output.write('%.04f %s' % (price, zone))\n else:\n output.write('Spot instances of this type are not available.')\n" }, { "path": "spotty/providers/aws/config/__init__.py", "content": "" }, { "path": "spotty/providers/aws/config/ebs_volume.py", "content": "from spotty.config.abstract_instance_volume import AbstractInstanceVolume\nfrom spotty.providers.aws.config.validation import validate_ebs_volume_parameters\n\n\nclass EbsVolume(AbstractInstanceVolume):\n\n TYPE_NAME = 'EBS'\n\n DP_CREATE_SNAPSHOT = 'CreateSnapshot'\n DP_UPDATE_SNAPSHOT = 'UpdateSnapshot'\n DP_RETAIN = 'Retain'\n DP_DELETE = 'Delete'\n\n def __init__(self, volume_config: dict, project_name: str, instance_name: str):\n super().__init__(volume_config)\n\n self._project_name = project_name\n self._instance_name = instance_name\n\n def _validate_volume_parameters(self, params: dict) -> dict:\n return validate_ebs_volume_parameters(params)\n\n @property\n def title(self):\n return 'EBS volume'\n\n @property\n def size(self) -> int:\n return self._params['size']\n\n @property\n def type(self) -> str:\n return self._params['type']\n\n @property\n def deletion_policy(self) -> str:\n return self._params['deletionPolicy']\n\n @property\n def deletion_policy_title(self) -> str:\n return {\n EbsVolume.DP_CREATE_SNAPSHOT: 'Create Snapshot',\n EbsVolume.DP_UPDATE_SNAPSHOT: 'Update Snapshot',\n EbsVolume.DP_RETAIN: 'Retain Volume',\n EbsVolume.DP_DELETE: 'Delete Volume',\n }[self.deletion_policy]\n\n @property\n def ec2_volume_name(self) -> str:\n \"\"\"Returns EBS volume name.\"\"\"\n volume_name = self._params['volumeName']\n if not volume_name:\n volume_name = '%s-%s-%s' % (self._project_name.lower(), self._instance_name.lower(), self.name.lower())\n\n return volume_name\n\n @property\n def mount_dir(self) -> str:\n \"\"\"A directory where the volume will be mounted on the host OS.\"\"\"\n if self._params['mountDir']:\n mount_dir = self._params['mountDir']\n else:\n mount_dir = '/mnt/%s' % self.ec2_volume_name\n\n return mount_dir\n\n @property\n def host_path(self) -> str:\n \"\"\"A path on the host OS that will be mounted to the container.\"\"\"\n return self.mount_dir\n" }, { "path": "spotty/providers/aws/config/instance_config.py", "content": "from typing import List\nfrom spotty.config.abstract_instance_config import AbstractInstanceConfig, VolumeMount\nfrom spotty.config.abstract_instance_volume import AbstractInstanceVolume\nfrom spotty.providers.aws.config.validation import validate_instance_parameters\nfrom spotty.providers.aws.config.ebs_volume import EbsVolume\n\n\nDEFAULT_AMI_NAME = 'SpottyAMI'\n\n\nclass InstanceConfig(AbstractInstanceConfig):\n\n def _validate_instance_params(self, params: dict) -> dict:\n return validate_instance_parameters(params)\n\n def _get_instance_volumes(self) -> List[AbstractInstanceVolume]:\n volumes = []\n for volume_config in self._params['volumes']:\n volume_type = volume_config['type']\n if volume_type == EbsVolume.TYPE_NAME:\n volumes.append(EbsVolume(volume_config, self.project_config.project_name, self.name))\n else:\n raise ValueError('AWS volume type \"%s\" not supported.' % volume_type)\n\n return volumes\n\n @property\n def user(self):\n return 'ubuntu'\n\n @property\n def ec2_instance_name(self) -> str:\n return '%s-%s' % (self.project_config.project_name.lower(), self.name.lower())\n\n @property\n def region(self) -> str:\n return self._params['region']\n\n @property\n def availability_zone(self) -> str:\n return self._params['availabilityZone']\n\n @property\n def subnet_id(self) -> str:\n return self._params['subnetId']\n\n @property\n def instance_type(self) -> str:\n return self._params['instanceType']\n\n @property\n def is_spot_instance(self) -> bool:\n return self._params['spotInstance']\n\n @property\n def ami_name(self) -> str:\n return self._params['amiName']\n\n @property\n def ami_id(self) -> str:\n return self._params['amiId']\n\n @property\n def root_volume_size(self) -> int:\n return self._params['rootVolumeSize']\n\n @property\n def ports(self) -> List[int]:\n return list(set(self._params['ports']))\n\n @property\n def max_price(self) -> float:\n return self._params['maxPrice']\n\n @property\n def managed_policy_arns(self) -> list:\n return self._params['managedPolicyArns']\n \n @property\n def instance_profile_arn(self) -> str:\n return self._params['instanceProfileArn']\n" }, { "path": "spotty/providers/aws/config/validation.py", "content": "import os\nfrom schema import Schema, Optional, And, Regex, Or, Use\nfrom spotty.config.validation import validate_config, get_instance_parameters_schema, has_prefix\n\n\ndef validate_instance_parameters(params: dict):\n from spotty.providers.aws.config.ebs_volume import EbsVolume\n\n instance_parameters = {\n 'region': And(str, Regex(r'^[a-z0-9-]+$')),\n Optional('availabilityZone', default=''): And(str, Regex(r'^[a-z0-9-]+$')),\n Optional('subnetId', default=''): And(str, Regex(r'^subnet-[a-z0-9]+$')),\n 'instanceType': str,\n Optional('spotInstance', default=False): bool,\n Optional('amiName', default=None): And(str, len, Regex(r'^[\\w\\(\\)\\[\\]\\s\\.\\/\\'@-]{3,128}$')),\n Optional('amiId', default=None): And(str, len, Regex(r'^ami-[a-z0-9]+$')),\n Optional('rootVolumeSize', default=0): And(Or(int, str), Use(str),\n Regex(r'^\\d+$', error='Incorrect value for \"rootVolumeSize\".'),\n Use(int),\n And(lambda x: x > 0,\n error='\"rootVolumeSize\" should be greater than 0 or should '\n 'not be specified.'),\n ),\n Optional('ports', default=[]): [And(int, lambda x: 0 < x < 65536)],\n Optional('maxPrice', default=0): And(Or(float, int, str), Use(str),\n Regex(r'^\\d+(\\.\\d{1,6})?$', error='Incorrect value for \"maxPrice\".'),\n Use(float),\n And(lambda x: x > 0, error='\"maxPrice\" should be greater than 0 or '\n 'should not be specified.'),\n ),\n Optional('managedPolicyArns', default=[]): [str],\n Optional('instanceProfileArn', default=None): str,\n }\n\n volumes_checks = [\n And(lambda x: len(x) < 12, error='Maximum 11 volumes are supported at the moment.'),\n And(lambda x: not has_prefix([(volume['parameters']['mountDir'] + '/') for volume in x\n if volume['parameters'].get('mountDir')]),\n error='Mount directories cannot be prefixes for each other.'),\n ]\n\n instance_checks = [\n And(lambda x: not (x['maxPrice'] and not x['spotInstance']),\n error='\"maxPrice\" can be specified only for spot instances.'),\n And(lambda x: not (x['amiName'] and x['amiId']),\n error='\"amiName\" and \"amiId\" parameters cannot be used together.'),\n ]\n\n schema = get_instance_parameters_schema(instance_parameters, EbsVolume.TYPE_NAME, instance_checks, volumes_checks)\n\n return validate_config(schema, params)\n\n\ndef validate_ebs_volume_parameters(params: dict):\n from spotty.providers.aws.config.ebs_volume import EbsVolume\n\n old_deletion_policies_map = {\n 'create_snapshot': EbsVolume.DP_CREATE_SNAPSHOT,\n 'update_snapshot': EbsVolume.DP_UPDATE_SNAPSHOT,\n 'retain': EbsVolume.DP_RETAIN,\n 'delete': EbsVolume.DP_DELETE,\n }\n\n schema = Schema({\n Optional('volumeName', default=''): And(str, Regex(r'^[\\w-]{1,255}$')),\n Optional('mountDir', default=''): And(\n str,\n And(os.path.isabs, error='Use absolute paths in the \"mountDir\" parameters'),\n Use(lambda x: x.rstrip('/'))\n ),\n Optional('size', default=0): And(int, lambda x: x > 0),\n # TODO: add the \"iops\" parameter to support the \"io1\" EBS volume type\n Optional('type', default='gp2'): lambda x: x in ['gp2', 'sc1', 'st1', 'standard'],\n Optional('deletionPolicy', default=EbsVolume.DP_RETAIN): And(\n str,\n lambda x: x in [EbsVolume.DP_CREATE_SNAPSHOT,\n EbsVolume.DP_UPDATE_SNAPSHOT,\n EbsVolume.DP_RETAIN,\n EbsVolume.DP_DELETE] + list(old_deletion_policies_map.keys()),\n Use(lambda x: old_deletion_policies_map.get(x, x)),\n error='Incorrect value for \"deletionPolicy\".',\n ),\n })\n\n return validate_config(schema, params)\n" }, { "path": "spotty/providers/aws/data_transfer.py", "content": "import logging\nimport subprocess\nfrom spotty.deployment.abstract_cloud_instance.abstract_data_transfer import AbstractDataTransfer\nfrom spotty.providers.aws.helpers.s3_sync import get_s3_sync_command, check_aws_installed\n\n\nclass DataTransfer(AbstractDataTransfer):\n\n def __init__(self, local_project_dir: str, host_project_dir: str, sync_filters: list, instance_name: str,\n region: str):\n super().__init__(local_project_dir, host_project_dir, sync_filters, instance_name)\n\n self._region = region\n\n @property\n def scheme_name(self) -> str:\n return 's3'\n\n def upload_local_to_bucket(self, bucket_name: str, dry_run: bool = False):\n \"\"\"Uploads files from local to the bucket.\"\"\"\n # check AWS CLI is installed\n check_aws_installed()\n\n # sync the project with S3, deleted files will be deleted from S3\n local_cmd = get_s3_sync_command(self._local_project_dir, self._get_bucket_project_path(bucket_name),\n region=self._region, filters=self._sync_filters, delete=True, dry_run=dry_run)\n\n # execute the command locally\n logging.debug('Local sync command: ' + local_cmd)\n exit_code = subprocess.call(local_cmd, shell=True)\n if exit_code != 0:\n raise ValueError('Failed to upload the project files to the S3 bucket.')\n\n def download_bucket_to_local(self, bucket_name: str, download_filters: list):\n \"\"\"Downloads files from the bucket to local.\"\"\"\n # check AWS CLI is installed\n check_aws_installed()\n\n # download files from S3 bucket to local\n local_cmd = get_s3_sync_command(self._get_bucket_downloads_path(bucket_name), self._local_project_dir,\n region=self._region, filters=download_filters, exact_timestamp=True)\n\n # execute the command locally\n logging.debug('Local sync command: ' + local_cmd)\n exit_code = subprocess.call(local_cmd, shell=True)\n if exit_code != 0:\n raise ValueError('Failed to download files from the S3 bucket to local')\n\n def get_download_bucket_to_instance_command(self, bucket_name: str, use_sudo: bool = False) -> str:\n \"\"\"A remote command to download files from the bucket to the instance.\"\"\"\n remote_cmd = get_s3_sync_command(self._get_bucket_project_path(bucket_name), self._host_project_dir,\n region=self._region, filters=self._sync_filters, exact_timestamp=True)\n if use_sudo:\n remote_cmd = 'sudo ' + remote_cmd\n\n return remote_cmd\n\n def get_upload_instance_to_bucket_command(self, bucket_name: str, download_filters: list, use_sudo: bool = False,\n dry_run: bool = False) -> str:\n \"\"\"A remote command to upload files from the instance to the bucket.\n\n It uses a temporary S3 directory that is unique for the instance. This\n directory keeps all downloaded from the instance files to sync only changed\n files with local.\n \"\"\"\n\n # \"sudo\" should be called with the \"-i\" flag to use the root environment, so aws-cli will read\n # the config file from the root home directory\n remote_cmd = get_s3_sync_command(self._host_project_dir, self._get_bucket_downloads_path(bucket_name),\n region=self._region, filters=download_filters, delete=True, dry_run=dry_run)\n if use_sudo:\n remote_cmd = 'sudo ' + remote_cmd\n\n return remote_cmd\n" }, { "path": "spotty/providers/aws/deletion_policies.py", "content": "from typing import List\nfrom spotty.commands.writers.abstract_output_writrer import AbstractOutputWriter\nfrom spotty.config.abstract_instance_volume import AbstractInstanceVolume\nfrom spotty.providers.aws.resources.snapshot import Snapshot\nfrom spotty.providers.aws.resources.volume import Volume\nfrom spotty.providers.aws.config.ebs_volume import EbsVolume\n\n\ndef apply_deletion_policies(ec2, volumes: List[AbstractInstanceVolume], output: AbstractOutputWriter):\n \"\"\"Applies deletion policies to the EBS volumes.\"\"\"\n\n # get volumes\n ebs_volumes = [volume for volume in volumes if isinstance(volume, EbsVolume)]\n\n # no volumes\n if not ebs_volumes:\n output.write('- no EBS volumes configured')\n return\n\n # apply deletion policies\n wait_snapshots = []\n for volume in ebs_volumes:\n # get EC2 volume\n try:\n ec2_volume = Volume.get_by_name(ec2, volume.ec2_volume_name)\n except Exception as e:\n output.write('- volume \"%s\" not found. Error: %s' % (volume.ec2_volume_name, str(e)))\n continue\n\n if not ec2_volume:\n output.write('- volume \"%s\" not found' % volume.ec2_volume_name)\n continue\n\n if not ec2_volume.is_available():\n output.write('- volume \"%s\" is not available (state: %s)'\n % (volume.ec2_volume_name, ec2_volume.state))\n continue\n\n # apply deletion policies\n if volume.deletion_policy == EbsVolume.DP_RETAIN:\n # do nothing\n output.write('- volume \"%s\" is retained' % ec2_volume.name)\n\n elif volume.deletion_policy == EbsVolume.DP_DELETE:\n # delete EBS volume\n _delete_ec2_volume(ec2_volume, output)\n\n elif volume.deletion_policy == EbsVolume.DP_CREATE_SNAPSHOT \\\n or volume.deletion_policy == EbsVolume.DP_UPDATE_SNAPSHOT:\n try:\n # rename a previous snapshot\n prev_snapshot = Snapshot.get_by_name(ec2, volume.ec2_volume_name)\n if prev_snapshot:\n prev_snapshot.rename('%s-%d' % (prev_snapshot.name, prev_snapshot.creation_time))\n\n output.write('- creating a snapshot for the volume \"%s\"...' % ec2_volume.name)\n\n # create a new snapshot\n new_snapshot = ec2_volume.create_snapshot()\n\n # delete the EBS volume and a previous snapshot only after a new snapshot will be created\n wait_snapshots.append({\n 'new_snapshot': new_snapshot,\n 'prev_snapshot': prev_snapshot,\n 'ec2_volume': ec2_volume,\n 'deletion_policy': volume.deletion_policy,\n })\n except Exception as e:\n output.write('- snapshot for the volume \"%s\" was not created. Error: %s'\n % (volume.ec2_volume_name, str(e)))\n\n else:\n raise ValueError('Unsupported deletion policy: \"%s\".' % volume.deletion_policy)\n\n # wait until all snapshots will be created\n for resources in wait_snapshots:\n try:\n resources['new_snapshot'].wait_snapshot_completed()\n output.write('- snapshot for the volume \"%s\" was created' % resources['new_snapshot'].name)\n except Exception as e:\n output.write('- snapshot \"%s\" was not created. Error: %s' % (resources['new_snapshot'].name, str(e)))\n continue\n\n # delete a previous snapshot if it's the \"update_snapshot\" deletion policy\n if (resources['deletion_policy'] == EbsVolume.DP_UPDATE_SNAPSHOT) and resources['prev_snapshot']:\n _delete_snapshot(resources['prev_snapshot'], output)\n\n # delete the EBS volume\n _delete_ec2_volume(resources['ec2_volume'], output)\n\n\ndef _delete_ec2_volume(ec2_volume: Volume, output: AbstractOutputWriter):\n try:\n ec2_volume.delete()\n output.write('- volume \"%s\" was deleted' % ec2_volume.name)\n except Exception as e:\n output.write('- volume \"%s\" was not deleted. Error: %s' % (ec2_volume.name, str(e)))\n\n\ndef _delete_snapshot(snapshot: Snapshot, output: AbstractOutputWriter):\n try:\n snapshot.delete()\n output.write('- previous snapshot \"%s\" was deleted' % snapshot.name)\n except Exception as e:\n output.write('- previous snapshot \"%s\" was not deleted. Error: %s' % (snapshot.name, str(e)))\n" }, { "path": "spotty/providers/aws/errors/__init__.py", "content": "" }, { "path": "spotty/providers/aws/errors/volume_not_found.py", "content": "class VolumeNotFoundError(Exception):\n def __init__(self, volume_name):\n super().__init__('Volume \"%s\" not found' % volume_name)\n" }, { "path": "spotty/providers/aws/helpers/__init__.py", "content": "" }, { "path": "spotty/providers/aws/helpers/ami.py", "content": "from spotty.providers.aws.config.instance_config import DEFAULT_AMI_NAME\nfrom spotty.providers.aws.resources.image import Image\n\n\ndef get_ami(ec2, ami_id: str = None, ami_name: str = None) -> Image:\n \"\"\"Returns an AMI that should be used for deployment.\n\n Raises:\n ValueError: If an AMI not found.\n \"\"\"\n if ami_id:\n # get an AMI by ID if the \"amiId\" parameter is specified\n image = Image.get_by_id(ec2, ami_id)\n if not image:\n raise ValueError('AMI with ID=%s not found.' % ami_id)\n elif ami_name:\n # get an AMI by name if the \"amiName\" parameter is specified\n image = Image.get_by_name(ec2, ami_name)\n if not image:\n # if an AMI name was explicitly specified in the config, but the AMI was not found, raise an error\n raise ValueError('AMI with the name \"%s\" was not found.' % ami_name)\n else:\n # if the \"amiName\" parameter is not specified, try to use the default AMI name\n image = Image.get_by_name(ec2, DEFAULT_AMI_NAME)\n if not image:\n # get the latest \"Deep Learning Base AMI\"\n res = ec2.describe_images(\n Owners=['amazon'],\n Filters=[{'Name': 'name', 'Values': ['Deep Learning AMI (Ubuntu 16.04) Version*']}],\n )\n\n if not len(res['Images']):\n raise ValueError('AWS Deep Learning AMI not found.\\n'\n 'Use the \"spotty aws create-ami\" command to create an AMI with NVIDIA Docker.')\n\n image_info = sorted(res['Images'], key=lambda x: x['CreationDate'], reverse=True)[0]\n image = Image(ec2, image_info)\n\n return image\n" }, { "path": "spotty/providers/aws/helpers/availability_zone.py", "content": "from typing import List\nfrom spotty.config.abstract_instance_volume import AbstractInstanceVolume\nfrom spotty.providers.aws.config.ebs_volume import EbsVolume\nfrom spotty.providers.aws.resources.volume import Volume\n\n\ndef update_availability_zone(ec2, availability_zone: str, volumes: List[AbstractInstanceVolume]):\n \"\"\"Checks that existing volumes located in the same AZ and the AZ from the\n config file matches volumes AZ.\n\n Args:\n ec2: EC2 boto3 client\n availability_zone: Availability Zone from the configuration.\n volumes: List of volume objects.\n\n Returns:\n The final AZ where the instance should be run or an empty string if\n the instance can be run in any AZ.\n\n Raises:\n ValueError: AZ in the config file doesn't match the AZs of the volumes or\n AZs of the volumes are different.\n \"\"\"\n availability_zone = availability_zone\n for volume in volumes:\n if isinstance(volume, EbsVolume):\n ec2_volume = Volume.get_by_name(ec2, volume.ec2_volume_name)\n if ec2_volume:\n if availability_zone and (availability_zone != ec2_volume.availability_zone):\n raise ValueError(\n 'The availability zone in the configuration file doesn\\'t match the availability zone '\n 'of the existing volume or you have two existing volumes in different availability '\n 'zones.')\n\n # update availability zone\n availability_zone = ec2_volume.availability_zone\n\n return availability_zone\n" }, { "path": "spotty/providers/aws/helpers/instance_prices.py", "content": "import datetime\nimport json\nimport logging\nimport boto3\nfrom pkg_resources import resource_filename\n\n\ndef get_spot_prices(ec2, instance_type: str):\n \"\"\"Returns current Spot Instance prices for all availability zones for particular instance type and region.\n AWS region specified implicitly in the \"ec2\" object.\n \"\"\"\n tomorrow_date = datetime.datetime.today() + datetime.timedelta(days=1)\n res = ec2.describe_spot_price_history(InstanceTypes=[instance_type],\n StartTime=tomorrow_date,\n ProductDescriptions=['Linux/UNIX'])\n\n prices_by_zone = {}\n for row in res['SpotPriceHistory']:\n prices_by_zone[row['AvailabilityZone']] = float(row['SpotPrice'])\n\n return prices_by_zone\n\n\ndef get_current_spot_price(ec2, instance_type, availability_zone=''):\n \"\"\"Returns the current Spot price for an availability zone.\n If an availability zone is not specified, returns the minimum price for the region.\n \"\"\"\n spot_prices = get_spot_prices(ec2, instance_type)\n if availability_zone:\n if availability_zone not in spot_prices:\n raise ValueError('Spot price for the \"%s\" availability zone not found.' % availability_zone)\n\n current_price = spot_prices[availability_zone]\n else:\n current_price = min(spot_prices.values())\n\n return current_price\n\n\ndef get_on_demand_price(instance_type: str, region: str):\n client = boto3.client('pricing', region_name='us-east-1') # the API available only in \"us-east-1\"\n\n try:\n response = client.get_products(\n ServiceCode='AmazonEC2',\n Filters=[\n {'Type': 'TERM_MATCH', 'Field': 'location', 'Value': _get_region_name(region)},\n {'Type': 'TERM_MATCH', 'Field': 'instanceType', 'Value': instance_type},\n {'Type': 'TERM_MATCH', 'Field': 'operatingSystem', 'Value': 'Linux'},\n {'Type': 'TERM_MATCH', 'Field': 'tenancy', 'Value': 'shared'},\n {'Type': 'TERM_MATCH', 'Field': 'capacitystatus', 'Value': 'Used'},\n ],\n )\n\n prices = json.loads(response['PriceList'][0])['terms']['OnDemand']\n price = float(list(list(prices.values())[0]['priceDimensions'].values())[0]['pricePerUnit']['USD'])\n except Exception as e:\n logging.debug('Couldn\\'t find a price for the instance: ' + str(e))\n price = None\n\n return price\n\n\ndef _get_region_name(region: str):\n endpoint_file = resource_filename('botocore', 'data/endpoints.json')\n try:\n with open(endpoint_file, 'r') as f:\n data = json.load(f)\n region_name = data['partitions'][0]['regions'][region]['description']\n except Exception as e:\n logging.debug('Couldn\\'t obtain the region name: ' + str(e))\n region_name = None\n\n return region_name\n\n\ndef check_max_spot_price(ec2, instance_type: str, is_spot_instance: bool, max_price: float,\n availability_zone: str = ''):\n \"\"\"Checks that the specified maximum Spot price is less than the\n current Spot price.\n\n Args:\n ec2: EC2 client\n instance_type (str): Instance Type\n is_spot_instance (bool): True if it's a spot instance\n max_price (float): requested maximum price for the instance\n availability_zone (str): Availability zone to check. If it's an empty string,\n checks the cheapest AZ.\n\n Raises:\n ValueError: Current price for the instance is higher than the\n maximum price in the configuration file.\n \"\"\"\n if is_spot_instance and max_price:\n current_price = get_current_spot_price(ec2, instance_type, availability_zone)\n if current_price > max_price:\n raise ValueError('Current price for the instance (%.04f) is higher than the maximum price in the '\n 'configuration file (%.04f).' % (current_price, max_price))\n" }, { "path": "spotty/providers/aws/helpers/logs.py", "content": "import os\nimport subprocess\nimport tempfile\nfrom glob import glob\nfrom spotty.providers.aws.helpers.s3_sync import get_s3_sync_command\n\n\ndef get_logs_s3_path(bucket_name: str, instance_name: str) -> str:\n return 's3://%s/logs/aws/%s' % (bucket_name, instance_name)\n\n\ndef download_logs(bucket_name: str, instance_name: str, stack_uuid: str, region: str) -> list:\n \"\"\"Downloads logs from S3 bucket to temporary directory.\"\"\"\n logs_s3_path = '%s/%s' % (get_logs_s3_path(bucket_name, instance_name), stack_uuid)\n local_logs_dir = tempfile.mkdtemp()\n\n # download logs\n download_cmd = get_s3_sync_command(logs_s3_path, local_logs_dir, region=region, exact_timestamp=True, quiet=True)\n subprocess.call(download_cmd, shell=True)\n\n # get paths to the downloaded files\n log_paths = glob(os.path.join(local_logs_dir, '**', '*'), recursive=True)\n\n return log_paths\n" }, { "path": "spotty/providers/aws/helpers/s3_sync.py", "content": "from shutil import which\nfrom spotty.deployment.utils.cli import shlex_join\n\n\ndef check_aws_installed():\n \"\"\"Checks that AWS CLI is installed.\"\"\"\n if which('aws') is None:\n raise ValueError('AWS CLI is not installed.')\n\n\ndef get_s3_sync_command(from_path: str, to_path: str, profile: str = None, region: str = None, filters: list = None,\n exact_timestamp: bool = False, delete: bool = False, quiet: bool = False,\n dry_run: bool = False):\n \"\"\"Builds an \"aws s3 sync\" command.\"\"\"\n args = ['aws']\n\n if profile:\n args += ['--profile', profile]\n\n if region:\n args += ['--region', region]\n\n args += ['s3', 'sync', from_path, to_path]\n\n if filters:\n for sync_filter in filters:\n if ('exclude' in sync_filter and 'include' in sync_filter) \\\n or ('exclude' not in sync_filter and 'include' not in sync_filter):\n raise ValueError('S3 sync filter has wrong format.')\n\n if 'exclude' in sync_filter:\n for path in sync_filter['exclude']:\n args += ['--exclude', path]\n\n if 'include' in sync_filter:\n for path in sync_filter['include']:\n args += ['--include', path]\n\n if exact_timestamp:\n args.append('--exact-timestamp')\n\n if delete:\n args.append('--delete')\n\n if quiet:\n args.append('--quiet')\n\n if dry_run:\n args.append('--dryrun')\n\n command = shlex_join(args)\n\n return command\n" }, { "path": "spotty/providers/aws/helpers/subnet.py", "content": "from spotty.providers.aws.resources.subnet import Subnet\n\n\ndef check_az_and_subnet(ec2, region: str, availability_zone: str, subnet_id: str):\n # get all availability zones for the region\n zones = ec2.describe_availability_zones()\n zone_names = [zone['ZoneName'] for zone in zones['AvailabilityZones']]\n\n # check availability zone\n if availability_zone and availability_zone not in zone_names:\n raise ValueError('Availability zone \"%s\" doesn\\'t exist in the \"%s\" region.'\n % (availability_zone, region))\n\n if availability_zone:\n if subnet_id:\n subnet = Subnet.get_by_id(ec2, subnet_id)\n if not subnet:\n raise ValueError('Subnet \"%s\" not found.' % subnet_id)\n\n if subnet.availability_zone != availability_zone:\n raise ValueError('Availability zone of the subnet doesn\\'t match the specified availability zone')\n else:\n default_subnets = Subnet.get_default_subnets(ec2)\n default_subnet = [subnet for subnet in default_subnets\n if subnet.availability_zone == availability_zone]\n if not default_subnet:\n raise ValueError('Default subnet for the \"%s\" availability zone not found.\\n'\n 'Use the \"subnetId\" parameter to specify a subnet for this availability zone.'\n % availability_zone)\n else:\n if subnet_id:\n raise ValueError('An availability zone should be specified if a custom subnet is used.')\n else:\n default_subnets = Subnet.get_default_subnets(ec2)\n default_azs = {subnet.availability_zone for subnet in default_subnets}\n zones_wo_subnet = [zone_name for zone_name in zone_names if zone_name not in default_azs]\n if zones_wo_subnet:\n raise ValueError('Default subnets for the following availability zones were not found: %s.\\n'\n 'Use \"subnetId\" and \"availabilityZone\" parameters or create missing default '\n 'subnets.' % ', '.join(zones_wo_subnet))\n" }, { "path": "spotty/providers/aws/helpers/vpc.py", "content": "from spotty.providers.aws.resources.subnet import Subnet\nfrom spotty.providers.aws.resources.vpc import Vpc\n\n\ndef get_vpc_id(ec2, subnet_id: str = None) -> str:\n \"\"\"Returns VPC ID that should be used for deployment.\"\"\"\n if subnet_id:\n vpc_id = Subnet.get_by_id(ec2, subnet_id).vpc_id\n else:\n default_vpc = Vpc.get_default_vpc(ec2)\n if not default_vpc:\n raise ValueError('Default VPC not found')\n\n vpc_id = default_vpc.vpc_id\n\n return vpc_id\n" }, { "path": "spotty/providers/aws/instance_deployment.py", "content": "import boto3\nfrom spotty.commands.writers.abstract_output_writrer import AbstractOutputWriter\nfrom spotty.deployment.abstract_cloud_instance.abstract_instance_deployment import AbstractInstanceDeployment\nfrom spotty.deployment.container.docker.docker_commands import DockerCommands\nfrom spotty.providers.aws.cfn_templates.instance.template import prepare_instance_template, get_template_parameters\nfrom spotty.providers.aws.data_transfer import DataTransfer\nfrom spotty.providers.aws.helpers.availability_zone import update_availability_zone\nfrom spotty.providers.aws.helpers.instance_prices import check_max_spot_price\nfrom spotty.providers.aws.helpers.subnet import check_az_and_subnet\nfrom spotty.providers.aws.resource_managers.key_pair_manager import KeyPairManager\nfrom spotty.deployment.utils.print_info import render_volumes_info_table\nfrom spotty.providers.aws.resources.instance import Instance\nfrom spotty.providers.aws.config.instance_config import InstanceConfig\nfrom spotty.providers.aws.deletion_policies import apply_deletion_policies\nfrom spotty.providers.aws.resource_managers.instance_profile_stack_manager import InstanceProfileStackManager\nfrom spotty.providers.aws.helpers.logs import download_logs\nfrom spotty.providers.aws.resource_managers.instance_stack_manager import InstanceStackManager\n\n\nclass InstanceDeployment(AbstractInstanceDeployment):\n\n instance_config: InstanceConfig\n\n def __init__(self, instance_config: InstanceConfig):\n super().__init__(instance_config)\n\n self._project_name = instance_config.project_config.project_name\n self._ec2 = boto3.client('ec2', region_name=instance_config.region)\n\n @property\n def stack_manager(self) -> InstanceStackManager:\n return InstanceStackManager(self._project_name, self.instance_config.name, self.instance_config.region)\n\n @property\n def key_pair_manager(self) -> KeyPairManager:\n return KeyPairManager(self._ec2, self._project_name, self.instance_config.region)\n\n def get_instance(self) -> Instance:\n return Instance.get_by_stack_name(self._ec2, self.stack_manager.name)\n\n def deploy(self, container_commands: DockerCommands, bucket_name: str,\n data_transfer: DataTransfer, output: AbstractOutputWriter, dry_run: bool = False):\n # get deployment availability zone\n availability_zone = update_availability_zone(self._ec2, self.instance_config.availability_zone,\n self.instance_config.volumes)\n\n # check availability zone and subnet configuration\n check_az_and_subnet(self._ec2, self.instance_config.region, availability_zone, self.instance_config.subnet_id)\n\n # check the maximum price for a spot instance\n check_max_spot_price(self._ec2, self.instance_config.instance_type, self.instance_config.is_spot_instance,\n self.instance_config.max_price, availability_zone)\n\n # sync the project with the S3 bucket\n if bucket_name is not None:\n output.write('Syncing the project with the S3 bucket...')\n data_transfer.upload_local_to_bucket(bucket_name, dry_run=dry_run)\n\n # create or update instance profile\n if not dry_run:\n instance_profile_stack_manager = InstanceProfileStackManager(\n self._project_name, self.instance_config.name, self.instance_config.region)\n if not self.instance_config.instance_profile_arn:\n instance_profile_arn = instance_profile_stack_manager.create_or_update_stack(\n self.instance_config.managed_policy_arns, output=output)\n else:\n instance_profile_arn = self.instance_config.instance_profile_arn\n else:\n instance_profile_arn = None\n\n # create a key pair if it doesn't exist\n if not dry_run:\n self.key_pair_manager.maybe_create_key()\n\n output.write('Preparing CloudFormation template...')\n\n # prepare CloudFormation template\n with output.prefix(' '):\n template = prepare_instance_template(\n ec2=self._ec2,\n instance_config=self.instance_config,\n docker_commands=container_commands,\n availability_zone=availability_zone,\n sync_project_cmd=data_transfer.get_download_bucket_to_instance_command(bucket_name=bucket_name),\n output=output,\n )\n\n # get parameters for the template\n parameters = get_template_parameters(\n ec2=self._ec2,\n instance_config=self.instance_config,\n instance_profile_arn=instance_profile_arn,\n bucket_name=bucket_name,\n key_pair_name=self.key_pair_manager.key_name,\n output=output,\n )\n\n # print information about the volumes\n output.write('\\nVolumes:\\n%s\\n'\n % render_volumes_info_table(self.instance_config.volume_mounts, self.instance_config.volumes))\n\n # create stack\n if not dry_run:\n stack = self.stack_manager.create_or_update_stack(template, parameters, self.instance_config, output)\n if stack.status != 'CREATE_COMPLETE':\n logs_str = 'Please, see CloudFormation logs for the details.'\n\n # download CloudFormation logs from the instance if it was created\n if self.get_instance():\n log_paths = download_logs(\n bucket_name=bucket_name,\n instance_name=self.instance_config.name,\n stack_uuid=stack.stack_uuid,\n region=self.instance_config.region,\n )\n\n logs_str = 'Please, see the logs for the details:\\n '\n logs_str += '\\n '.join(log_paths)\n\n raise ValueError('Stack \"%s\" was not created.\\n%s' % (stack.name, logs_str))\n\n def delete(self, output: AbstractOutputWriter):\n # terminate the instance\n instance = self.get_instance()\n if instance:\n output.write('Terminating the instance... ', newline=False)\n instance.terminate()\n output.write('DONE')\n else:\n output.write('The instance was already terminated.')\n\n # delete the stack in background if it exists\n self.stack_manager.delete_stack(output, no_wait=True)\n\n output.write('Applying deletion policies for the volumes...')\n\n # apply deletion policies for the volumes\n with output.prefix(' '):\n apply_deletion_policies(self._ec2, self.instance_config.volumes, output)\n" }, { "path": "spotty/providers/aws/instance_manager.py", "content": "from spotty.errors.instance_not_running import InstanceNotRunningError\nfrom spotty.deployment.abstract_cloud_instance.abstract_cloud_instance_manager import AbstractCloudInstanceManager\nfrom spotty.providers.aws.resource_managers.bucket_manager import BucketManager\nfrom spotty.providers.aws.config.instance_config import InstanceConfig\nfrom spotty.providers.aws.data_transfer import DataTransfer\nfrom spotty.providers.aws.instance_deployment import InstanceDeployment\nfrom spotty.utils import render_table\n\n\nclass InstanceManager(AbstractCloudInstanceManager):\n\n instance_config: InstanceConfig\n bucket_manager: BucketManager\n data_transfer: DataTransfer\n instance_deployment: InstanceDeployment\n\n def _get_instance_config(self, instance_config: dict) -> InstanceConfig:\n \"\"\"Validates the instance config and returns an InstanceConfig object.\"\"\"\n return InstanceConfig(instance_config, self.project_config)\n\n def _get_bucket_manager(self) -> BucketManager:\n \"\"\"Returns an bucket manager.\"\"\"\n return BucketManager(self.instance_config.project_config.project_name, self.instance_config.region)\n\n def _get_data_transfer(self) -> DataTransfer:\n \"\"\"Returns a data transfer object.\"\"\"\n return DataTransfer(\n local_project_dir=self.project_config.project_dir,\n host_project_dir=self.instance_config.host_project_dir,\n sync_filters=self.project_config.sync_filters,\n instance_name=self.instance_config.name,\n region=self.instance_config.region,\n )\n\n def _get_instance_deployment(self) -> InstanceDeployment:\n \"\"\"Returns an instance deployment manager.\"\"\"\n return InstanceDeployment(self.instance_config)\n\n def get_status_text(self):\n instance = self.instance_deployment.get_instance()\n if not instance:\n raise InstanceNotRunningError(self.instance_config.name)\n\n table = [\n ('Instance State', instance.state),\n ('Instance Type', instance.instance_type),\n ('Availability Zone', instance.availability_zone),\n ]\n\n if instance.public_ip_address:\n table.append(('Public IP Address', instance.public_ip_address))\n elif instance.private_ip_address:\n table.append(('Private IP Address', instance.private_ip_address))\n\n if instance.lifecycle == 'spot':\n spot_price = instance.get_spot_price()\n table.append(('Purchasing Option', 'Spot Instance'))\n table.append(('Spot Instance Price', '$%.04f' % spot_price))\n else:\n on_demand_price = instance.get_on_demand_price()\n table.append(('Purchasing Option', 'On-Demand Instance'))\n table.append(('Instance Price', ('$%.04f (us-east-1)' % on_demand_price) if on_demand_price else 'Unknown'))\n\n return render_table(table)\n\n @property\n def ssh_key_path(self):\n return self.instance_deployment.key_pair_manager.key_path\n" }, { "path": "spotty/providers/aws/resource_managers/__init__.py", "content": "" }, { "path": "spotty/providers/aws/resource_managers/bucket_manager.py", "content": "import boto3\nimport re\nfrom spotty.deployment.abstract_cloud_instance.abstract_bucket_manager import AbstractBucketManager\nfrom spotty.deployment.abstract_cloud_instance.errors.bucket_not_found import BucketNotFoundError\nfrom spotty.providers.aws.resources.bucket import Bucket\nfrom spotty.utils import random_string\n\n\nclass BucketManager(AbstractBucketManager):\n\n def __init__(self, project_name: str, region: str):\n super().__init__(project_name)\n\n self._s3 = boto3.client('s3', region_name=region)\n self._region = region\n self._bucket_prefix = 'spotty-%s' % project_name.lower()\n\n def get_bucket(self) -> Bucket:\n res = self._s3.list_buckets()\n regex = re.compile('-'.join([self._bucket_prefix, '[a-z0-9]{12}', self._region]))\n buckets = [bucket for bucket in res['Buckets'] if regex.match(bucket['Name']) is not None]\n\n if len(buckets) > 1:\n raise ValueError('Found several buckets in the same region: %s.'\n % ', '.join(bucket['Name'] for bucket in buckets))\n\n if not len(buckets):\n raise BucketNotFoundError\n\n bucket = Bucket(buckets[0])\n\n return bucket\n\n def create_bucket(self) -> Bucket:\n bucket_name = '-'.join([self._bucket_prefix, random_string(12), self._region])\n\n # a fix for the boto3 issue: https://github.com/boto/boto3/issues/125\n if self._region == 'us-east-1':\n self._s3.create_bucket(ACL='private', Bucket=bucket_name)\n else:\n self._s3.create_bucket(ACL='private', Bucket=bucket_name,\n CreateBucketConfiguration={'LocationConstraint': self._region})\n\n return Bucket({'Name': bucket_name})\n\n def delete_bucket(self):\n pass\n" }, { "path": "spotty/providers/aws/resource_managers/instance_profile_stack_manager.py", "content": "import boto3\nfrom botocore.exceptions import ClientError, WaiterError\nfrom spotty.commands.writers.abstract_output_writrer import AbstractOutputWriter\nfrom spotty.providers.aws.cfn_templates.instance_profile.template import prepare_instance_profile_template\nfrom spotty.providers.aws.resources.stack import Stack\n\n\nclass InstanceProfileStackManager(object):\n\n def __init__(self, project_name: str, instance_name: str, region: str):\n self._cf = boto3.client('cloudformation', region_name=region)\n self._region = region\n self._stack_name = 'spotty-instance-profile-%s-%s' % (project_name.lower(), instance_name.lower())\n\n def create_or_update_stack(self, managed_policy_arns: list, output: AbstractOutputWriter):\n \"\"\"Creates or updates an instance profile.\n It was moved to a separate stack because creating of an instance profile resource takes 2 minutes.\n \"\"\"\n # check that policies exist\n iam = boto3.client('iam', region_name=self._region)\n for policy_arn in managed_policy_arns:\n # if the policy doesn't exist, an error will be raised\n iam.get_policy(PolicyArn=policy_arn)\n\n template = prepare_instance_profile_template(managed_policy_arns)\n\n stack = Stack.get_by_name(self._cf, self._stack_name)\n try:\n if stack:\n # update the stack and wait until it will be updated\n self._update_stack(template, output)\n else:\n # create the stack and wait until it will be created\n self._create_stack(template, output)\n\n stack = Stack.get_by_name(self._cf, self._stack_name)\n except WaiterError:\n stack = None\n\n if not stack or stack.status not in ['CREATE_COMPLETE', 'UPDATE_COMPLETE']:\n raise ValueError('Stack \"%s\" was not created.\\n'\n 'Please, see CloudFormation logs for the details.' % self._stack_name)\n\n profile_arn = [row['OutputValue'] for row in stack.outputs if row['OutputKey'] == 'ProfileArn'][0]\n\n return profile_arn\n\n def _create_stack(self, template: str, output: AbstractOutputWriter):\n \"\"\"Creates the stack and waits until it will be created.\"\"\"\n output.write('Creating IAM role for the instance...')\n\n stack = Stack.create_stack(\n cf=self._cf,\n StackName=self._stack_name,\n TemplateBody=template,\n Capabilities=['CAPABILITY_IAM'],\n OnFailure='DELETE',\n )\n\n # wait for the stack to be created\n stack.wait_stack_created(delay_secs=15)\n\n def _update_stack(self, template: str, output: AbstractOutputWriter):\n \"\"\"Updates the stack and waits until it will be updated.\"\"\"\n try:\n updated_stack = Stack.update_stack(\n cf=self._cf,\n StackName=self._stack_name,\n TemplateBody=template,\n Capabilities=['CAPABILITY_IAM'],\n )\n except ClientError as e:\n # the stack was not updated because there are no changes\n updated_stack = None\n error_code = e.response.get('Error', {}).get('Code', 'Unknown')\n if error_code != 'ValidationError':\n raise e\n\n if updated_stack:\n # wait for the stack to be updated\n output.write('Updating IAM role for the instance...')\n updated_stack.wait_stack_updated(delay=15)\n" }, { "path": "spotty/providers/aws/resource_managers/instance_stack_manager.py", "content": "import boto3\nfrom spotty.commands.writers.abstract_output_writrer import AbstractOutputWriter\nfrom spotty.providers.aws.resources.stack import Stack, Task\nfrom spotty.providers.aws.config.instance_config import InstanceConfig\n\n\nclass InstanceStackManager(object):\n\n def __init__(self, project_name: str, instance_name: str, region: str):\n self._cf = boto3.client('cloudformation', region_name=region)\n self._ec2 = boto3.client('ec2', region_name=region)\n self._region = region\n self._stack_name = 'spotty-instance-%s-%s' % (project_name.lower(), instance_name.lower())\n\n @property\n def name(self):\n return self._stack_name\n\n def create_or_update_stack(self, template: str, parameters: dict, instance_config: InstanceConfig,\n output: AbstractOutputWriter):\n \"\"\"Runs CloudFormation template.\"\"\"\n\n # delete the stack if it exists\n stack = Stack.get_by_name(self._cf, self._stack_name)\n if stack:\n self.delete_stack(output)\n\n # create new stack\n stack = Stack.create_stack(\n cf=self._cf,\n StackName=self._stack_name,\n TemplateBody=template,\n Parameters=[{'ParameterKey': key, 'ParameterValue': value} for key, value in parameters.items()],\n Capabilities=['CAPABILITY_IAM'],\n OnFailure='DO_NOTHING',\n )\n\n output.write('Waiting for the stack to be created...')\n\n tasks = [\n Task(\n message='launching the instance',\n start_resource=None,\n finish_resource='Instance',\n enabled=True,\n ),\n Task(\n message='preparing the instance',\n start_resource='Instance',\n finish_resource='MountingVolumesSignal',\n enabled=True,\n ),\n Task(\n message='mounting volumes',\n start_resource='MountingVolumesSignal',\n finish_resource='SettingDockerRootSignal',\n enabled=bool(instance_config.volumes),\n ),\n Task(\n message='setting Docker data root',\n start_resource='SettingDockerRootSignal',\n finish_resource='SyncingProjectSignal',\n enabled=bool(instance_config.docker_data_root),\n ),\n Task(\n message='syncing project files',\n start_resource='SyncingProjectSignal',\n finish_resource='RunningInstanceStartupCommandsSignal',\n enabled=True,\n ),\n Task(\n message='running instance startup commands',\n start_resource='RunningInstanceStartupCommandsSignal',\n finish_resource='BuildingDockerImageSignal',\n enabled=bool(instance_config.commands),\n ),\n Task(\n message='building Docker image',\n start_resource='BuildingDockerImageSignal',\n finish_resource='StartingContainerSignal',\n enabled=bool(instance_config.dockerfile_path),\n ),\n Task(\n message='starting container',\n start_resource='StartingContainerSignal',\n finish_resource='RunningContainerStartupCommandsSignal',\n enabled=True,\n ),\n Task(\n message='running container startup commands',\n start_resource='RunningContainerStartupCommandsSignal',\n finish_resource='DockerReadyWaitCondition',\n enabled=bool(instance_config.container_config.commands),\n ),\n ]\n\n # wait for the stack to be created\n with output.prefix(' '):\n stack.wait_tasks(tasks, resource_success_status='CREATE_COMPLETE', resource_fail_status='CREATE_FAILED',\n output=output)\n stack = stack.wait_status_changed(stack_waiting_status='CREATE_IN_PROGRESS', output=output)\n\n return stack\n\n def delete_stack(self, output: AbstractOutputWriter, no_wait=False):\n stack = Stack.get_by_name(self._cf, self._stack_name)\n if not stack:\n return\n\n if not no_wait:\n output.write('Waiting for the stack to be deleted...')\n\n # delete the stack\n try:\n stack.delete()\n if not no_wait:\n stack.wait_stack_deleted()\n except Exception as e:\n raise ValueError('Stack \"%s\" was not deleted. Error: %s\\n'\n 'See CloudFormation logs for details.' % (self._stack_name, str(e)))\n" }, { "path": "spotty/providers/aws/resource_managers/key_pair_manager.py", "content": "import os\nfrom spotty.configuration import get_spotty_keys_dir\nfrom spotty.providers.instance_manager_factory import PROVIDER_AWS\n\n\nclass KeyPairManager(object):\n\n def __init__(self, ec2, project_name: str, region: str):\n self._ec2 = ec2\n self._key_name = 'spotty-key-%s-%s' % (project_name.lower(), region)\n self._key_path = os.path.join(get_spotty_keys_dir(PROVIDER_AWS), self._key_name)\n\n @property\n def key_name(self):\n return self._key_name\n\n @property\n def key_path(self):\n return self._key_path\n\n def maybe_create_key(self):\n\n key_file_exists = os.path.isfile(self.key_path)\n ec2_key_exists = self._ec2_key_exists()\n\n if not ec2_key_exists or not key_file_exists:\n # remove key from AWS (key file not found)\n if ec2_key_exists:\n self._ec2.delete_key_pair(KeyName=self._key_name)\n\n # remove the key file (in case it was the old path)\n if key_file_exists:\n os.unlink(self.key_path)\n\n # create new key\n res = self._ec2.create_key_pair(KeyName=self._key_name)\n\n # create a provider subdirectory\n keys_dir = os.path.dirname(self.key_path)\n if not os.path.isdir(keys_dir):\n os.makedirs(keys_dir, mode=0o755, exist_ok=True)\n\n # save the key to the new path\n with open(self.key_path, 'w') as f:\n f.write(res['KeyMaterial'])\n\n os.chmod(self.key_path, 0o600)\n\n def delete_key(self):\n # delete EC2 Key Pair\n if self._ec2_key_exists():\n self._ec2.delete_key_pair(KeyName=self._key_name)\n\n # delete the key file\n if os.path.isfile(self.key_path):\n os.unlink(self.key_path)\n\n def _ec2_key_exists(self):\n res = self._ec2.describe_key_pairs(Filters=[{'Name': 'key-name', 'Values': [self._key_name]}])\n if 'KeyPairs' not in res:\n return False\n\n if len(res['KeyPairs']) > 1:\n raise ValueError('Several keys with the name \"%s\" found.' % self._key_name)\n\n return bool(res['KeyPairs'])\n" }, { "path": "spotty/providers/aws/resources/__init__.py", "content": "" }, { "path": "spotty/providers/aws/resources/bucket.py", "content": "from spotty.deployment.abstract_cloud_instance.resources.abstract_bucket import AbstractBucket\n\n\nclass Bucket(AbstractBucket):\n\n def __init__(self, data: dict):\n self._data = data\n\n @property\n def name(self) -> str:\n return self._data['Name']\n" }, { "path": "spotty/providers/aws/resources/image.py", "content": "class Image(object):\n\n def __init__(self, ec2, ami_info):\n self._ec2 = ec2\n self._ami_info = ami_info\n\n @staticmethod\n def get_by_name(ec2, ami_name: str):\n \"\"\"Returns a AMI by its name.\"\"\"\n res = ec2.describe_images(Owners=['self'], Filters=[\n {'Name': 'name', 'Values': [ami_name]},\n ])\n\n if len(res['Images']) > 1:\n raise ValueError('Several AMIs use the same name: \"%s\".' % ami_name)\n\n if not len(res['Images']):\n return None\n\n return Image(ec2, res['Images'][0])\n\n @staticmethod\n def get_by_id(ec2, ami_id: str):\n \"\"\"Returns a AMI by its ID.\"\"\"\n res = ec2.describe_images(Filters=[{'Name': 'image-id', 'Values': [ami_id]}])\n\n if not len(res['Images']):\n return None\n\n return Image(ec2, res['Images'][0])\n\n @property\n def image_id(self) -> str:\n return self._ami_info['ImageId']\n\n @property\n def name(self) -> str:\n return self._ami_info['Name']\n\n @property\n def size(self) -> int:\n return self._ami_info['BlockDeviceMappings'][0]['Ebs']['VolumeSize']\n\n def get_tag_value(self, tag_name):\n tag_values = [tag['Value'] for tag in self._ami_info['Tags'] if tag['Key'] == tag_name]\n if not tag_values:\n return None\n\n return tag_values[0]\n" }, { "path": "spotty/providers/aws/resources/instance.py", "content": "from datetime import datetime\nfrom spotty.deployment.abstract_cloud_instance.resources.abstract_instance import AbstractInstance\nfrom spotty.providers.aws.helpers.instance_prices import get_current_spot_price, get_on_demand_price\n\n\nclass Instance(AbstractInstance):\n\n def __init__(self, ec2, data: dict):\n self._ec2 = ec2\n self._data = data\n\n @staticmethod\n def get_by_stack_name(ec2, stack_name):\n \"\"\"Returns the running instance by its stack name\n or None if the instance is not running.\n \"\"\"\n res = ec2.describe_instances(Filters=[\n {'Name': 'tag:aws:cloudformation:stack-name', 'Values': [stack_name]},\n {'Name': 'instance-state-name', 'Values': ['running']},\n ])\n\n if len(res['Reservations']) > 1:\n raise ValueError('Several running instances for the stack \"%s\" are found.' % stack_name)\n\n if not len(res['Reservations']):\n return None\n\n return Instance(ec2, res['Reservations'][0]['Instances'][0])\n\n @property\n def instance_id(self):\n return self._data['InstanceId']\n\n @property\n def public_ip_address(self) -> str:\n return self._data.get('PublicIpAddress', None)\n\n @property\n def private_ip_address(self) -> str:\n return self._data.get('PrivateIpAddress', None)\n\n @property\n def state(self) -> str:\n return self._data['State']['Name']\n\n @property\n def instance_type(self) -> str:\n return self._data['InstanceType']\n\n @property\n def availability_zone(self) -> str:\n return self._data['Placement']['AvailabilityZone']\n\n @property\n def launch_time(self) -> datetime:\n return self._data['LaunchTime']\n\n @property\n def lifecycle(self) -> str:\n return self._data.get('InstanceLifecycle')\n\n @property\n def is_running(self):\n return self.state == 'running'\n\n @property\n def is_stopped(self):\n return self.state == 'stopped'\n\n def get_spot_price(self):\n \"\"\"Get current Spot Instance price for this instance.\"\"\"\n return get_current_spot_price(self._ec2, self.instance_type, self.availability_zone)\n\n def get_on_demand_price(self):\n \"\"\"Get On-demand Instance price for the same instance in the us-east-1 region.\"\"\"\n return get_on_demand_price(self.instance_type, 'us-east-1')\n\n def terminate(self, wait: bool = True):\n self._ec2.terminate_instances(InstanceIds=[self.instance_id])\n if wait:\n waiter = self._ec2.get_waiter('instance_terminated')\n waiter.wait(InstanceIds=[self.instance_id])\n\n def stop(self, wait: bool = True):\n self._ec2.stop_instances(InstanceIds=[self.instance_id])\n if wait:\n waiter = self._ec2.get_waiter('instance_stopped')\n waiter.wait(InstanceIds=[self.instance_id])\n" }, { "path": "spotty/providers/aws/resources/snapshot.py", "content": "import time\n\n\nclass Snapshot(object):\n\n def __init__(self, ec2, snapshot_info):\n self._ec2 = ec2\n self._snapshot_info = snapshot_info\n\n @staticmethod\n def get_by_name(ec2, snapshot_name: str):\n \"\"\"Returns a snapshot by its name.\"\"\"\n res = ec2.describe_snapshots(Filters=[\n {'Name': 'tag:Name', 'Values': [snapshot_name]},\n ])\n\n if len(res['Snapshots']) > 1:\n raise ValueError('Several snapshots with Name=%s found.' % snapshot_name)\n\n if not len(res['Snapshots']):\n return None\n\n return Snapshot(ec2, res['Snapshots'][0])\n\n @property\n def name(self) -> str:\n snapshot_name = [tag['Value'] for tag in self._snapshot_info['Tags'] if tag['Key'] == 'Name']\n if not snapshot_name:\n return ''\n\n return snapshot_name[0]\n\n @property\n def snapshot_id(self):\n return self._snapshot_info['SnapshotId']\n\n @property\n def size(self) -> int:\n return self._snapshot_info['VolumeSize']\n\n @property\n def creation_time(self) -> int:\n return int(time.mktime(self._snapshot_info['StartTime'].timetuple()))\n\n def rename(self, new_name):\n return self._ec2.create_tags(Resources=[self.snapshot_id],\n Tags=[{'Key': 'Name', 'Value': new_name}])\n\n def delete(self):\n return self._ec2.delete_snapshot(SnapshotId=self.snapshot_id)\n\n def wait_snapshot_completed(self):\n waiter = self._ec2.get_waiter('snapshot_completed')\n waiter.wait(SnapshotIds=[self.snapshot_id])\n" }, { "path": "spotty/providers/aws/resources/stack.py", "content": "from collections import namedtuple\nfrom time import sleep\nfrom typing import List, Dict\nfrom botocore.exceptions import EndpointConnectionError, ClientError\nfrom spotty.commands.writers.abstract_output_writrer import AbstractOutputWriter\nimport logging\n\n\nTask = namedtuple('Task', ['message', 'start_resource', 'finish_resource', 'enabled'])\n\n\nclass Stack(object):\n\n def __init__(self, cf, stack_info):\n self._cf = cf\n self._stack_info = stack_info\n\n @staticmethod\n def get_by_name(cf, stack_name: str):\n \"\"\"Returns a Stack by its name.\"\"\"\n try:\n res = cf.describe_stacks(StackName=stack_name)\n except ClientError as e:\n # ignore an exception if it raised because the stack doesn't exist\n error_code = e.response.get('Error', {}).get('Code')\n if error_code != 'ValidationError':\n raise e\n\n res = {'Stacks': []}\n\n if not len(res['Stacks']):\n return None\n\n return Stack(cf, res['Stacks'][0])\n\n @staticmethod\n def create_stack(cf, *args, **kwargs):\n res = cf.create_stack(*args, **kwargs)\n return Stack(cf, res)\n\n @staticmethod\n def update_stack(cf, *args, **kwargs):\n res = cf.update_stack(*args, **kwargs)\n return Stack(cf, res)\n\n @property\n def stack_id(self) -> str:\n return self._stack_info['StackId']\n\n @property\n def stack_uuid(self) -> str:\n return self.stack_id.rsplit('/', 1)[-1]\n\n @property\n def name(self) -> str:\n return self._stack_info['StackName']\n\n @property\n def status(self) -> str:\n return self._stack_info['StackStatus']\n\n @property\n def outputs(self) -> str:\n return self._stack_info['Outputs']\n\n def delete(self):\n return self._cf.delete_stack(StackName=self.stack_id)\n\n def wait_stack_created(self, delay_secs: int = 30):\n waiter = self._cf.get_waiter('stack_create_complete')\n waiter.wait(StackName=self.stack_id, WaiterConfig={'Delay': delay_secs})\n\n def wait_stack_updated(self, delay_secs: int = 30):\n waiter = self._cf.get_waiter('stack_update_complete')\n waiter.wait(StackName=self.stack_id, WaiterConfig={'Delay': delay_secs})\n\n def wait_stack_deleted(self, delay_secs: int = 30):\n waiter = self._cf.get_waiter('stack_delete_complete')\n waiter.wait(StackName=self.stack_id, WaiterConfig={'Delay': delay_secs})\n\n def wait_status_changed(self, stack_waiting_status: str, output: AbstractOutputWriter, delay_secs: int = 5):\n stack = None\n while True:\n # get the latest status of the stack\n try:\n stack = self.get_by_name(self._cf, self.stack_id)\n except EndpointConnectionError as e:\n output.write(str(e))\n continue\n\n if stack.status != stack_waiting_status:\n break\n\n sleep(delay_secs)\n\n return stack\n\n def wait_tasks(self, tasks: List[Task], resource_success_status: str, resource_fail_status: str,\n output: AbstractOutputWriter, delay_secs: int = 5):\n resource_statuses = self._get_resource_statuses()\n\n for task in tasks:\n if not task.enabled:\n continue\n\n task_started = task_finished = False\n while not task_finished:\n start_status = resource_statuses.get(task.start_resource)\n finish_status = resource_statuses.get(task.finish_resource)\n\n if not task_started and (not task.start_resource or (start_status == resource_success_status)):\n task_started = True\n output.write('- %s... ' % task.message, newline=False)\n elif task_started and (finish_status == resource_success_status):\n task_finished = True\n output.write('DONE')\n else:\n sleep(delay_secs)\n resource_statuses = self._get_resource_statuses()\n\n # check that the stack is not failed\n for status in resource_statuses.values():\n if status == resource_fail_status:\n if task_started and not task_finished:\n output.write('')\n return\n\n def _get_resource_statuses(self) -> Dict[str, str]:\n stack_resources = None\n try:\n stack_resources = self._cf.list_stack_resources(StackName=self.stack_id)\n except Exception as e:\n logging.warning(str(e))\n\n resource_statuses = {}\n if stack_resources:\n resource_statuses = {row['LogicalResourceId']: row['ResourceStatus']\n for row in stack_resources['StackResourceSummaries']}\n\n return resource_statuses\n" }, { "path": "spotty/providers/aws/resources/subnet.py", "content": "class Subnet(object):\n\n def __init__(self, ec2, subnet_info):\n self._ec2 = ec2\n self._subnet_info = subnet_info\n\n @staticmethod\n def get_by_id(ec2, subnet_id: str):\n \"\"\"Returns a subnet by its ID.\"\"\"\n res = ec2.describe_subnets(Filters=[\n {'Name': 'subnet-id', 'Values': [subnet_id]},\n ])\n\n if not len(res['Subnets']):\n return None\n\n return Subnet(ec2, res['Subnets'][0])\n\n @staticmethod\n def get_default_subnets(ec2):\n res = ec2.describe_subnets(Filters=[\n {'Name': 'defaultForAz', 'Values': ['true']},\n ])\n\n subnets = [Subnet(ec2, subnet_info) for subnet_info in res['Subnets']]\n\n return subnets\n\n @property\n def availability_zone(self) -> str:\n return self._subnet_info['AvailabilityZone']\n\n @property\n def vpc_id(self) -> str:\n return self._subnet_info['VpcId']\n" }, { "path": "spotty/providers/aws/resources/volume.py", "content": "from spotty.providers.aws.resources.snapshot import Snapshot\n\n\nclass Volume(object):\n\n def __init__(self, ec2, volume_info):\n self._ec2 = ec2\n self._volume_info = volume_info\n\n @staticmethod\n def get_by_name(ec2, volume_name: str):\n \"\"\"Returns a volume by its name.\"\"\"\n res = ec2.describe_volumes(Filters=[\n {'Name': 'tag:Name', 'Values': [volume_name]},\n ])\n\n if len(res['Volumes']) > 1:\n raise ValueError('Several volumes with Name=%s found.' % volume_name)\n\n if not len(res['Volumes']):\n return None\n\n return Volume(ec2, res['Volumes'][0])\n\n @property\n def name(self) -> str:\n volume_name = [tag['Value'] for tag in self._volume_info['Tags'] if tag['Key'] == 'Name']\n if not volume_name:\n return ''\n\n return volume_name[0]\n\n @property\n def volume_id(self) -> str:\n return self._volume_info['VolumeId']\n\n @property\n def size(self) -> int:\n return self._volume_info['Size']\n\n @property\n def availability_zone(self) -> str:\n return self._volume_info['AvailabilityZone']\n\n @property\n def state(self) -> str:\n return self._volume_info['State']\n\n def is_available(self):\n return self.state == 'available'\n\n def create_snapshot(self) -> Snapshot:\n snapshot_info = self._ec2.create_snapshot(\n VolumeId=self._volume_info['VolumeId'],\n TagSpecifications=[{\n 'ResourceType': 'snapshot',\n 'Tags': [{\n 'Key': 'Name',\n 'Value': self.name,\n }],\n }],\n )\n\n return Snapshot(self._ec2, snapshot_info)\n\n def delete(self):\n return self._ec2.delete_volume(VolumeId=self._volume_info['VolumeId'])\n" }, { "path": "spotty/providers/aws/resources/vpc.py", "content": "class Vpc(object):\n\n def __init__(self, ec2, vpc_info):\n self._ec2 = ec2\n self._vpc_info = vpc_info\n\n @staticmethod\n def get_default_vpc(ec2):\n \"\"\"Returns a default VPC.\"\"\"\n res = ec2.describe_vpcs(Filters=[{'Name': 'isDefault', 'Values': ['true']}])\n if not len(res['Vpcs']):\n return None\n\n return Vpc(ec2, res['Vpcs'][0])\n\n @property\n def vpc_id(self) -> str:\n return self._vpc_info['VpcId']\n" }, { "path": "spotty/providers/gcp/__init__.py", "content": "" }, { "path": "spotty/providers/gcp/config/__init__.py", "content": "" }, { "path": "spotty/providers/gcp/config/disk_volume.py", "content": "from spotty.config.abstract_instance_volume import AbstractInstanceVolume\nfrom spotty.providers.gcp.config.validation import validate_disk_volume_parameters\n\n\nclass DiskVolume(AbstractInstanceVolume):\n\n TYPE_NAME = 'Disk'\n\n DP_CREATE_SNAPSHOT = 'CreateSnapshot'\n DP_UPDATE_SNAPSHOT = 'UpdateSnapshot'\n DP_RETAIN = 'Retain'\n DP_DELETE = 'Delete'\n\n def __init__(self, volume_config: dict, project_name: str, instance_name: str):\n super().__init__(volume_config)\n\n self._project_name = project_name\n self._instance_name = instance_name\n\n def _validate_volume_parameters(self, params: dict) -> dict:\n return validate_disk_volume_parameters(params)\n\n @property\n def title(self):\n return 'Disk'\n\n @property\n def size(self) -> int:\n return self._params['size']\n\n @property\n def deletion_policy(self) -> str:\n return self._params['deletionPolicy']\n\n @property\n def deletion_policy_title(self) -> str:\n return {\n DiskVolume.DP_CREATE_SNAPSHOT: 'Create Snapshot',\n DiskVolume.DP_UPDATE_SNAPSHOT: 'Update Snapshot',\n DiskVolume.DP_RETAIN: 'Retain Volume',\n DiskVolume.DP_DELETE: 'Delete Volume',\n }[self.deletion_policy]\n\n @property\n def disk_name(self) -> str:\n \"\"\"Returns the disk name.\"\"\"\n disk_name = self._params['diskName']\n if not disk_name:\n disk_name = '%s-%s-%s' % (self._project_name.lower(), self._instance_name.lower(), self.name.lower())\n\n return disk_name\n\n @property\n def mount_dir(self) -> str:\n \"\"\"A directory where the volume will be mounted on the host OS.\"\"\"\n if self._params['mountDir']:\n mount_dir = self._params['mountDir']\n else:\n mount_dir = '/mnt/%s' % self.disk_name\n\n return mount_dir\n\n @property\n def host_path(self) -> str:\n \"\"\"A path on the host OS that will be mounted to the container.\"\"\"\n return self.mount_dir\n" }, { "path": "spotty/providers/gcp/config/image_uri.py", "content": "import re\n\n\nIMAGE_URI_REGEX = '^(?:(?:https://compute.googleapis.com/compute/v1/)?projects/([a-z](?:[-a-z0-9]*[a-z0-9])?)/)?' \\\n 'global/images/(family/)?([a-z](?:[-a-z0-9]*[a-z0-9])?)$'\n\n\nclass ImageUri(object):\n\n def __init__(self, image_uri: str):\n res = re.match(IMAGE_URI_REGEX, image_uri)\n if not res:\n raise ValueError('Image URI has a wrong format')\n\n self._project_id, self._is_family, self._name = res.groups()\n\n @property\n def project_id(self) -> str:\n return self._project_id\n\n @property\n def is_family(self):\n return bool(self._is_family)\n\n @property\n def name(self):\n \"\"\"Image name or image family name.\"\"\"\n return self._name\n" }, { "path": "spotty/providers/gcp/config/instance_config.py", "content": "from typing import List\nfrom spotty.config.abstract_instance_config import AbstractInstanceConfig, VolumeMount\nfrom spotty.config.abstract_instance_volume import AbstractInstanceVolume\nfrom spotty.providers.gcp.config.disk_volume import DiskVolume\nfrom spotty.providers.gcp.config.validation import validate_instance_parameters\n\n\nVOLUME_TYPE_DISK = 'Disk'\nDEFAULT_IMAGE_NAME = 'spotty'\n\n\nclass InstanceConfig(AbstractInstanceConfig):\n\n def _validate_instance_params(self, params: dict) -> dict:\n return validate_instance_parameters(params)\n\n def _get_instance_volumes(self) -> List[AbstractInstanceVolume]:\n volumes = []\n for volume_config in self._params['volumes']:\n volume_type = volume_config['type']\n if volume_type == DiskVolume.TYPE_NAME:\n volumes.append(DiskVolume(volume_config, self.project_config.project_name, self.name))\n else:\n raise ValueError('GCP volume type \"%s\" not supported.' % volume_type)\n\n return volumes\n\n @property\n def user(self):\n return 'spotty'\n\n @property\n def machine_name(self) -> str:\n \"\"\"Name of the Compute Engine instance.\"\"\"\n return '%s-%s' % (self.project_config.project_name.lower(), self.name.lower())\n\n @property\n def project_id(self) -> str:\n return self._params['projectId']\n\n @property\n def zone(self) -> str:\n return self._params['zone']\n\n @property\n def machine_type(self) -> str:\n return self._params['machineType']\n\n @property\n def gpu(self) -> dict:\n return self._params['gpu']\n\n @property\n def is_preemptible_instance(self) -> bool:\n return self._params['preemptibleInstance']\n\n @property\n def boot_disk_size(self) -> int:\n return self._params['bootDiskSize']\n\n @property\n def ports(self) -> List[int]:\n return list(set(self._params['ports']))\n\n @property\n def image_name(self) -> str:\n return self._params['imageName']\n\n @property\n def has_image_name(self) -> bool:\n return bool(self._params['imageName'])\n\n @property\n def image_uri(self) -> str:\n return self._params['imageUri']\n" }, { "path": "spotty/providers/gcp/config/validation.py", "content": "import os\nfrom schema import Schema, Optional, And, Regex, Or, Use\nfrom spotty.config.validation import validate_config, get_instance_parameters_schema, has_prefix\nfrom spotty.providers.gcp.config.image_uri import IMAGE_URI_REGEX\n\n\ndef validate_instance_parameters(params: dict):\n from spotty.providers.gcp.config.disk_volume import DiskVolume\n\n instance_parameters = {\n 'zone': And(str, Regex(r'^[a-z0-9-]+$')),\n 'machineType': str,\n Optional('gpu', default=None): {\n 'type': str,\n Optional('count', default=1): int,\n },\n Optional('preemptibleInstance', default=False): bool,\n Optional('imageName', default=None): And(str, len, Regex(r'^[\\w-]+$')),\n Optional('imageUri', default=None): And(str, len, Regex(IMAGE_URI_REGEX)),\n Optional('bootDiskSize', default=0): And(Or(int, str), Use(str),\n Regex(r'^\\d+$', error='Incorrect value for \"bootDiskSize\".'),\n Use(int),\n And(lambda x: x > 0,\n error='\"rootVolumeSize\" should be greater than 0 or should '\n 'not be specified.'),\n ),\n Optional('ports', default=[]): [And(int, lambda x: 0 < x < 65536)],\n }\n\n instance_checks = [\n And(lambda x: not (x['imageName'] and x['imageUri']),\n error='\"imageName\" and \"imageUri\" parameters cannot be used together.'),\n ]\n\n volume_checks = [\n And(lambda x: not has_prefix([(volume['parameters']['mountDir'] + '/') for volume in x\n if volume['parameters'].get('mountDir')]),\n error='Mount directories cannot be prefixes for each other.'),\n ]\n\n schema = get_instance_parameters_schema(instance_parameters, DiskVolume.TYPE_NAME, instance_checks, volume_checks)\n\n return validate_config(schema, params)\n\n\ndef validate_disk_volume_parameters(params: dict):\n from spotty.providers.gcp.config.disk_volume import DiskVolume\n\n schema = Schema({\n Optional('diskName', default=''): And(str, Regex(r'^[\\w-]{1,255}$')),\n Optional('mountDir', default=''): And(\n str,\n And(os.path.isabs, error='Use absolute paths in the \"mountDir\" parameters'),\n Use(lambda x: x.rstrip('/'))\n ),\n Optional('size', default=0): And(int, lambda x: x > 0),\n Optional('deletionPolicy', default=DiskVolume.DP_RETAIN): And(\n str,\n lambda x: x in [DiskVolume.DP_CREATE_SNAPSHOT,\n DiskVolume.DP_UPDATE_SNAPSHOT,\n DiskVolume.DP_RETAIN,\n DiskVolume.DP_DELETE], error='Incorrect value for \"deletionPolicy\".'\n ),\n })\n\n return validate_config(schema, params)\n" }, { "path": "spotty/providers/gcp/data_transfer.py", "content": "import logging\nimport subprocess\nfrom spotty.deployment.abstract_cloud_instance.abstract_data_transfer import AbstractDataTransfer\nfrom spotty.providers.gcp.helpers.gsutil_rsync import check_gsutil_installed, get_rsync_command\n\n\nclass DataTransfer(AbstractDataTransfer):\n\n @property\n def scheme_name(self) -> str:\n return 'gs'\n\n def upload_local_to_bucket(self, bucket_name: str, dry_run: bool = False):\n \"\"\"Uploads files from local to the bucket.\"\"\"\n # check gsutil is installed\n check_gsutil_installed()\n\n # sync the project with S3, deleted files will be deleted from S3\n local_cmd = get_rsync_command(self._local_project_dir, self._get_bucket_project_path(bucket_name),\n filters=self._sync_filters, delete=True, dry_run=dry_run)\n\n # execute the command locally\n logging.debug('Local sync command: ' + local_cmd)\n exit_code = subprocess.call(local_cmd, shell=True)\n if exit_code != 0:\n raise ValueError('Failed to upload the project files to the GS bucket.')\n\n def download_bucket_to_local(self, bucket_name: str, download_filters: list):\n \"\"\"Downloads files from the bucket to local.\"\"\"\n raise NotImplementedError\n\n def get_download_bucket_to_instance_command(self, bucket_name: str, use_sudo: bool = False) -> str:\n \"\"\"A remote command to download files from the bucket to the instance.\"\"\"\n remote_cmd = get_rsync_command(self._get_bucket_project_path(bucket_name), self._host_project_dir,\n filters=self._sync_filters)\n if use_sudo:\n remote_cmd = 'sudo ' + remote_cmd\n\n return remote_cmd\n\n def get_upload_instance_to_bucket_command(self, bucket_name: str, download_filters: list, use_sudo: bool = False,\n dry_run: bool = False) -> str:\n \"\"\"A remote command to upload files from the instance to the bucket.\n\n It uses a temporary directory on the bucket that is unique for the instance. This\n directory keeps all downloaded from the instance files to sync only changed\n files with local.\n \"\"\"\n raise NotImplementedError\n" }, { "path": "spotty/providers/gcp/dm_templates/__init__.py", "content": "" }, { "path": "spotty/providers/gcp/dm_templates/instance/__init__.py", "content": "" }, { "path": "spotty/providers/gcp/dm_templates/instance/data/startup_script.sh.tpl", "content": "#!/usr/bin/env bash\n\nset -x\n\nmkdir -p \"{{INSTANCE_STARTUP_SCRIPTS_DIR}}\"\n\n# create startup scripts\n{{#STARTUP_SCRIPTS}}\ncat <<'EOF' > {{INSTANCE_STARTUP_SCRIPTS_DIR}}/{{filename}}\n{{{content}}}\nEOF\nchmod +x {{INSTANCE_STARTUP_SCRIPTS_DIR}}/{{filename}}\n\n{{/STARTUP_SCRIPTS}}\n\n# run startup scripts\n{{#STARTUP_SCRIPTS}}\n{{INSTANCE_STARTUP_SCRIPTS_DIR}}/{{filename}} && \\\n{{/STARTUP_SCRIPTS}}\ntrue\n\n# send signal that the Docker container is ready or failed\nEXIT_CODE=$?\nif [ $EXIT_CODE -eq 0 ]; then\n gcloud beta runtime-config configs variables set /success/1 1 --config-name {{MACHINE_NAME}}-docker-status --is-text\nelse\n gcloud beta runtime-config configs variables set /failure/1 1 --config-name {{MACHINE_NAME}}-docker-status --is-text\n exit $EXIT_CODE\nfi\n" }, { "path": "spotty/providers/gcp/dm_templates/instance/data/startup_scripts/01_prepare_instance.sh", "content": "#!/bin/bash -xe\n\n# install jq\napt-get install -y jq\n\n# create tmux config\necho \"bind-key x kill-pane\" > /home/{{SSH_USERNAME}}/.tmux.conf\n\n# create the \"container bash\" script\nmkdir -p \"$(dirname '{{CONTAINER_BASH_SCRIPT_PATH}}')\"\ncat > \"{{CONTAINER_BASH_SCRIPT_PATH}}\" <<'EOF2'\n{{{CONTAINER_BASH_SCRIPT}}}\nEOF2\nchmod +x \"{{CONTAINER_BASH_SCRIPT_PATH}}\"\n\n# create an alias to connect to the docker container\nCONTAINER_BASH_ALIAS=container\necho \"alias $CONTAINER_BASH_ALIAS=\\\"{{CONTAINER_BASH_SCRIPT_PATH}}\\\"\" >> /home/{{SSH_USERNAME}}/.bashrc\necho \"alias $CONTAINER_BASH_ALIAS=\\\"{{CONTAINER_BASH_SCRIPT_PATH}}\\\"\" >> /root/.bashrc\n\n{{#IS_GPU_INSTANCE}}\n# install NVIDIA driver\nif ! command -v nvidia-smi &> /dev/null; then\n DRIVER_INSTALLER_PATH=/opt/deeplearning/install-driver.sh\n if [ -f \"$DRIVER_INSTALLER_PATH\" ]; then\n $DRIVER_INSTALLER_PATH\n fi\nfi\n{{/IS_GPU_INSTANCE}}\n\n# create common temporary directories\nmkdir -pm 777 '{{SPOTTY_TMP_DIR}}'\nmkdir -pm 777 '{{CONTAINERS_TMP_DIR}}'\n" }, { "path": "spotty/providers/gcp/dm_templates/instance/data/startup_scripts/02_mount_volumes.sh", "content": "#!/bin/bash -xe\n\nDEVICE_NAMES=({{{DISK_DEVICE_NAMES}}})\nMOUNT_DIRS=({{{DISK_MOUNT_DIRS}}})\n\nfor i in ${!DEVICE_NAMES[*]}\ndo\n DEVICE=/dev/disk/by-id/google-${DEVICE_NAMES[$i]}\n MOUNT_DIR=${MOUNT_DIRS[$i]}\n\n blkid -o value -s TYPE $DEVICE || mkfs -t ext4 $DEVICE\n mkdir -p $MOUNT_DIR\n mount $DEVICE $MOUNT_DIR\n chmod 777 $MOUNT_DIR\n resize2fs $DEVICE\ndone\n\n# create directories for temporary container volumes\n{{#TMP_VOLUME_DIRS}}\nmkdir -p {{PATH}}\nchmod 777 {{PATH}}\n{{/TMP_VOLUME_DIRS}}\n" }, { "path": "spotty/providers/gcp/dm_templates/instance/data/startup_scripts/03_set_docker_root.sh", "content": "#!/bin/bash -xe\n\n# change docker data root directory\nif [ -n \"{{DOCKER_DATA_ROOT_DIR}}\" ]; then\n jq '. + { \"data-root\": \"{{DOCKER_DATA_ROOT_DIR}}\" }' /etc/docker/daemon.json > /tmp/docker_daemon.json \\\n && mv /tmp/docker_daemon.json /etc/docker/daemon.json\n service docker restart\nfi\n" }, { "path": "spotty/providers/gcp/dm_templates/instance/data/startup_scripts/04_sync_project.sh", "content": "#!/bin/bash -xe\n\n# create a project directory\nif [ -n \"{{HOST_PROJECT_DIR}}\" ]; then\n mkdir -p \"{{HOST_PROJECT_DIR}}\"\n chmod 777 \"{{HOST_PROJECT_DIR}}\"\nfi\n\n{{{SYNC_PROJECT_CMD}}}\n" }, { "path": "spotty/providers/gcp/dm_templates/instance/data/startup_scripts/05_run_instance_startup_commands.sh", "content": "#!/bin/bash -xe\n\nmkdir -p \"{{INSTANCE_STARTUP_SCRIPTS_DIR}}\"\ncat > \"{{INSTANCE_STARTUP_SCRIPTS_DIR}}/instance_startup_commands.sh\" <<'EOF2'\n{{{INSTANCE_STARTUP_COMMANDS}}}\nEOF2\n\n/bin/bash -xe \"{{INSTANCE_STARTUP_SCRIPTS_DIR}}/instance_startup_commands.sh\"\n" }, { "path": "spotty/providers/gcp/dm_templates/instance/data/template.yaml", "content": "resources:\n - name: {{MACHINE_NAME}}\n type: compute.v1.instance\n properties:\n zone: {{ZONE}}\n machineType: zones/{{ZONE}}/machineTypes/{{MACHINE_TYPE}}\n scheduling:\n {{#GPU_TYPE}}\n onHostMaintenance: TERMINATE\n automaticRestart: false\n {{/GPU_TYPE}}\n preemptible: {{PREEMPTIBLE}}\n serviceAccounts:\n - email: {{SERVICE_ACCOUNT_EMAIL}}\n scopes: ['https://www.googleapis.com/auth/cloud-platform']\n tags:\n items:\n - {{MACHINE_NAME}}\n disks:\n - deviceName: boot\n type: PERSISTENT\n boot: true\n autoDelete: true\n initializeParams:\n sourceImage: {{SOURCE_IMAGE}}\n {{#BOOT_DISK_SIZE}}\n diskSizeGb: {{BOOT_DISK_SIZE}}\n {{/BOOT_DISK_SIZE}}\n\n {{#DISK_ATTACHMENTS}}\n - source: {{DISK_LINK}}\n deviceName: {{DEVICE_NAME}}\n type: PERSISTENT\n mode: READ_WRITE\n boot: false\n autoDelete: false\n {{/DISK_ATTACHMENTS}}\n networkInterfaces:\n - network: global/networks/default\n accessConfigs:\n - name: External NAT\n type: ONE_TO_ONE_NAT\n {{#GPU_TYPE}}\n guestAccelerators:\n - acceleratorType: zones/{{ZONE}}/acceleratorTypes/{{GPU_TYPE}}\n acceleratorCount: {{GPU_COUNT}}\n {{/GPU_TYPE}}\n metadata:\n items:\n - key: 'ssh-keys'\n value: |\n {{SSH_USERNAME}}:ssh-rsa {{{PUB_KEY_VALUE}}} {{SSH_USERNAME}}\n - key: 'startup-script'\n value: |\n {{> STARTUP_SCRIPT}}\n\n - name: {{MACHINE_NAME}}-firewall-rule\n type: compute.v1.firewall\n properties:\n network: global/networks/default\n sourceRanges:\n - 0.0.0.0/0\n targetTags:\n - {{MACHINE_NAME}}\n allowed:\n - IPProtocol: tcp\n ports: [{{PORTS}}]\n\n - name: {{MACHINE_NAME}}-docker-status\n type: runtimeconfig.v1beta1.config\n properties:\n config: {{MACHINE_NAME}}-docker-status\n description: Docker status\n\n - name: {{MACHINE_NAME}}-docker-waiter\n type: runtimeconfig.v1beta1.waiter\n metadata:\n dependsOn:\n - {{MACHINE_NAME}}\n properties:\n parent: $(ref.{{MACHINE_NAME}}-docker-status.name)\n waiter: {{MACHINE_NAME}}-docker-waiter\n timeout: 1800s\n success:\n cardinality:\n path: /success\n number: 1\n failure:\n cardinality:\n path: /failure\n number: 1\n" }, { "path": "spotty/providers/gcp/dm_templates/instance/instance_template.py", "content": "import os\nfrom typing import List\nimport chevron\nfrom spotty.commands.writers.abstract_output_writrer import AbstractOutputWriter\nfrom spotty.config.tmp_dir_volume import TmpDirVolume\nfrom spotty.config.validation import is_subdir\nfrom spotty.config.abstract_instance_volume import AbstractInstanceVolume\nfrom spotty.deployment.container.docker.docker_commands import DockerCommands\nfrom spotty.deployment.container.docker.scripts.container_bash_script import ContainerBashScript\nfrom spotty.deployment.container.docker.scripts.start_container_script import StartContainerScript\nfrom spotty.deployment.abstract_cloud_instance.file_structure import CONTAINER_BASH_SCRIPT_PATH, \\\n INSTANCE_STARTUP_SCRIPTS_DIR, CONTAINERS_TMP_DIR, INSTANCE_SPOTTY_TMP_DIR\nfrom spotty.providers.gcp.config.disk_volume import DiskVolume\nfrom spotty.providers.gcp.config.instance_config import InstanceConfig\n\n\ndef prepare_instance_template(instance_config: InstanceConfig, docker_commands: DockerCommands, image_link: str,\n bucket_name: str, sync_project_cmd: str, public_key_value: str,\n service_account_email: str, output: AbstractOutputWriter):\n \"\"\"Prepares deployment template to run an instance.\"\"\"\n\n # get disk attachments\n disk_attachments, disk_device_names, disk_mount_dirs = \\\n _get_disk_attachments(instance_config.volumes, instance_config.zone)\n\n # run sync command as a non-root user\n if instance_config.container_config.run_as_host_user:\n sync_project_cmd = 'sudo -u %s %s' % (instance_config.user, sync_project_cmd)\n\n startup_scripts_templates = [\n {\n 'filename': '01_prepare_instance.sh',\n 'params': {\n 'CONTAINER_BASH_SCRIPT_PATH': CONTAINER_BASH_SCRIPT_PATH,\n 'CONTAINER_BASH_SCRIPT': ContainerBashScript(docker_commands).render(),\n 'IS_GPU_INSTANCE': bool(instance_config.gpu),\n 'SSH_USERNAME': instance_config.user,\n 'SPOTTY_TMP_DIR': INSTANCE_SPOTTY_TMP_DIR,\n 'CONTAINERS_TMP_DIR': CONTAINERS_TMP_DIR,\n },\n },\n {\n 'filename': '02_mount_volumes.sh',\n 'params': {\n 'DISK_DEVICE_NAMES': ('\"%s\"' % '\" \"'.join(disk_device_names)) if disk_device_names else '',\n 'DISK_MOUNT_DIRS': ('\"%s\"' % '\" \"'.join(disk_mount_dirs)) if disk_mount_dirs else '',\n 'TMP_VOLUME_DIRS': [{'PATH': volume.host_path} for volume in instance_config.volumes\n if isinstance(volume, TmpDirVolume)],\n },\n },\n {\n 'filename': '03_set_docker_root.sh',\n 'params': {\n 'DOCKER_DATA_ROOT_DIR': instance_config.docker_data_root,\n },\n },\n {\n 'filename': '04_sync_project.sh',\n 'params': {\n 'HOST_PROJECT_DIR': instance_config.host_project_dir,\n 'SYNC_PROJECT_CMD': sync_project_cmd,\n },\n },\n {\n 'filename': '05_run_instance_startup_commands.sh',\n 'params': {\n 'INSTANCE_STARTUP_SCRIPTS_DIR': INSTANCE_STARTUP_SCRIPTS_DIR,\n 'INSTANCE_STARTUP_COMMANDS': instance_config.commands,\n },\n },\n ]\n\n # render startup scripts\n startup_scripts_content = []\n for template in startup_scripts_templates:\n with open(os.path.join(os.path.dirname(__file__), 'data', 'startup_scripts', template['filename'])) as f:\n content = f.read()\n\n startup_scripts_content.append({\n 'filename': template['filename'],\n 'content': chevron.render(content, template['params'])\n })\n\n startup_scripts_content.append({\n 'filename': '06_start_container.sh',\n 'content': StartContainerScript(docker_commands).render(print_trace=True),\n })\n\n # render the main startup script\n with open(os.path.join(os.path.dirname(__file__), 'data', 'startup_script.sh.tpl')) as f:\n startup_script = f.read()\n\n startup_script = chevron.render(startup_script, {\n 'MACHINE_NAME': instance_config.machine_name,\n 'INSTANCE_STARTUP_SCRIPTS_DIR': INSTANCE_STARTUP_SCRIPTS_DIR,\n 'STARTUP_SCRIPTS': startup_scripts_content,\n })\n\n # render the template\n with open(os.path.join(os.path.dirname(__file__), 'data', 'template.yaml')) as f:\n template = f.read()\n\n template = chevron.render(template, {\n 'SERVICE_ACCOUNT_EMAIL': service_account_email,\n 'ZONE': instance_config.zone,\n 'MACHINE_TYPE': instance_config.machine_type,\n 'SOURCE_IMAGE': image_link,\n 'BOOT_DISK_SIZE': instance_config.boot_disk_size,\n 'MACHINE_NAME': instance_config.machine_name,\n 'PREEMPTIBLE': 'true' if instance_config.is_preemptible_instance else 'false',\n 'GPU_TYPE': instance_config.gpu['type'] if instance_config.gpu else '',\n 'GPU_COUNT': instance_config.gpu['count'] if instance_config.gpu else 0,\n 'DISK_ATTACHMENTS': disk_attachments,\n 'SSH_USERNAME': instance_config.user,\n 'PUB_KEY_VALUE': public_key_value,\n 'PORTS': ', '.join([str(port) for port in set([22] + instance_config.ports)]),\n }, partials_dict={\n 'STARTUP_SCRIPT': startup_script,\n })\n\n # print some information about the deployment\n output.write('- image URL: ' + '/'.join(image_link.split('/')[-5:]))\n output.write('- zone: ' + instance_config.zone)\n output.write('- preemptible VM' if instance_config.is_preemptible_instance else '- on-demand VM')\n output.write(('- GPUs: %d x %s' % (instance_config.gpu['count'], instance_config.gpu['type']))\n if instance_config.gpu else '- no GPUs')\n\n # print name of the volume where Docker data will be stored\n if instance_config.docker_data_root:\n docker_data_volume_name = [volume.name for volume in instance_config.volumes\n if is_subdir(instance_config.docker_data_root, volume.host_path)][0]\n output.write('- Docker data will be stored on the \"%s\" volume' % docker_data_volume_name)\n\n return template\n\n\ndef _get_disk_attachments(volumes: List[AbstractInstanceVolume], zone: str):\n disk_attachments = []\n disk_device_names = []\n disk_mount_dirs = []\n\n for i, volume in enumerate(volumes):\n if isinstance(volume, DiskVolume):\n device_name = 'disk-%d' % (i + 1)\n disk_device_names.append(device_name)\n disk_mount_dirs.append(volume.mount_dir)\n disk_attachments.append({\n 'DISK_LINK': 'zones/%s/disks/%s' % (zone, volume.disk_name),\n 'DEVICE_NAME': device_name,\n })\n\n return disk_attachments, disk_device_names, disk_mount_dirs\n" }, { "path": "spotty/providers/gcp/errors/__init__.py", "content": "" }, { "path": "spotty/providers/gcp/errors/image_not_found.py", "content": "class ImageNotFoundError(Exception):\n def __init__(self, image_name):\n super().__init__('The image \"%s\" was not found.\\n'\n 'Use the \"spotty gcp create-image\" command to create an image with NVIDIA Docker.'\n % image_name)\n" }, { "path": "spotty/providers/gcp/helpers/__init__.py", "content": "" }, { "path": "spotty/providers/gcp/helpers/ce_client.py", "content": "from collections import OrderedDict\nfrom time import sleep\nimport googleapiclient.discovery\n\n\nclass CEClient(object):\n \"\"\"Compute Engine client.\"\"\"\n\n def __init__(self, project_id: str, zone: str):\n self._project_id = project_id\n self._zone = zone\n self._client = googleapiclient.discovery.build('compute', 'v1', cache_discovery=False)\n\n @property\n def zone(self):\n return self._zone\n\n def list_images(self, image_name: str = None, project_id: str = None):\n \"\"\"Returns a list of images that satisfy the name.\n This method is used instead of the \"get\" because it doesn't raise an exception if an image doesn't exist.\n \"\"\"\n if not project_id:\n project_id = self._project_id\n\n filter_str = ('name=%s' % image_name) if image_name else None\n res = self._client.images().list(project=project_id, filter=filter_str).execute()\n\n if not res.get('items'):\n return []\n\n return res['items']\n\n def get_image_from_family(self, family_name: str, project_id: str = None):\n if not project_id:\n project_id = self._project_id\n\n res = self._client.images().getFromFamily(project=project_id, family=family_name).execute()\n\n return res\n\n def list_instances(self, machine_name=None):\n filter_str = ('name=%s' % machine_name) if machine_name else None\n res = self._client.instances().list(project=self._project_id, zone=self._zone, filter=filter_str).execute()\n\n if not res.get('items'):\n return []\n\n return res['items']\n\n def list_disks(self, disk_name=None):\n filter_str = ('name=%s' % disk_name) if disk_name else None\n res = self._client.disks().list(project=self._project_id, zone=self._zone, filter=filter_str).execute()\n\n if not res.get('items'):\n return []\n\n return res['items']\n\n def list_snapshots(self, snapshot_name=None):\n filter_str = ('name=%s' % snapshot_name) if snapshot_name else None\n res = self._client.snapshots().list(project=self._project_id, filter=filter_str).execute()\n\n if not res.get('items'):\n return []\n\n return res['items']\n\n def get_accelerator_types(self) -> OrderedDict:\n res = self._client.acceleratorTypes().list(project=self._project_id, zone=self._zone).execute()\n accelerator_types = OrderedDict([(item['name'], item['maximumCardsPerInstance'])\n for item in res.get('items', [])])\n\n return accelerator_types\n\n def create_disk(self, name: str, size: int = None, snapshot_link: str = None) -> str:\n params = {\n 'name': name,\n 'type': 'zones/%s/diskTypes/pd-standard' % self._zone,\n 'physicalBlockSizeBytes': 4096,\n }\n\n if size:\n params['sizeGb'] = size\n\n if snapshot_link:\n params['sourceSnapshot'] = snapshot_link\n\n res = self._client.disks().insert(project=self._project_id, zone=self._zone, body=params).execute()\n\n return res['targetLink']\n\n def get_machine_types(self, machine_type: str = None):\n \"\"\"Returns a list of images that satisfy the name.\n This method is used instead of the \"get\" because it doesn't raise an exception if an image doesn't exist.\n \"\"\"\n filter_str = ('name=%s' % machine_type) if machine_type else None\n res = self._client.machineTypes().list(project=self._project_id, zone=self._zone, filter=filter_str).execute()\n\n if not res.get('items'):\n return []\n\n return res['items']\n\n def stop_instance(self, machine_name: str, wait: bool = True) -> str:\n \"\"\"Stops the instance.\"\"\"\n operation = self._client.instances().stop(project=self._project_id, zone=self._zone,\n instance=machine_name).execute()\n if wait:\n operation = self._wait_operation(operation)\n\n return operation['targetLink']\n\n def delete_instance(self, machine_name: str, wait: bool = True) -> str:\n \"\"\"Deletes the instance.\"\"\"\n operation = self._client.instances().delete(project=self._project_id, zone=self._zone,\n instance=machine_name).execute()\n if wait:\n operation = self._wait_operation(operation)\n\n return operation['targetLink']\n\n def _wait_operation(self, operation: dict):\n \"\"\"Waits util the operation is finished/\"\"\"\n while operation['status'] != 'DONE':\n sleep(5)\n operation = self._client.zoneOperations().wait(project=self._project_id, zone=self._zone,\n operation=operation['name']).execute()\n\n return operation\n" }, { "path": "spotty/providers/gcp/helpers/deployment.py", "content": "import logging\nfrom collections import OrderedDict\nfrom time import sleep\nfrom httplib2 import ServerNotFoundError\nfrom spotty.commands.writers.abstract_output_writrer import AbstractOutputWriter\nfrom spotty.providers.gcp.resources.instance import Instance\nfrom spotty.providers.gcp.resources.stack import Stack\nfrom spotty.providers.gcp.helpers.ce_client import CEClient\nfrom spotty.providers.gcp.helpers.dm_client import DMClient\nfrom spotty.providers.gcp.helpers.dm_resource import DMResource\n\n\ndef wait_resources(dm: DMClient, ce: CEClient, deployment_name: str, resource_messages: OrderedDict,\n instance_resource_name: str, machine_name: str, output: AbstractOutputWriter, delay: int = 5):\n # make sure that the instance resource is in the messages list\n assert any(resource_name == instance_resource_name for resource_name, _ in resource_messages.items())\n\n created_resources = set()\n for resource_name, message in resource_messages.items():\n output.write('- %s...' % message)\n\n is_created = False\n while not is_created:\n sleep(delay)\n\n # get the resource info\n try:\n # check that the deployment is not failed\n stack = Stack.get_by_name(dm, deployment_name)\n if stack.error:\n raise ValueError('Deployment \"%s\" failed.\\n'\n 'Error: %s' % (deployment_name, stack.error['message']))\n\n # check if the instance was preempted, terminated or deleted right after creation\n if instance_resource_name in created_resources:\n instance = Instance.get_by_name(ce, machine_name)\n if not instance or instance.is_stopped:\n raise ValueError('Error: the instance was unexpectedly terminated. Please, check out the '\n 'instance logs to find out the reason.\\n')\n\n # get resource\n resource = DMResource.get_by_name(dm, deployment_name, resource_name)\n except (ConnectionResetError, ServerNotFoundError):\n logging.warning('Connection problem')\n continue\n\n # resource doesn't exist yet\n if not resource:\n continue\n\n # resource failed\n if resource.is_failed:\n error_msg = ('Error: ' + resource.error_message) if resource.error_message \\\n else 'Please, see Deployment Manager logs for the details.' % deployment_name\n\n raise ValueError('Deployment \"%s\" failed.\\n%s' % (deployment_name, error_msg))\n\n # resource was successfully created\n is_created = resource.is_created\n\n created_resources.add(resource_name)\n\n\ndef check_gpu_configuration(ce: CEClient, gpu_parameters: dict):\n if not gpu_parameters:\n return\n\n # check GPU type\n accelerator_types = ce.get_accelerator_types()\n gpu_type = gpu_parameters['type']\n if gpu_type not in accelerator_types:\n if accelerator_types:\n error_msg = 'GPU type \"%s\" is not supported in the \"%s\" zone.\\nAvailable GPU types are: %s.' \\\n % (gpu_type, ce.zone, ', '.join(accelerator_types.keys()))\n else:\n error_msg = 'The \"%s\" zone doesn\\'t support any GPU accelerators.' % ce.zone\n\n raise ValueError(error_msg)\n\n # check the number of GPUs is not exceed the maximum\n max_cards_per_instance = accelerator_types[gpu_parameters['type']]\n if gpu_parameters['count'] > max_cards_per_instance:\n raise ValueError('Maximum allowed number of cards per instance for the \"%s\" type is %d.'\n % (gpu_parameters['type'], max_cards_per_instance))\n\n" }, { "path": "spotty/providers/gcp/helpers/dm_client.py", "content": "import json\nimport googleapiclient.discovery\nfrom googleapiclient.errors import HttpError\n\n\nclass DMClient(object):\n \"\"\"Deployment Manager client.\"\"\"\n\n def __init__(self, project_id: str, zone: str):\n self._project_id = project_id\n self._zone = zone\n self._client = googleapiclient.discovery.build('deploymentmanager', 'v2', cache_discovery=False)\n\n def get(self, deployment_name: str):\n try:\n res = self._client.deployments().get(project=self._project_id, deployment=deployment_name).execute()\n except HttpError as e:\n data = json.loads(e.content.decode('utf-8'))\n if data['error']['code'] != 404:\n raise e\n res = None\n\n return res\n\n def deploy(self, deployment_name: str, template: str, dry_run: bool = False):\n res = self._client.deployments().insert(project=self._project_id, body={\n 'name': deployment_name,\n 'target': {\n 'config': {\n 'content': template,\n },\n },\n }, preview=dry_run).execute()\n\n return res\n\n def stop(self, deployment_name: str, fingerprint: str):\n res = self._client.deployments().stop(project=self._project_id, deployment=deployment_name, body={\n 'fingerprint': fingerprint,\n }).execute()\n\n return res\n\n def delete(self, deployment_name: str):\n \"\"\"Deletes a deployment and all of the resources in the deployment.\"\"\"\n res = self._client.deployments().delete(project=self._project_id, deployment=deployment_name).execute()\n return res\n\n def get_resource(self, deployment_name: str, resource_name: str) -> dict:\n try:\n res = self._client.resources().get(project=self._project_id,\n deployment=deployment_name,\n resource=resource_name).execute()\n except HttpError as e:\n data = json.loads(e.content.decode('utf-8'))\n if data['error']['code'] != 404:\n raise e\n res = None\n\n return res\n" }, { "path": "spotty/providers/gcp/helpers/dm_resource.py", "content": "from spotty.providers.gcp.helpers.dm_client import DMClient\n\n\nclass DMResource(object):\n\n def __init__(self, dm: DMClient, data: dict):\n \"\"\"\n Args:\n dm (DMClient): Deployment Manager client\n data (dict): Stack info.\n Example #1:\n {'id': '1760655646875625396',\n 'insertTime': '2019-08-25T16:27:23.544-07:00',\n 'name': 'x11-test-i2-docker-waiter',\n 'type': 'runtimeconfig.v1beta1.waiter',\n 'update': {'finalProperties': 'failure:\\n'\n ' cardinality:\\n'\n ' number: 1\\n'\n ' path: /failure\\n'\n 'parent: '\n 'projects/spotty-221422/configs/x11-test-i2-docker-status\\n'\n 'success:\\n'\n ' cardinality:\\n'\n ' number: 1\\n'\n ' path: /success\\n'\n 'timeout: 1800s\\n'\n 'waiter: x11-test-i2-docker-waiter\\n',\n 'intent': 'CREATE_OR_ACQUIRE',\n 'manifest': 'https://www.googleapis.com/deploymentmanager/v2/projects/spotty-221422/global/deployments/spotty-instance-x11-test-i2/manifests/manifest-1566775635906',\n 'properties': 'failure:\\n'\n ' cardinality:\\n'\n ' number: 1\\n'\n ' path: /failure\\n'\n 'parent: $(ref.x11-test-i2-docker-status.name)\\n'\n 'success:\\n'\n ' cardinality:\\n'\n ' number: 1\\n'\n ' path: /success\\n'\n 'timeout: 1800s\\n'\n 'waiter: x11-test-i2-docker-waiter\\n',\n 'state': 'IN_PROGRESS'},\n 'updateTime': '2019-08-25T16:27:23.544-07:00'}\n\n Example #2:\n {'finalProperties': 'config: x11-test-i2-docker-status\\n'\n 'description: Docker status\\n',\n 'id': '314866945194106123',\n 'insertTime': '2019-08-25T17:12:20.140-07:00',\n 'manifest': 'https://www.googleapis.com/deploymentmanager/v2/projects/spotty-221422/global/deployments/spotty-instance-x11-test-i2/manifests/manifest-1566778333272',\n 'name': 'x11-test-i2-docker-status',\n 'properties': 'config: x11-test-i2-docker-status\\n'\n 'description: Docker status\\n',\n 'type': 'runtimeconfig.v1beta1.config',\n 'updateTime': '2019-08-25T17:12:30.254-07:00',\n 'url': 'https://runtimeconfig.googleapis.com/v1beta1/projects/spotty-221422/configs/x11-test-i2-docker-status'}\n \"\"\"\n self._dm = dm\n self._data = data\n\n @staticmethod\n def get_by_name(dm: DMClient, deployment_name: str, resource_name: str):\n \"\"\"Returns an instance by its stack name.\"\"\"\n res = dm.get_resource(deployment_name, resource_name)\n if not res:\n return None\n\n return DMResource(dm, res)\n\n @property\n def is_created(self) -> bool:\n return 'finalProperties' in self._data\n\n @property\n def error_message(self) -> str:\n if 'error' not in self._data.get('update', {}):\n return None\n\n return self._data['update']['error']['errors'][0]['message']\n\n @property\n def state(self) -> str:\n return self._data['update']['state'] if 'state' in self._data.get('update', {}) else None\n\n @property\n def is_in_progress(self) -> bool:\n return self.state == 'IN_PROGRESS'\n\n @property\n def is_failed(self) -> bool:\n # an error occurred or the resource is in an unexpected status\n return self.error_message or (self.state is not None and\n self.state not in ['PENDING', 'IN_PROGRESS', 'COMPLETED', 'IN_PREVIEW'])\n" }, { "path": "spotty/providers/gcp/helpers/gcp_credentials.py", "content": "from google.auth import default\n\n\nclass GcpCredentials(object):\n def __init__(self):\n credentials, effective_project_id = default()\n\n self._credentials = credentials\n self._project_id = effective_project_id\n\n @property\n def project_id(self):\n return self._project_id\n\n @property\n def service_account_email(self):\n return self._credentials.service_account_email\n" }, { "path": "spotty/providers/gcp/helpers/gs_client.py", "content": "from typing import List\nfrom google.cloud import storage\nfrom google.cloud.storage import Bucket\n\n\nclass GSClient(object):\n \"\"\"Google Storage client.\"\"\"\n\n def __init__(self):\n self._client = storage.Client()\n\n def list_buckets(self) -> List[Bucket]:\n res = list(self._client.list_buckets())\n return res\n\n def create_bucket(self, bucket_name: str, region: str) -> Bucket:\n bucket = Bucket(self._client, name=bucket_name)\n bucket.create(location=region)\n\n return bucket\n\n def create_dir(self, bucket_name: str, path: str):\n bucket = Bucket(self._client, name=bucket_name)\n blob = bucket.blob(path.rstrip('/') + '/')\n blob.upload_from_string('')\n" }, { "path": "spotty/providers/gcp/helpers/gsutil_rsync.py", "content": "import fnmatch\nfrom shutil import which\nimport os\nfrom typing import List\n\nfrom spotty.deployment.utils.cli import shlex_join\n\n\ndef check_gsutil_installed():\n \"\"\"Checks that gsutil is installed.\"\"\"\n if which('gsutil') is None:\n raise ValueError('gsutil is not installed.')\n\n\ndef get_rsync_command(from_path: str, to_path: str, filters: List[dict] = None, delete: bool = False,\n quiet: bool = False, dry_run: bool = False):\n args = ['gsutil', '-m']\n if quiet:\n args.append('-q')\n\n args += ['rsync', '-r']\n\n if filters:\n if (len(filters) > 1) or (len(filters[0]) > 1) or ('include' in filters[0]):\n raise ValueError('At the moment GCP provider supports only one list of exclude filters.')\n\n path_regs = []\n for path in filters[0]['exclude']:\n path = path.replace('/', os.sep) # fix for Windows machines\n path_regs.append(fnmatch.translate(path)[4:-3])\n\n filter_regex = '^(%s)$' % '|'.join(path_regs)\n args += ['-x', filter_regex]\n\n if delete:\n args.append('-d')\n\n if dry_run:\n args.append('-n')\n\n args += [from_path, to_path]\n\n return shlex_join(args)\n" }, { "path": "spotty/providers/gcp/helpers/image.py", "content": "from spotty.providers.gcp.config.instance_config import DEFAULT_IMAGE_NAME\nfrom spotty.providers.gcp.helpers.ce_client import CEClient\nfrom spotty.providers.gcp.resources.image import Image\n\n\ndef get_image(ce: CEClient, image_uri: str = None, image_name: str = None) -> Image:\n \"\"\"Returns an image that should be used for deployment.\n\n Raises:\n ValueError: If an image not found.\n \"\"\"\n if image_uri:\n # get an image by its URL if the \"imageUri\" parameter is specified\n image = Image.get_by_uri(ce, image_uri)\n if not image:\n raise ValueError('Image \"%s\" not found.' % image_uri)\n elif image_name:\n # get an image by name if the \"imageName\" parameter is specified\n image = Image.get_by_name(ce, image_name)\n if not image:\n # if an image name was explicitly specified, but the image was not found, raise an error\n raise ValueError('Image with the name \"%s\" was not found.' % image_name)\n else:\n # if the \"imageName\" parameter is not specified, try to use the default image name\n image = Image.get_by_name(ce, DEFAULT_IMAGE_NAME)\n if not image:\n # get the latest \"common-gce-gpu-image\" image\n image_family_url = 'projects/ml-images/global/images/family/common-gce-gpu-image'\n image = Image.get_by_uri(ce, image_family_url)\n if not image:\n raise ValueError('The \"common-gce-gpu-image\" image was not found.')\n\n return image\n" }, { "path": "spotty/providers/gcp/helpers/rtc_client.py", "content": "import googleapiclient.discovery\n\n\nclass RtcClient(object):\n\n def __init__(self, project_id: str, zone: str):\n self._project_id = project_id\n self._zone = zone\n self._rtc = googleapiclient.discovery.build('runtimeconfig', 'v1beta1', cache_discovery=False)\n\n def get_value(self, config_name, template):\n config_name = 'projects/%s/configs/%s' % (self._project_id, config_name)\n fields = ['/failure']\n res = self._rtc.projects().get(name=config_name, fields=fields).execute()\n\n return res\n\n def set_value(self, config_name: str, variable_name: str, value: str):\n config_name = 'projects/%s/configs/%s' % (self._project_id, config_name)\n res = self._rtc.projects().configs().variables().create(parent=config_name, body={\n 'name': '%s/variables/%s' % (config_name, variable_name),\n 'text': str(value),\n }).execute()\n\n return res\n" }, { "path": "spotty/providers/gcp/helpers/volumes.py", "content": "from typing import List\nfrom spotty.commands.writers.abstract_output_writrer import AbstractOutputWriter\nfrom spotty.config.abstract_instance_volume import AbstractInstanceVolume\nfrom spotty.providers.gcp.config.disk_volume import DiskVolume\nfrom spotty.providers.gcp.helpers.ce_client import CEClient\nfrom spotty.providers.gcp.resources.disk import Disk\nfrom spotty.providers.gcp.resources.snapshot import Snapshot\n\n\ndef create_disks(ce: CEClient, volumes: List[AbstractInstanceVolume], output: AbstractOutputWriter,\n dry_run: bool = False):\n disks_to_create = []\n\n # do some checks and prepare disk parameters\n for i, volume in enumerate(volumes):\n if isinstance(volume, DiskVolume):\n # check if the disk already exists\n disk = Disk.get_by_name(ce, volume.disk_name)\n if disk:\n # check if the volume is available\n if not disk.is_available():\n raise ValueError('Disk \"%s\" is not available (status: %s).'\n % (volume.disk_name, disk.status))\n\n # check size of the volume\n if volume.size and (volume.size != disk.size):\n raise ValueError('Specified size for the \"%s\" volume (%dGB) doesn\\'t match the size of the '\n 'existing disk (%dGB).' % (volume.name, volume.size, disk.size))\n\n output.write('- disk \"%s\" will be attached' % disk.name)\n else:\n # check if the snapshot exists\n snapshot = Snapshot.get_by_name(ce, volume.disk_name)\n if snapshot:\n # disk will be restored from the snapshot\n # check size of the volume\n if volume.size and (volume.size < snapshot.size):\n raise ValueError('Specified size for the \"%s\" volume (%dGB) is less than size of the '\n 'snapshot (%dGB).'\n % (volume.name, volume.size, snapshot.size))\n\n output.write('- disk \"%s\" will be restored from the snapshot' % volume.disk_name)\n\n disks_to_create.append((volume.disk_name, volume.size, snapshot.self_link))\n else:\n # empty volume will be created, check that the size is specified\n if not volume.size:\n raise ValueError('Size for the new disk is required.')\n\n if volume.size < 10:\n raise ValueError('Size of a disk cannot be less than 10GB.')\n\n disks_to_create.append((volume.disk_name, volume.size, None))\n\n # create disks\n for disk_name, disk_size, snapshot_link in disks_to_create:\n if not dry_run:\n ce.create_disk(disk_name, disk_size, snapshot_link)\n\n output.write('- disk \"%s\" was created' % disk_name)\n" }, { "path": "spotty/providers/gcp/instance_deployment.py", "content": "from spotty.commands.writers.abstract_output_writrer import AbstractOutputWriter\nfrom spotty.deployment.abstract_cloud_instance.abstract_instance_deployment import AbstractInstanceDeployment\nfrom spotty.deployment.container.docker.docker_commands import DockerCommands\nfrom spotty.deployment.utils.print_info import render_volumes_info_table\nfrom spotty.providers.gcp.config.instance_config import InstanceConfig\nfrom spotty.providers.gcp.data_transfer import DataTransfer\nfrom spotty.providers.gcp.dm_templates.instance.instance_template import prepare_instance_template\nfrom spotty.providers.gcp.helpers.image import get_image\nfrom spotty.providers.gcp.helpers.volumes import create_disks\nfrom spotty.providers.gcp.resource_managers.instance_stack_manager import InstanceStackManager\nfrom spotty.providers.gcp.helpers.ce_client import CEClient\nfrom spotty.providers.gcp.helpers.gcp_credentials import GcpCredentials\nfrom spotty.providers.gcp.resource_managers.ssh_key_manager import SshKeyManager\nfrom spotty.providers.gcp.resources.instance import Instance\nfrom spotty.providers.gcp.helpers.deployment import check_gpu_configuration\n\n\nclass InstanceDeployment(AbstractInstanceDeployment):\n\n instance_config: InstanceConfig\n\n def __init__(self, instance_config: InstanceConfig):\n super().__init__(instance_config)\n\n self._project_name = instance_config.project_config.project_name\n self._credentials = GcpCredentials()\n self._ce = CEClient(self._credentials.project_id, instance_config.zone)\n\n @property\n def stack_manager(self) -> InstanceStackManager:\n return InstanceStackManager(self.instance_config.machine_name, self._credentials.project_id, self.instance_config.zone)\n\n @property\n def ssh_key_manager(self) -> SshKeyManager:\n return SshKeyManager(self._project_name, self.instance_config.zone)\n\n def get_instance(self) -> Instance:\n return Instance.get_by_name(self._ce, self.instance_config.machine_name)\n\n def deploy(self, container_commands: DockerCommands, bucket_name: str,\n data_transfer: DataTransfer, output: AbstractOutputWriter, dry_run: bool = False):\n # check machine type\n if not self._ce.get_machine_types(self.instance_config.machine_type):\n raise ValueError('\"%s\" machine type is not available in the \"%s\" zone.'\n % (self.instance_config.machine_type, self.instance_config.zone))\n\n # check GPU configuration\n check_gpu_configuration(self._ce, self.instance_config.gpu)\n\n # remove the stack it it exists to make all the disks available\n stack_manager = self.stack_manager\n stack_manager.delete_stack(output=output)\n\n # sync the project with the S3 bucket\n if bucket_name is not None:\n output.write('Syncing the project with the bucket...')\n data_transfer.upload_local_to_bucket(bucket_name, dry_run=dry_run)\n\n # create volumes\n if self.instance_config.volumes:\n # create disks\n output.write('\\nCreating disks...')\n with output.prefix(' '):\n create_disks(self._ce, self.instance_config.volumes, output=output, dry_run=dry_run)\n output.write('')\n\n # prepare Deployment Manager template\n output.write('Preparing the deployment template...')\n with output.prefix(' '):\n # get an image\n image_link = get_image(self._ce, self.instance_config.image_uri, self.instance_config.image_name).self_link\n\n # get or create an SSH key\n public_key_value = self.ssh_key_manager.get_public_key_value()\n\n # prepare the deployment template\n sync_project_cmd = data_transfer.get_download_bucket_to_instance_command(bucket_name=bucket_name)\n template = prepare_instance_template(\n instance_config=self.instance_config,\n docker_commands=container_commands,\n image_link=image_link,\n bucket_name=bucket_name,\n sync_project_cmd=sync_project_cmd,\n public_key_value=public_key_value,\n service_account_email=self._credentials.service_account_email,\n output=output,\n )\n\n output.write('')\n\n # print information about the volumes\n output.write('Volumes:\\n%s\\n' % render_volumes_info_table(self.instance_config.volume_mounts,\n self.instance_config.volumes))\n\n # create stack\n if not dry_run:\n stack_manager.create_stack(template, output=output)\n\n def delete(self, output: AbstractOutputWriter):\n self.stack_manager.delete_stack(output)\n\n # TODO: apply deletion policies\n" }, { "path": "spotty/providers/gcp/instance_manager.py", "content": "from spotty.commands.writers.abstract_output_writrer import AbstractOutputWriter\nfrom spotty.errors.instance_not_running import InstanceNotRunningError\nfrom spotty.deployment.abstract_cloud_instance.abstract_cloud_instance_manager import AbstractCloudInstanceManager\nfrom spotty.providers.gcp.config.instance_config import InstanceConfig\nfrom spotty.providers.gcp.data_transfer import DataTransfer\nfrom spotty.providers.gcp.instance_deployment import InstanceDeployment\nfrom spotty.providers.gcp.resource_managers.bucket_manager import BucketManager\nfrom spotty.utils import render_table\n\n\nclass InstanceManager(AbstractCloudInstanceManager):\n\n instance_config: InstanceConfig\n bucket_manager: BucketManager\n data_transfer: DataTransfer\n instance_deployment: InstanceDeployment\n\n def _get_instance_config(self, instance_config: dict) -> InstanceConfig:\n \"\"\"Validates the instance config and returns an InstanceConfig object.\"\"\"\n return InstanceConfig(instance_config, self.project_config)\n\n def _get_bucket_manager(self) -> BucketManager:\n region = '-'.join(self.instance_config.zone.split('-')[:-1])\n return BucketManager(self.instance_config.project_config.project_name, region)\n\n def _get_data_transfer(self) -> DataTransfer:\n \"\"\"Returns a data transfer object.\"\"\"\n return DataTransfer(\n local_project_dir=self.project_config.project_dir,\n host_project_dir=self.instance_config.host_project_dir,\n sync_filters=self.project_config.sync_filters,\n instance_name=self.instance_config.name,\n )\n\n def _get_instance_deployment(self) -> InstanceDeployment:\n \"\"\"Returns an instance deployment manager.\"\"\"\n return InstanceDeployment(self.instance_config)\n\n def download(self, download_filters: list, output: AbstractOutputWriter, dry_run=False):\n raise NotImplementedError('GCP provider doesn\\'t have an implementation of the \"download\" command yet.')\n\n def get_status_text(self) -> str:\n instance = self.instance_deployment.get_instance()\n if not instance:\n raise InstanceNotRunningError(self.instance_config.name)\n\n table = [\n ('Instance Status', instance.status),\n ('Machine Type', instance.machine_type),\n ('Zone', instance.zone),\n ]\n\n if instance.public_ip_address:\n table.append(('Public IP Address', instance.public_ip_address))\n\n table.append(('Purchasing Option', 'Preemtible VM' if instance.is_preemtible else 'On-demand VM'))\n\n return render_table(table)\n\n @property\n def ssh_key_path(self):\n return self.instance_deployment.ssh_key_manager.private_key_file\n" }, { "path": "spotty/providers/gcp/resource_managers/__init__.py", "content": "" }, { "path": "spotty/providers/gcp/resource_managers/bucket_manager.py", "content": "import re\nfrom spotty.deployment.abstract_cloud_instance.abstract_bucket_manager import AbstractBucketManager\nfrom spotty.deployment.abstract_cloud_instance.errors.bucket_not_found import BucketNotFoundError\nfrom spotty.providers.gcp.helpers.gs_client import GSClient\nfrom spotty.providers.gcp.resources.bucket import Bucket\nfrom spotty.utils import random_string\n\n\nclass BucketManager(AbstractBucketManager):\n\n def __init__(self, project_name: str, region: str):\n super().__init__(project_name)\n\n self._gs = GSClient()\n self._region = region\n self._bucket_prefix = 'spotty-%s' % project_name.lower()\n\n def get_bucket(self) -> Bucket:\n buckets = self._gs.list_buckets()\n\n regex = re.compile('-'.join([self._bucket_prefix, '[a-z0-9]{12}', self._region]))\n buckets = [bucket for bucket in buckets if regex.match(bucket.name) is not None]\n\n if len(buckets) > 1:\n raise ValueError('Found several project buckets in the same region: %s.'\n % ', '.join(bucket.name for bucket in buckets))\n\n if not len(buckets):\n raise BucketNotFoundError\n\n bucket = Bucket(buckets[0])\n\n return bucket\n\n def create_bucket(self) -> Bucket:\n bucket_name = '-'.join([self._bucket_prefix, random_string(12), self._region])\n bucket = self._gs.create_bucket(bucket_name, self._region)\n\n return Bucket(bucket)\n" }, { "path": "spotty/providers/gcp/resource_managers/instance_stack_manager.py", "content": "from collections import OrderedDict\nfrom spotty.commands.writers.abstract_output_writrer import AbstractOutputWriter\nfrom spotty.providers.gcp.resources.stack import Stack\nfrom spotty.providers.gcp.helpers.ce_client import CEClient\nfrom spotty.providers.gcp.helpers.deployment import wait_resources\nfrom spotty.providers.gcp.helpers.dm_client import DMClient\nfrom spotty.providers.gcp.helpers.dm_resource import DMResource\nfrom spotty.providers.gcp.helpers.rtc_client import RtcClient\n\n\nclass InstanceStackManager(object):\n\n def __init__(self, machine_name: str, project_id: str, zone: str):\n self._dm = DMClient(project_id, zone)\n self._ce = CEClient(project_id, zone)\n self._rtc = RtcClient(project_id, zone)\n self._machine_name = machine_name\n self._stack_name = 'spotty-instance-' + machine_name\n\n # resource names\n self._INSTANCE_RESOURCE_NAME = machine_name\n self._DOCKER_WAITER_RESOURCE_NAME = machine_name + '-docker-waiter'\n self._DOCKER_STATUS_CONFIG_RESOURCE_NAME = machine_name + '-docker-status'\n\n @property\n def name(self):\n return self._stack_name\n\n def create_stack(self, template: str, output: AbstractOutputWriter):\n \"\"\"Deploys a Deployment Manager template.\"\"\"\n\n # create a stack\n res = Stack.create(self._dm, self._stack_name, template)\n # print(res)\n # exit()\n\n output.write('Waiting for the stack to be created...')\n\n resource_messages = OrderedDict([\n (self._INSTANCE_RESOURCE_NAME, 'launching the instance'),\n (self._DOCKER_WAITER_RESOURCE_NAME, 'running the Docker container'),\n ])\n\n # wait for the stack to be created\n with output.prefix(' '):\n wait_resources(self._dm, self._ce, self._stack_name, resource_messages,\n instance_resource_name=self._INSTANCE_RESOURCE_NAME, machine_name=self._machine_name,\n output=output)\n\n def delete_stack(self, output: AbstractOutputWriter):\n stack = Stack.get_by_name(self._dm, self._stack_name)\n if not stack:\n return\n\n output.write('Waiting for the stack to be deleted...')\n\n # delete the stack\n try:\n if stack.is_running:\n # stop an ongoing operation first to make sure the delete method\n # won't raise an error \"Resource '...' has an ongoing conflicting operation\"\n stack.stop()\n\n # if the docker-waiter resource is still waiting for a signal, send a failure signal\n # to be able to delete the stack\n resource = DMResource.get_by_name(self._dm, self._stack_name, self._DOCKER_WAITER_RESOURCE_NAME)\n if resource.is_in_progress:\n self._rtc.set_value(self._DOCKER_STATUS_CONFIG_RESOURCE_NAME, '/failure/1', '1')\n\n # wait until the stack will be created or will fail\n stack.wait_stack_done()\n\n stack.delete()\n stack.wait_stack_deleted()\n except Exception as e:\n raise ValueError('Stack \"%s\" was not deleted. Error: %s\\n'\n 'See Deployment Manager logs for details.' % (self._stack_name, str(e)))\n" }, { "path": "spotty/providers/gcp/resource_managers/ssh_key_manager.py", "content": "import os\nimport subprocess\nfrom spotty.configuration import get_spotty_keys_dir\nfrom shutil import which\nfrom spotty.providers.instance_manager_factory import PROVIDER_GCP\n\n\nclass SshKeyManager(object):\n\n def __init__(self, project_name: str, zone: str):\n self._key_name = 'spotty-key-%s-%s' % (project_name.lower(), zone)\n self._keys_dir = get_spotty_keys_dir(PROVIDER_GCP)\n\n @property\n def private_key_file(self):\n return os.path.join(self._keys_dir, self._key_name)\n\n @property\n def public_key_file(self):\n return os.path.join(self._keys_dir, self._key_name + '.pub')\n\n def get_public_key_value(self):\n # generate a key if it doesn't exist\n if not os.path.isfile(self.private_key_file) or not os.path.isfile(self.public_key_file):\n self._generate_ssh_key()\n\n # read the public key value\n with open(self.public_key_file, 'r') as f:\n public_key_value = f.read().split()[1]\n\n return public_key_value\n\n def _generate_ssh_key(self):\n # delete the private key file if it already exists\n if os.path.isfile(self.private_key_file):\n os.unlink(self.private_key_file)\n\n # create a provider subdirectory\n if not os.path.isdir(self._keys_dir):\n os.makedirs(self._keys_dir, mode=0o755, exist_ok=True)\n\n # check that the \"ssh-keygen\" tool is installed\n ssh_keygen_cmd = 'ssh-keygen'\n if which(ssh_keygen_cmd) is None:\n raise ValueError('\"ssh-keygen\" command not found.')\n\n generate_key_cmd = [ssh_keygen_cmd, '-t', 'rsa', '-N', '', '-f', self.private_key_file, '-q']\n\n # generate a key pair\n res = subprocess.run(generate_key_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)\n if res.returncode:\n raise subprocess.CalledProcessError(res.returncode, generate_key_cmd)\n" }, { "path": "spotty/providers/gcp/resources/__init__.py", "content": "" }, { "path": "spotty/providers/gcp/resources/bucket.py", "content": "from spotty.deployment.abstract_cloud_instance.resources.abstract_bucket import AbstractBucket\nfrom google.cloud.storage import Bucket as GSBucket\n\n\nclass Bucket(AbstractBucket):\n\n def __init__(self, bucket: GSBucket):\n self._bucket = bucket\n\n @property\n def name(self) -> str:\n return self._bucket.name\n" }, { "path": "spotty/providers/gcp/resources/disk.py", "content": "from spotty.providers.gcp.helpers.ce_client import CEClient\n\n\nclass Disk(object):\n\n def __init__(self, ce: CEClient, data: dict):\n \"\"\"\n Args:\n data (dict): Example:\n {'creationTimestamp': '2019-04-20T16:21:49.579-07:00',\n 'guestOsFeatures': [{'type': 'VIRTIO_SCSI_MULTIQUEUE'}],\n 'id': '1546539587132069731',\n 'kind': 'compute#disk',\n 'labelFingerprint': '42WmSpB8rSM=',\n 'lastAttachTimestamp': '2019-04-20T16:21:49.580-07:00',\n 'licenseCodes': ['1000205'],\n 'licenses': ['https://www.googleapis.com/compute/v1/projects/debian-cloud/global/licenses/debian-9-stretch'],\n 'name': 'instance-1',\n 'physicalBlockSizeBytes': '4096',\n 'selfLink': 'https://www.googleapis.com/compute/v1/projects/spotty-221422/zones/us-east1-b/disks/instance-1',\n 'sizeGb': '10',\n 'sourceImage': 'https://www.googleapis.com/compute/v1/projects/debian-cloud/global/images/debian-9-stretch-v20190326',\n 'sourceImageId': '6831652533131678657',\n 'status': 'READY',\n 'type': 'https://www.googleapis.com/compute/v1/projects/spotty-221422/zones/us-east1-b/diskTypes/pd-standard',\n 'users': ['https://www.googleapis.com/compute/v1/projects/spotty-221422/zones/us-east1-b/instances/instance-1'],\n 'zone': 'https://www.googleapis.com/compute/v1/projects/spotty-221422/zones/us-east1-b'}\n \"\"\"\n self._ce = ce\n self._data = data\n\n @staticmethod\n def get_by_name(ce: CEClient, disk_name: str):\n \"\"\"Returns a disk by its name.\"\"\"\n res = ce.list_disks(disk_name)\n if not res:\n return None\n\n return Disk(ce, res[0])\n\n @property\n def name(self) -> str:\n return self._data['name']\n\n @property\n def status(self) -> str:\n return self._data['status']\n\n @property\n def size(self) -> int:\n return int(self._data['sizeGb'])\n\n @property\n def users(self) -> list:\n return self._data.get('users', [])\n\n def is_available(self):\n return (self.status == 'READY') and not self.users\n" }, { "path": "spotty/providers/gcp/resources/image.py", "content": "from spotty.providers.gcp.config.image_uri import ImageUri\nfrom spotty.providers.gcp.helpers.ce_client import CEClient\n\n\nclass Image(object):\n\n def __init__(self, data: dict):\n \"\"\"\n Args:\n data (dict): Example:\n {'archiveSizeBytes': '3652446976',\n 'creationTimestamp': '2018-11-03T19:00:48.577-07:00',\n 'description': '',\n 'diskSizeGb': '10',\n 'guestOsFeatures': [{'type': 'VIRTIO_SCSI_MULTIQUEUE'}],\n 'id': '7541350343606791231',\n 'kind': 'compute#image',\n 'labelFingerprint': '42WmSpB8rSM=',\n 'licenseCodes': ['1000201'],\n 'licenses': ['https://www.googleapis.com/compute/v1/projects/ubuntu-os-cloud/global/licenses/ubuntu-1604-xenial'],\n 'name': 'spotty-ami',\n 'selfLink': 'https://www.googleapis.com/compute/v1/projects/spotty-221422/global/images/spotty-ami',\n 'sourceDisk': 'https://www.googleapis.com/compute/v1/projects/spotty-221422/zones/us-central1-a/disks/spotty-ami',\n 'sourceDiskId': '3401548142858207031',\n 'sourceType': 'RAW',\n 'status': 'READY'}\n \"\"\"\n self._data = data\n\n @staticmethod\n def get_by_name(ce: CEClient, image_name: str):\n \"\"\"Returns an image by its name.\"\"\"\n res = ce.list_images(image_name)\n if not res:\n return None\n\n return Image(res[0])\n\n @staticmethod\n def get_by_uri(ce: CEClient, image_uri: str):\n image_uri = ImageUri(image_uri)\n if image_uri.is_family:\n image_data = ce.get_image_from_family(family_name=image_uri.name, project_id=image_uri.project_id)\n else:\n res = ce.list_images(image_name=image_uri.name, project_id=image_uri.project_id)\n image_data = res[0] if res else None\n\n if not image_data:\n return None\n\n return Image(image_data)\n\n @property\n def image_id(self) -> str:\n return self._data['id']\n\n @property\n def name(self) -> str:\n return self._data['name']\n\n @property\n def size(self) -> int:\n return self._data['diskSizeGb']\n\n @property\n def self_link(self) -> str:\n return self._data['selfLink']\n\n @property\n def source_disk(self):\n return self._data['sourceDisk']\n" }, { "path": "spotty/providers/gcp/resources/instance.py", "content": "from datetime import datetime\nfrom spotty.deployment.abstract_cloud_instance.resources.abstract_instance import AbstractInstance\nfrom spotty.providers.gcp.helpers.ce_client import CEClient\n\n\nclass Instance(AbstractInstance):\n\n def __init__(self, ce: CEClient, data: dict):\n \"\"\"\n Args:\n data (dict): Example:\n {'canIpForward': False,\n 'cpuPlatform': 'Intel Haswell',\n 'creationTimestamp': '2019-04-20T16:21:49.536-07:00',\n 'deletionProtection': False,\n 'description': '',\n 'disks': [{'autoDelete': True,\n 'boot': True,\n 'deviceName': 'instance-1',\n 'guestOsFeatures': [{'type': 'VIRTIO_SCSI_MULTIQUEUE'}],\n 'index': 0,\n 'interface': 'SCSI',\n 'kind': 'compute#attachedDisk',\n 'licenses': ['https://www.googleapis.com/compute/v1/projects/debian-cloud/global/licenses/debian-9-stretch'],\n 'mode': 'READ_WRITE',\n 'source': 'https://www.googleapis.com/compute/v1/projects/spotty-221422/zones/us-east1-b/disks/instance-1',\n 'type': 'PERSISTENT'}],\n 'id': '928537266896639843',\n 'kind': 'compute#instance',\n 'labelFingerprint': '42WmSpB8rSM=',\n 'machineType': 'https://www.googleapis.com/compute/v1/projects/spotty-221422/zones/us-east1-b/machineTypes/n1-standard-1',\n 'metadata': {'fingerprint': 'IoRxXrApBlw=',\n 'kind': 'compute#metadata'},\n 'name': 'instance-1',\n 'networkInterfaces': [{'accessConfigs': [{'kind': 'compute#accessConfig',\n 'name': 'External NAT',\n 'natIP': '34.73.140.188',\n 'networkTier': 'PREMIUM',\n 'type': 'ONE_TO_ONE_NAT'}],\n 'fingerprint': 'COAWpxIgZx0=',\n 'kind': 'compute#networkInterface',\n 'name': 'nic0',\n 'network': 'https://www.googleapis.com/compute/v1/projects/spotty-221422/global/networks/default',\n 'networkIP': '10.142.0.2',\n 'subnetwork': 'https://www.googleapis.com/compute/v1/projects/spotty-221422/regions/us-east1/subnetworks/default'}],\n 'scheduling': {'automaticRestart': False,\n 'onHostMaintenance': 'TERMINATE',\n 'preemptible': True},\n 'selfLink': 'https://www.googleapis.com/compute/v1/projects/spotty-221422/zones/us-east1-b/instances/instance-1',\n 'serviceAccounts': [{'email': '293101887402-compute@developer.gserviceaccount.com',\n 'scopes': ['https://www.googleapis.com/auth/devstorage.read_only',\n 'https://www.googleapis.com/auth/logging.write',\n 'https://www.googleapis.com/auth/monitoring.write',\n 'https://www.googleapis.com/auth/servicecontrol',\n 'https://www.googleapis.com/auth/service.management.readonly',\n 'https://www.googleapis.com/auth/trace.append']}],\n 'startRestricted': False,\n 'status': 'RUNNING',\n 'tags': {'fingerprint': '42WmSpB8rSM='},\n 'zone': 'https://www.googleapis.com/compute/v1/projects/spotty-221422/zones/us-east1-b'}\n \"\"\"\n self._ce = ce\n self._data = data\n\n @staticmethod\n def get_by_name(ce: CEClient, machine_name: str):\n \"\"\"Returns an instance by its stack name.\"\"\"\n res = ce.list_instances(machine_name)\n if not res:\n return None\n\n return Instance(ce, res[0])\n\n @property\n def name(self) -> str:\n return self._data['name']\n\n @property\n def is_running(self) -> bool:\n return self.status == 'RUNNING'\n\n @property\n def is_stopped(self) -> bool:\n # see Instance Life Cycle: https://cloud.google.com/compute/docs/instances/instance-life-cycle\n return self.status == 'TERMINATED'\n\n @property\n def public_ip_address(self) -> str:\n return self._data['networkInterfaces'][0]['accessConfigs'][0].get('natIP')\n\n @property\n def status(self) -> str:\n return self._data['status']\n\n @property\n def machine_type(self) -> str:\n return self._data['machineType'].split('/')[-1]\n\n @property\n def zone(self) -> str:\n return self._data['zone'].split('/')[-1]\n\n @property\n def creation_timestamp(self) -> datetime:\n # fix the format: '2019-04-20T16:21:49.536-07:00' -> '2019-04-20T16:21:49-0700'\n time_str = self._data['creationTimestamp'][:-10] + \\\n self._data['creationTimestamp'][-6:-3] + \\\n self._data['creationTimestamp'][-2:]\n return datetime.strptime(time_str, '%Y-%m-%dT%H:%M:%S%z')\n\n @property\n def is_preemtible(self) -> bool:\n return self._data['scheduling']['preemptible']\n\n def terminate(self, wait: bool = True):\n self._ce.delete_instance(self.name, wait)\n\n def stop(self, wait: bool = True):\n self._ce.stop_instance(self.name, wait)\n" }, { "path": "spotty/providers/gcp/resources/snapshot.py", "content": "from spotty.providers.gcp.helpers.ce_client import CEClient\n\n\nclass Snapshot(object):\n\n def __init__(self, data: dict):\n \"\"\"\n Args:\n data (dict): Example:\n {'creationTimestamp': '2019-04-20T12:40:13.291-07:00',\n 'diskSizeGb': '10',\n 'id': '714587297862306675',\n 'kind': 'compute#snapshot',\n 'labelFingerprint': '42WmSpB8rSM=',\n 'name': 'snapshot-test',\n 'selfLink': 'https://www.googleapis.com/compute/v1/projects/spotty-221422/global/snapshots/snapshot-test',\n 'sourceDisk': 'https://www.googleapis.com/compute/v1/projects/spotty-221422/zones/us-east1-b/disks/disk-test',\n 'sourceDiskId': '599723469887162882',\n 'status': 'READY',\n 'storageBytes': '0',\n 'storageBytesStatus': 'UP_TO_DATE',\n 'storageLocations': ['us-central1']}\n \"\"\"\n self._data = data\n\n @staticmethod\n def get_by_name(ce: CEClient, snapshot_name: str):\n \"\"\"Returns a snapshot by its name.\"\"\"\n res = ce.list_snapshots(snapshot_name)\n if not res:\n return None\n\n return Snapshot(res[0])\n\n @property\n def name(self) -> str:\n return self._data['name']\n\n @property\n def size(self) -> int:\n return self._data['diskSizeGb']\n\n @property\n def self_link(self) -> str:\n return self._data['selfLink']\n" }, { "path": "spotty/providers/gcp/resources/stack.py", "content": "import logging\nfrom time import sleep\nfrom httplib2 import ServerNotFoundError\nfrom spotty.providers.gcp.helpers.dm_client import DMClient\n\n\nclass Stack(object):\n\n def __init__(self, dm: DMClient, data: dict):\n \"\"\"\n Args:\n dm (DMClient): Deployment Manager client\n data (dict): Stack info. Example:\n {'fingerprint': 'vvtbwT7F953T0YC9tQ9CUg==',\n 'id': '3128259442476717093',\n 'insertTime': '2019-04-20T16:27:06.739-07:00',\n 'name': 'spotty-instance-x11-test-i2',\n 'operation': {'endTime': '2019-04-20T16:27:23.141-07:00',\n 'error': {'errors': [{'code': 'RESOURCE_ERROR',\n 'location': '/deployments/spotty-instance-x11-test-i2/resources/spotty-instance-x11-test-i2-disk-1',\n 'message': '{\"ResourceType\":\"compute.v1.disk\",\"ResourceErrorCode\":\"400\",\"ResourceErrorMessage\":{\"code\":400,\"errors\":[{\"domain\":\"global\",\"location\":\"zone\",\"locationType\":\"parameter\",\"message\":\"Invalid '\n \"value 'zones/us-east1-b'. \"\n 'Values must match the '\n 'following regular expression: '\n '\\'[a-z](?:[-a-z0-9]{0,61}[a-z0-9])?\\'\",\"reason\":\"invalidParameter\"}],\"message\":\"Invalid '\n \"value 'zones/us-east1-b'. \"\n 'Values must match the '\n 'following regular expression: '\n '\\'[a-z](?:[-a-z0-9]{0,61}[a-z0-9])?\\'\",\"statusMessage\":\"Bad '\n 'Request\",\"requestPath\":\"https://www.googleapis.com/compute/v1/projects/spotty-221422/zones/zones%2Fus-east1-b/disks\",\"httpMethod\":\"POST\"}}'}]},\n 'httpErrorMessage': 'BAD REQUEST',\n 'httpErrorStatusCode': 400,\n 'id': '1965130840716743717',\n 'insertTime': '2019-04-20T16:27:06.900-07:00',\n 'kind': 'deploymentmanager#operation',\n 'name': 'operation-1555802826515-586fe92d0a605-6166491c-7051043e',\n 'operationType': 'insert',\n 'progress': 100,\n 'selfLink': 'https://www.googleapis.com/deploymentmanager/v2/projects/spotty-221422/global/operations/operation-1555802826515-586fe92d0a605-6166491c-7051043e',\n 'startTime': '2019-04-20T16:27:06.908-07:00',\n 'status': 'DONE',\n 'targetId': '3128259442476717093',\n 'targetLink': 'https://www.googleapis.com/deploymentmanager/v2/projects/spotty-221422/global/deployments/spotty-instance-x11-test-i2',\n 'user': 'spotty@spotty-221422.iam.gserviceaccount.com'},\n 'selfLink': 'https://www.googleapis.com/deploymentmanager/v2/projects/spotty-221422/global/deployments/spotty-instance-x11-test-i2',\n 'update': {'manifest': 'https://www.googleapis.com/deploymentmanager/v2/projects/spotty-221422/global/deployments/spotty-instance-x11-test-i2/manifests/manifest-1555802826772'},\n 'updateTime': '2019-04-20T16:27:23.107-07:00'}\n \"\"\"\n self._dm = dm\n self._data = data\n\n @staticmethod\n def get_by_name(dm: DMClient, deployment_name: str):\n \"\"\"Returns an instance by its stack name.\"\"\"\n res = dm.get(deployment_name)\n if not res:\n return None\n\n return Stack(dm, res)\n\n @staticmethod\n def create(dm: DMClient, deployment_name: str, template: str):\n return dm.deploy(deployment_name, template)\n\n @property\n def name(self) -> str:\n return self._data['name']\n\n @property\n def status(self) -> str:\n return self._data.get('operation', {}).get('status')\n\n @property\n def is_running(self):\n return self.status == 'RUNNING'\n\n @property\n def is_done(self):\n \"\"\"A deployment has the done status when it's successfully created or failed.\"\"\"\n return self.status == 'DONE'\n\n @property\n def error(self) -> str:\n \"\"\"Returns an error in the format: {'code': '...', 'message': '...'}.\"\"\"\n return self._data.get('operation', {}).get('error', {}).get('errors', [None])[0]\n\n @property\n def fingerprint(self) -> str:\n return self._data['fingerprint']\n\n def stop(self):\n self._dm.stop(self.name, self.fingerprint)\n\n def delete(self):\n self._dm.delete(self.name)\n\n def wait_stack_deleted(self, delay=15):\n stack = True\n while stack:\n try:\n stack = self.get_by_name(self._dm, self.name)\n except (ConnectionResetError, ServerNotFoundError):\n logging.warning('Connection problem')\n continue\n\n sleep(delay)\n\n def wait_stack_done(self, delay=5):\n is_done = False\n while not is_done:\n try:\n stack = self.get_by_name(self._dm, self.name)\n is_done = stack.is_done\n except (ConnectionResetError, ServerNotFoundError):\n logging.warning('Connection problem')\n continue\n\n sleep(delay)\n" }, { "path": "spotty/providers/instance_manager_factory.py", "content": "from importlib import import_module\nfrom spotty.config.project_config import ProjectConfig\nfrom spotty.deployment.abstract_instance_manager import AbstractInstanceManager\n\n\nPROVIDER_AWS = 'aws'\nPROVIDER_GCP = 'gcp'\nPROVIDER_LOCAL = 'local'\nPROVIDER_REMOTE = 'remote'\n\n\nclass InstanceManagerFactory(object):\n\n SUPPORTED_PROVIDERS = [\n PROVIDER_AWS,\n PROVIDER_GCP,\n PROVIDER_LOCAL,\n PROVIDER_REMOTE,\n ]\n\n @classmethod\n def get_instance(cls, project_config: ProjectConfig, instance_config: dict) -> AbstractInstanceManager:\n provider_name = instance_config['provider']\n if provider_name not in cls.SUPPORTED_PROVIDERS:\n raise ValueError('Provider \"%s\" is not supported' % provider_name)\n\n # get Instance Manger class for the provider\n InstanceManagerClass = getattr(import_module('spotty.providers.%s.instance_manager' % provider_name),\n 'InstanceManager')\n\n return InstanceManagerClass(project_config, instance_config)\n" }, { "path": "spotty/providers/local/__init__.py", "content": "" }, { "path": "spotty/providers/local/config/__init__.py", "content": "" }, { "path": "spotty/providers/local/config/instance_config.py", "content": "import os\nfrom typing import List\nfrom spotty.config.abstract_instance_config import AbstractInstanceConfig, VolumeMount\nfrom spotty.config.abstract_instance_volume import AbstractInstanceVolume\nfrom spotty.config.container_config import PROJECT_VOLUME_MOUNT_NAME\nfrom spotty.config.project_config import ProjectConfig\nfrom spotty.config.host_path_volume import HostPathVolume\nfrom spotty.providers.local.config.validation import validate_instance_parameters\n\n\nclass InstanceConfig(AbstractInstanceConfig):\n\n def __init__(self, instance_config: dict, project_config: ProjectConfig):\n super().__init__(instance_config, project_config)\n\n def _validate_instance_params(self, params: dict):\n # validate the config and fill missing parameters with the default values\n return validate_instance_parameters(params)\n\n def _get_instance_volumes(self) -> List[AbstractInstanceVolume]:\n volumes = []\n for volume_config in self._params['volumes']:\n volume_type = volume_config['type']\n if volume_type == HostPathVolume.TYPE_NAME:\n volumes.append(HostPathVolume(volume_config, self.project_config.project_dir))\n else:\n raise ValueError('Volume type \"%s\" is not supported.' % volume_type)\n\n return volumes\n\n def _get_volume_mounts(self, volumes: List[AbstractInstanceVolume]) -> List[VolumeMount]:\n volume_mounts = super()._get_volume_mounts(volumes)\n\n # ignore a volume that matches the container project directory\n volume_mounts = [volume_mount for volume_mount in volume_mounts\n if os.path.relpath(self.container_config.project_dir, volume_mount.mount_path) != '.']\n\n # mount the local project directory to the container\n volume_mounts.append(VolumeMount(\n name=PROJECT_VOLUME_MOUNT_NAME,\n host_path=self.project_config.project_dir,\n mount_path=self.container_config.project_dir,\n mode='rw',\n hidden=True,\n ))\n\n return volume_mounts\n\n @property\n def user(self) -> str:\n return ''\n" }, { "path": "spotty/providers/local/config/validation.py", "content": "from spotty.config.validation import validate_config, get_instance_parameters_schema\n\n\ndef validate_instance_parameters(params: dict):\n from spotty.config.host_path_volume import HostPathVolume\n\n schema = get_instance_parameters_schema({}, HostPathVolume.TYPE_NAME)\n\n return validate_config(schema, params)\n" }, { "path": "spotty/providers/local/instance_manager.py", "content": "from spotty.commands.writers.abstract_output_writrer import AbstractOutputWriter\nfrom spotty.errors.nothing_to_do import NothingToDoError\nfrom spotty.deployment.abstract_docker_instance_manager import AbstractDockerInstanceManager\nfrom spotty.providers.local.config.instance_config import InstanceConfig\n\n\nclass InstanceManager(AbstractDockerInstanceManager):\n\n instance_config: InstanceConfig\n\n def _get_instance_config(self, instance_config: dict) -> InstanceConfig:\n \"\"\"Validates the instance config and returns an InstanceConfig object.\"\"\"\n return InstanceConfig(instance_config, self.project_config)\n\n def is_running(self):\n return True\n\n def clean(self, output: AbstractOutputWriter):\n pass\n\n def sync(self, output: AbstractOutputWriter, dry_run=False):\n raise NothingToDoError('Nothing to do. The project directory is mounted to the container.')\n\n def download(self, download_filters: list, output: AbstractOutputWriter, dry_run=False):\n raise NothingToDoError('Nothing to do. The project directory is mounted to the container.')\n" }, { "path": "spotty/providers/remote/__init__.py", "content": "" }, { "path": "spotty/providers/remote/config/__init__.py", "content": "" }, { "path": "spotty/providers/remote/config/instance_config.py", "content": "import os\nfrom typing import List\nfrom spotty.config.abstract_instance_config import AbstractInstanceConfig\nfrom spotty.config.abstract_instance_volume import AbstractInstanceVolume\nfrom spotty.config.project_config import ProjectConfig\nfrom spotty.config.host_path_volume import HostPathVolume\nfrom spotty.providers.remote.config.validation import validate_instance_parameters\n\n\nclass InstanceConfig(AbstractInstanceConfig):\n\n def __init__(self, instance_config: dict, project_config: ProjectConfig):\n super().__init__(instance_config, project_config)\n\n def _validate_instance_params(self, params: dict):\n # validate the config and fill missing parameters with the default values\n return validate_instance_parameters(params)\n\n @property\n def user(self) -> str:\n return self._params['user']\n\n @property\n def host(self) -> str:\n return self._params['host']\n\n @property\n def port(self) -> int:\n return self._params['port']\n\n @property\n def key_path(self) -> str:\n key_path = os.path.expanduser(self._params['keyPath'])\n if not os.path.isabs(key_path):\n key_path = os.path.join(self.project_config.project_dir, key_path)\n\n key_path = os.path.normpath(key_path)\n\n return key_path\n\n def _get_instance_volumes(self) -> List[AbstractInstanceVolume]:\n volumes = []\n for volume_config in self._params['volumes']:\n volume_type = volume_config['type']\n if volume_type == HostPathVolume.TYPE_NAME:\n volumes.append(HostPathVolume(volume_config))\n else:\n raise ValueError('Volume type \"%s\" is not supported.' % volume_type)\n\n return volumes\n" }, { "path": "spotty/providers/remote/config/validation.py", "content": "from schema import And, Optional\nfrom spotty.config.validation import validate_config, get_instance_parameters_schema\n\n\ndef validate_instance_parameters(params: dict):\n from spotty.config.host_path_volume import HostPathVolume\n\n instance_parameters = {\n 'user': str,\n 'host': str,\n Optional('port', default=22): And(int, lambda x: 0 < x < 65536),\n 'keyPath': str,\n }\n\n schema = get_instance_parameters_schema(instance_parameters, HostPathVolume.TYPE_NAME)\n\n return validate_config(schema, params)\n" }, { "path": "spotty/providers/remote/helpers/rsync.py", "content": "from shutil import which\nfrom typing import List\nfrom spotty.deployment.utils.cli import shlex_join\n\n\ndef check_rsync_installed():\n \"\"\"Checks that rsync is installed.\"\"\"\n if which('rsync') is None:\n raise ValueError('rsync is not installed.')\n\n\ndef get_upload_command(local_dir: str, remote_dir: str, ssh_user: str, ssh_host: str, ssh_port: int,\n ssh_key_path: str, filters: List[dict] = None, use_sudo: bool = False, dry_run: bool = False):\n # make sure there is only one list of exclude filters\n if (len(filters) > 1) or (len(filters[0]) > 1) or ('include' in filters[0]):\n raise ValueError('At the moment \"remote\" provider supports only one list of exclude filters.')\n\n remote_path = '%s@%s:%s' % (ssh_user, ssh_host, remote_dir)\n\n return _get_rsync_command(local_dir, remote_path, ssh_port, ssh_key_path, filters, mkdir=remote_dir,\n use_sudo=use_sudo, dry_run=dry_run)\n\n\ndef get_download_command(remote_dir: str, local_dir: str, ssh_user: str, ssh_host: str, ssh_port: int,\n ssh_key_path: str, filters: List[dict] = None, use_sudo: bool = False, dry_run: bool = False):\n filters = filters[::-1]\n remote_path = '%s@%s:%s' % (ssh_user, ssh_host, remote_dir)\n\n return _get_rsync_command(remote_path, local_dir, ssh_port, ssh_key_path, filters, use_sudo=use_sudo,\n dry_run=dry_run)\n\n\ndef _get_rsync_command(src_path: str, dst_path: str, ssh_port: int, ssh_key_path: str, filters: List[dict] = None,\n mkdir: str = None, use_sudo: bool = False, dry_run: bool = False):\n\n sudo_str = 'sudo ' if use_sudo else ''\n remote_rsync_cmd = sudo_str + 'rsync'\n if mkdir:\n remote_rsync_cmd = '%smkdir -p \\'%s\\' && %s' % (sudo_str, mkdir, remote_rsync_cmd)\n\n rsync_cmd = 'rsync -av ' \\\n '--no-owner ' \\\n '--no-group ' \\\n '--prune-empty-dirs ' \\\n '-e \"ssh -i \\'%s\\' -p %d -o StrictHostKeyChecking=no -o ConnectTimeout=10\" ' \\\n '--rsync-path=\"%s\"' \\\n % (ssh_key_path, ssh_port, remote_rsync_cmd)\n\n if dry_run:\n rsync_cmd += ' --dry-run'\n\n if filters:\n args = []\n for sync_filter in filters:\n if 'exclude' in sync_filter:\n for path in sync_filter['exclude']:\n args += ['--exclude', _fix_filter_path(path)]\n\n if 'include' in sync_filter:\n for path in sync_filter['include']:\n args += ['--include', _fix_filter_path(path)]\n\n rsync_cmd += ' ' + shlex_join(args)\n\n rsync_cmd += ' %s/ %s' % (src_path.rstrip('/'), dst_path)\n\n return rsync_cmd\n\n\ndef _fix_filter_path(path: str) -> str:\n return '/' + path.replace('*', '**').lstrip('/')\n" }, { "path": "spotty/providers/remote/instance_manager.py", "content": "import logging\nimport subprocess\nfrom spotty.commands.writers.abstract_output_writrer import AbstractOutputWriter\nfrom spotty.deployment.abstract_ssh_instance_manager import AbstractSshInstanceManager\nfrom spotty.providers.remote.config.instance_config import InstanceConfig\nfrom spotty.providers.remote.helpers.rsync import get_upload_command, check_rsync_installed, get_download_command\n\n\nclass InstanceManager(AbstractSshInstanceManager):\n\n instance_config: InstanceConfig\n\n def _get_instance_config(self, instance_config: dict) -> InstanceConfig:\n \"\"\"Validates the instance config and returns an InstanceConfig object.\"\"\"\n return InstanceConfig(instance_config, self.project_config)\n\n def is_running(self):\n \"\"\"Assuming the remote instance is running.\"\"\"\n return True\n\n def clean(self, output: AbstractOutputWriter):\n pass\n\n def sync(self, output: AbstractOutputWriter, dry_run=False):\n\n output.write('Syncing files with the instance...')\n\n # check rsync is installed\n check_rsync_installed()\n\n # sync the project with the instance\n rsync_cmd = get_upload_command(\n local_dir=self.project_config.project_dir,\n remote_dir=self.instance_config.host_project_dir,\n ssh_user=self.ssh_user,\n ssh_host=self.ssh_host,\n ssh_key_path=self.ssh_key_path,\n ssh_port=self.ssh_port,\n filters=self.project_config.sync_filters,\n use_sudo=(not self.instance_config.container_config.run_as_host_user),\n dry_run=dry_run,\n )\n\n # execute the command locally\n logging.debug('rsync command: ' + rsync_cmd)\n exit_code = subprocess.call(rsync_cmd, shell=True)\n if exit_code != 0:\n raise ValueError('Failed to upload files to the instance.')\n\n def download(self, download_filters: list, output: AbstractOutputWriter, dry_run=False):\n\n output.write('Downloading files from the instance...')\n\n # check rsync is installed\n check_rsync_installed()\n\n # sync the project with the instance\n rsync_cmd = get_download_command(\n local_dir=self.project_config.project_dir,\n remote_dir=self.instance_config.host_project_dir,\n ssh_user=self.ssh_user,\n ssh_host=self.ssh_host,\n ssh_key_path=self.ssh_key_path,\n ssh_port=self.ssh_port,\n filters=download_filters,\n use_sudo=(not self.instance_config.container_config.run_as_host_user),\n dry_run=dry_run,\n )\n\n # execute the command locally\n logging.debug('rsync command: ' + rsync_cmd)\n exit_code = subprocess.call(rsync_cmd, shell=True)\n if exit_code != 0:\n raise ValueError('Failed to download files from the instance.')\n\n @property\n def ssh_host(self) -> str:\n return self.instance_config.host\n\n @property\n def ssh_key_path(self) -> str:\n return self.instance_config.key_path\n\n @property\n def ssh_port(self) -> int:\n return self.instance_config.port\n" }, { "path": "spotty/utils.py", "content": "import os\nimport random\nimport string\nimport errno\n\n\ndef package_dir(path: str = ''):\n \"\"\"Returns an absolute path to the \"spotty\" package directory.\n\n Args:\n path: A relative path to add to the package path.\n\n \"\"\"\n res_path = os.path.dirname(os.path.abspath(__file__))\n if path:\n res_path = os.path.join(res_path, path)\n\n return res_path\n\n\ndef check_path(path):\n \"\"\"Creates a directory if it doesn't exist.\"\"\"\n if not os.path.exists(path):\n try:\n os.makedirs(path)\n except OSError as exception:\n if exception.errno != errno.EEXIST:\n raise\n\n\ndef random_string(length: int, chars: str = string.ascii_lowercase + string.digits):\n return ''.join(random.choice(chars) for _ in range(length))\n\n\ndef filter_list(list_of_dicts, key_name, value):\n return [row for row in list_of_dicts if row[key_name] == value]\n\n\ndef render_table(table: list, separate_title=False):\n column_lengths = [max([len(str(row[i])) for row in table]) for i in range(len(table[0]))]\n row_separator = '+-%s-+' % '-+-'.join(['-' * col_length for col_length in column_lengths])\n title_separator = '+=%s=+' % '=+='.join(['=' * col_length for col_length in column_lengths])\n\n lines = [row_separator]\n for i, row in enumerate(table):\n line = '| %s |' % ' | '.join([str(val).ljust(col_length) for val, col_length in zip(row, column_lengths)])\n lines.append(line)\n lines.append(title_separator if separate_title and not i else row_separator)\n\n return '\\n'.join(lines)\n" }, { "path": "tests/__init__.py", "content": "" }, { "path": "tests/container_config.py", "content": "import unittest\nfrom spotty.config.container_config import ContainerConfig\n\n\nclass TestContainerConfig(unittest.TestCase):\n\n def test_working_dir(self):\n container_config = ContainerConfig({\n 'projectDir': '/workspace/project',\n 'workingDir': '',\n })\n\n self.assertEqual(container_config.project_dir, '/workspace/project')\n self.assertEqual(container_config.working_dir, '/workspace/project')\n\n container_config = ContainerConfig({\n 'projectDir': '/workspace/project',\n 'workingDir': '/working-dir',\n })\n\n self.assertEqual(container_config.project_dir, '/workspace/project')\n self.assertEqual(container_config.working_dir, '/working-dir')\n\n\nif __name__ == '__main__':\n unittest.main()\n" }, { "path": "tests/helpers/__init__.py", "content": "" }, { "path": "tests/helpers/cli.py", "content": "import shlex\nimport subprocess\n\n\ndef run(command: str, capture_output: bool = False, assert_zero_code: bool = True) -> (int, str):\n # run the command\n stdout = subprocess.PIPE if capture_output else None\n res = subprocess.run(command, stdout=stdout, shell=True)\n\n # make sure the command is succeed\n if assert_zero_code:\n assert res.returncode == 0, 'Command \"%s\" is failed' % command\n\n # decode output\n output = res.stdout.decode('utf-8') if capture_output else None\n\n return res.returncode, output\n\n\ndef touch_file(file_path: str):\n run('touch ' + shlex.quote(file_path))\n" }, { "path": "tests/helpers/spotty_cli.py", "content": "import os\nimport shlex\nfrom typing import List\nfrom tests.helpers.cli import run\n\n\nclass SpottyCli:\n\n def __init__(self, instance_name: str):\n self._instance_name = instance_name\n\n def is_instance_running(self) -> bool:\n \"\"\"Checks whether the instance is running or not.\"\"\"\n exit_code, _ = run('spotty status ' + self._instance_name, capture_output=True, assert_zero_code=False)\n return exit_code == 0\n\n def start_instance(self):\n \"\"\"Starts an instance.\"\"\"\n if not self.is_instance_running():\n # start instance\n run('spotty start ' + self._instance_name)\n\n def list_remote_files(self) -> List[str]:\n \"\"\"Returns a list of files in the project directory on the remote machine.\"\"\"\n output = self.exec('find . -type f -print')\n remote_files = [os.path.normpath(file_path) for file_path in output.splitlines()]\n\n return remote_files\n\n def touch_file(self, file_path: str):\n \"\"\"Returns a list of files in the project directory on the remote machine.\"\"\"\n self.exec('touch ' + shlex.quote(file_path))\n\n def sync(self):\n \"\"\"Syncs files with the remote instance.\"\"\"\n _, output = run('spotty sync ' + self._instance_name, capture_output=True)\n\n uploaded_files = []\n downloaded_files = []\n for line in output.splitlines():\n if line.startswith('upload:'):\n uploaded_files.append(os.path.normpath(line.split()[1]))\n elif line.startswith('download:'):\n downloaded_files.append(line.split()[3].rsplit('.project/')[-1])\n\n return uploaded_files, downloaded_files\n\n def download(self, filter_pattern: str):\n \"\"\"Syncs files with the remote instance.\"\"\"\n _, output = run('spotty download %s -i %s' % (self._instance_name, shlex.quote(filter_pattern)),\n capture_output=True)\n\n print('---')\n print(output)\n print('---')\n\n uploaded_files = []\n downloaded_files = []\n for line in output.splitlines():\n if line.startswith('upload:'):\n uploaded_files.append(os.path.normpath(line.split()[1]))\n elif line.startswith('download:'):\n downloaded_files.append(line.split()[3].rsplit('.project/')[-1])\n\n return uploaded_files, downloaded_files\n\n def exec(self, container_command: str):\n \"\"\"Execs a custom command in the container.\"\"\"\n _, output = run('spotty exec %s --no-sync -- %s' % (self._instance_name, container_command),\n capture_output=True)\n\n return output\n" }, { "path": "tests/providers/__init__.py", "content": "" }, { "path": "tests/providers/aws/__init__.py", "content": "" }, { "path": "tests/providers/aws/commands/data/test-project/ignored-dir/ignored-file", "content": "" }, { "path": "tests/providers/aws/commands/data/test-project/ignored-dir/included-file", "content": "" }, { "path": "tests/providers/aws/commands/data/test-project/ignored-file", "content": "" }, { "path": "tests/providers/aws/commands/data/test-project/local-file", "content": "" }, { "path": "tests/providers/aws/commands/data/test-project/spotty.yaml", "content": "project:\n name: test-project\n syncFilters:\n - exclude:\n - ignored-dir/*\n - ignored-file\n - include:\n - ignored-dir/included-file\n\ncontainers:\n - projectDir: /workspace/project\n image: ubuntu:16.04\n\ninstances:\n - name: aws-1\n provider: aws\n parameters:\n region: us-east-2\n instanceType: t2.small\n" }, { "path": "tests/providers/aws/commands/download.py", "content": "import os\nimport unittest\nfrom tests.helpers.cli import touch_file\nfrom tests.helpers.spotty_cli import SpottyCli\n\n\nclass TestInstanceDownload(unittest.TestCase):\n\n spotty = SpottyCli('aws-1')\n\n @classmethod\n def setUpClass(cls):\n # set local project directory\n project_dir = os.path.join(os.path.dirname(__file__), 'data', 'test-project')\n os.chdir(project_dir)\n\n # make sure the instance is running\n assert cls.spotty.is_instance_running()\n\n # make sure all files are synced\n local_files = sorted([\n 'ignored-dir/included-file',\n 'local-file',\n 'spotty.yaml',\n ])\n\n remote_files = cls.spotty.list_remote_files()\n assert set(local_files).issubset(set(remote_files))\n\n def test_download_file(self):\n # touch the remote file\n self.spotty.touch_file('local-file')\n\n # download the file\n uploaded_files, downloaded_files = self.spotty.download('local-file')\n\n # only 1 file should be uploaded and downloaded\n self.assertEqual(len(uploaded_files), 1)\n self.assertEqual(len(downloaded_files), 1)\n\n # the updated file uploaded\n self.assertIn('local-file', uploaded_files)\n self.assertIn('local-file', downloaded_files)\n\n # download the file again\n uploaded_files, downloaded_files = self.spotty.download('local-file')\n\n # no files should be uploaded or downloaded\n self.assertFalse(uploaded_files)\n self.assertFalse(downloaded_files)\n\n # touch the remote file, then touch the local file\n self.spotty.touch_file('local-file')\n touch_file('local-file')\n\n # download the remote file again\n uploaded_files, downloaded_files = self.spotty.download('local-file')\n\n # the file should be uploaded, but not downloaded as the local file is newer than the remote one\n self.assertEqual(len(uploaded_files), 1)\n self.assertFalse(downloaded_files)\n self.assertIn('local-file', uploaded_files)\n\n def test_wildcard(self):\n # touch remote files\n self.spotty.touch_file('ignored-dir/ignored-file')\n self.spotty.touch_file('ignored-dir/included-file')\n\n # download the files\n uploaded_files, downloaded_files = self.spotty.download('ignored-dir/*')\n\n # 2 files should be uploaded and downloaded\n self.assertEqual(len(uploaded_files), 2)\n self.assertEqual(len(downloaded_files), 2)\n self.assertIn('ignored-dir/ignored-file', uploaded_files)\n self.assertIn('ignored-dir/ignored-file', downloaded_files)\n self.assertIn('ignored-dir/included-file', uploaded_files)\n self.assertIn('ignored-dir/included-file', downloaded_files)\n\n # touch one of the remote files\n self.spotty.touch_file('ignored-dir/ignored-file')\n\n # download the files\n uploaded_files, downloaded_files = self.spotty.download('ignored-dir/*')\n\n # 1 file should be uploaded and downloaded\n self.assertEqual(len(uploaded_files), 1)\n self.assertEqual(len(downloaded_files), 1)\n self.assertIn('ignored-dir/ignored-file', uploaded_files)\n self.assertIn('ignored-dir/ignored-file', downloaded_files)\n\n\nif __name__ == '__main__':\n unittest.main()\n" }, { "path": "tests/providers/aws/commands/sync.py", "content": "import os\nimport unittest\nfrom tests.helpers.cli import touch_file\nfrom tests.helpers.spotty_cli import SpottyCli\n\n\nclass TestInstanceSync(unittest.TestCase):\n\n spotty = SpottyCli('aws-1')\n\n @classmethod\n def setUpClass(cls):\n # set local project directory\n project_dir = os.path.join(os.path.dirname(__file__), 'data', 'test-project')\n os.chdir(project_dir)\n\n # start AWS instance\n cls.spotty.start_instance()\n\n # make sure all files are synced\n local_files = sorted([\n 'ignored-dir/included-file',\n 'local-file',\n 'spotty.yaml',\n ])\n\n remote_files = cls.spotty.list_remote_files()\n assert set(local_files).issubset(set(remote_files))\n\n def test_update_local_file(self):\n # touch local file\n touch_file('ignored-dir/included-file')\n touch_file('ignored-dir/ignored-file')\n\n # sync files with the remote instance\n uploaded_files, downloaded_files = self.spotty.sync()\n\n # only 1 file should be uploaded and downloaded\n self.assertEqual(len(uploaded_files), 1)\n self.assertEqual(len(downloaded_files), 1)\n\n # the updated file uploaded\n self.assertIn('ignored-dir/included-file', uploaded_files)\n self.assertIn('ignored-dir/included-file', downloaded_files)\n\n # the untouched file not uploaded\n self.assertNotIn('local-file', uploaded_files)\n self.assertNotIn('local-file', downloaded_files)\n\n # the ignored file not uploaded\n self.assertNotIn('ignored-dir/ignored-file', uploaded_files)\n self.assertNotIn('ignored-dir/ignored-file', downloaded_files)\n\n def test_update_remote_file(self):\n # touch remote files\n self.spotty.touch_file('local-file')\n self.spotty.touch_file('ignored-dir/ignored-file')\n\n # sync files with the remote instance\n uploaded_files, downloaded_files = self.spotty.sync()\n\n # local files were not changed, so should not be uploaded\n self.assertFalse(uploaded_files, 'No files should be uploaded')\n\n # the remote file that is newer still will be overwritten with the older file\n # from the bucket (this is the current \"aws s3 sync\" behaviour)\n self.assertIn('local-file', downloaded_files)\n self.assertEqual(len(downloaded_files), 1)\n\n # the ignored file should not be overwritten\n self.assertNotIn('ignored-dir/ignored-file', downloaded_files)\n\n def test_new_remote_file(self):\n # create new remote file\n self.spotty.touch_file('new-remote-file')\n\n # make sure file was created\n remote_files = self.spotty.list_remote_files()\n self.assertIn('new-remote-file', remote_files)\n\n # make sure the file with this name doesn't exist locally\n self.assertFalse(os.path.isfile('new-remote-file'))\n\n # sync files\n self.spotty.sync()\n\n # make sure the file wasn't delete on the remote machine\n remote_files = self.spotty.list_remote_files()\n self.assertIn('new-remote-file', remote_files)\n\n\nif __name__ == '__main__':\n unittest.main()\n" }, { "path": "tests/providers/aws/config/__init__.py", "content": "" }, { "path": "tests/providers/aws/config/container_deployment.py", "content": "import os\nimport unittest\nfrom spotty.config.abstract_instance_config import VolumeMount\nfrom spotty.config.config_utils import _read_yaml\nfrom spotty.config.project_config import ProjectConfig\nfrom spotty.providers.aws.config.instance_config import InstanceConfig\n\n\nclass TestContainerDeployment(unittest.TestCase):\n\n def test_instance_volume(self):\n local_project_dir = os.path.join(os.path.dirname(__file__), 'data')\n config = _read_yaml(os.path.join(local_project_dir, 'config1.yaml'))\n\n project_config = ProjectConfig(config, local_project_dir)\n instance_config = InstanceConfig(project_config.instances[0], project_config)\n\n self.assertEqual(instance_config.host_project_dir, '/mnt/test/project')\n self.assertEqual(instance_config.dockerfile_path, '/mnt/test/project/docker/Dockerfile')\n self.assertEqual(instance_config.docker_context_path, '/mnt/test/project/docker')\n self.assertEqual(len(instance_config.volume_mounts), 2)\n self.assertEqual(instance_config.volume_mounts[0], VolumeMount(name='workspace',\n host_path='/mnt/test',\n mount_path='/workspace',\n mode='rw',\n hidden=False))\n self.assertEqual(instance_config.volume_mounts[1], VolumeMount(name=None,\n host_path='/root/.aws',\n mount_path='/root/.aws',\n mode='ro',\n hidden=True,))\n\n def test_tmp_project_volume(self):\n local_project_dir = os.path.join(os.path.dirname(__file__), 'data')\n config = _read_yaml(os.path.join(local_project_dir, 'config-wo-mounts.yaml'))\n\n project_config = ProjectConfig(config, local_project_dir)\n instance_config = InstanceConfig(project_config.instances[0], project_config)\n\n host_project_dir = '/tmp/spotty/containers/spotty-my-project-aws-1-default/volumes/.project'\n\n self.assertEqual(instance_config.host_project_dir, host_project_dir)\n self.assertEqual(instance_config.dockerfile_path, os.path.join(host_project_dir, 'docker', 'Dockerfile'))\n self.assertEqual(instance_config.docker_context_path, os.path.join(host_project_dir, 'docker'))\n self.assertEqual(len(instance_config.volume_mounts), 1)\n self.assertEqual(instance_config.volume_mounts[0], VolumeMount(name=None,\n host_path=host_project_dir,\n mount_path='/workspace/project',\n mode='rw',\n hidden=True))\n\n\nif __name__ == '__main__':\n unittest.main()\n" }, { "path": "tests/providers/aws/config/data/config-wo-mounts.yaml", "content": "project:\n name: my-project\n\ncontainers:\n - projectDir: /workspace/project\n file: docker/Dockerfile\n\ninstances:\n - name: aws-1\n provider: aws\n parameters:\n region: us-east-2\n instanceType: t2.small\n" }, { "path": "tests/providers/aws/config/data/config1.yaml", "content": "project:\n name: my-project\n syncFilters:\n - exclude:\n - .git/*\n - .idea/*\n - '*/__pycache__/*'\n\ncontainer:\n projectDir: /workspace/project\n file: docker/Dockerfile\n volumeMounts:\n - name: workspace\n mountPath: /workspace\n commands: |\n echo test\n\ninstances:\n - name: aws-1\n provider: aws\n parameters:\n region: us-east-2\n instanceType: t2.small\n volumes:\n - name: workspace\n parameters:\n size: 10\n deletionPolicy: Delete\n mountDir: /mnt/test\n" }, { "path": "tests/providers/aws/config/instance_config_validation.py", "content": "import unittest\nfrom spotty.providers.aws.config.validation import validate_instance_parameters\n\n\nclass TestBucketResource(unittest.TestCase):\n\n def test_default_configuration(self):\n \"\"\"Checks the default values for an instance configuration are set correctly.\"\"\"\n required_params = {\n 'region': 'eu-west-1',\n 'instanceType': 'p2.xlarge',\n }\n\n expected_params = {\n **required_params,\n 'amiId': None,\n 'amiName': None,\n 'availabilityZone': '',\n 'commands': '',\n 'containerName': None,\n 'dockerDataRoot': '',\n 'instanceProfileArn': None,\n 'localSshPort': None,\n 'managedPolicyArns': [],\n 'maxPrice': 0,\n 'ports': [],\n 'rootVolumeSize': 0,\n 'spotInstance': False,\n 'subnetId': '',\n 'volumes': [],\n }\n\n self.assertEqual(expected_params, validate_instance_parameters(required_params))\n\n def test_failed_validation(self):\n # no params\n with self.assertRaises(ValueError):\n validate_instance_parameters({})\n\n # wrong case for the region\n with self.assertRaises(ValueError):\n validate_instance_parameters({\n 'region': 'EU-WEST-1',\n 'instanceType': 'p2.xlarge',\n })\n\n # unknown parameter\n with self.assertRaises(ValueError):\n validate_instance_parameters({\n 'region': 'eu-west-1',\n 'instanceType': 'p2.xlarge',\n 'unknownParameter': 'test',\n })\n\n\nif __name__ == '__main__':\n unittest.main()\n" }, { "path": "tests/providers/aws/project_resources/__init__.py", "content": "" }, { "path": "tests/providers/aws/project_resources/bucket.py", "content": "import unittest\nfrom spotty.deployment.abstract_cloud_instance.errors.bucket_not_found import BucketNotFoundError\nfrom spotty.providers.aws.resource_managers.bucket_manager import BucketManager\nfrom moto import mock_s3\n\n\nclass TestBucketResource(unittest.TestCase):\n\n @mock_s3\n def test_create_and_find_bucket(self):\n region = 'eu-central-1'\n project_name = 'TEST_PROJECT'\n bucket_resource = BucketManager(project_name, region)\n\n # bucket not found\n with self.assertRaises(BucketNotFoundError):\n bucket_resource.get_bucket()\n\n # bucket found\n bucket_name = bucket_resource.create_bucket().name\n self.assertEqual(bucket_name, bucket_resource.get_bucket().name)\n\n # several buckets found\n bucket_resource.create_bucket()\n with self.assertRaises(ValueError):\n bucket_resource.get_bucket()\n\n\nif __name__ == '__main__':\n unittest.main()\n" }, { "path": "tests/providers/aws/project_resources/key_pair.py", "content": "import unittest\nimport boto3\nimport os\nfrom moto import mock_ec2\nfrom spotty.providers.aws.resource_managers.key_pair_manager import KeyPairManager\nfrom spotty.providers.instance_manager_factory import PROVIDER_AWS\n\n\nclass TestKeyPairResource(unittest.TestCase):\n\n def test_key_path(self):\n region = 'eu-central-1'\n project_name = 'TEST_PROJECT'\n key_pair_manager = KeyPairManager(None, project_name, region)\n\n # check key path\n key_name = 'spotty-key-%s-%s' % (project_name.lower(), region)\n key_path = os.path.join(os.path.expanduser('~'), '.spotty', 'keys', PROVIDER_AWS, key_name)\n self.assertEqual(key_pair_manager.key_path, key_path)\n\n @mock_ec2\n def test_create_and_delete_key(self):\n region = 'eu-central-1'\n project_name = 'TEST_PROJECT'\n ec2 = boto3.client('ec2', region_name=region)\n key_pair_manager = KeyPairManager(ec2, project_name, region)\n\n # key doesn't exist\n self.assertFalse(key_pair_manager._ec2_key_exists())\n\n # create the key\n key_pair_manager.maybe_create_key()\n self.assertTrue(key_pair_manager._ec2_key_exists())\n self.assertTrue(os.path.isfile(key_pair_manager.key_path))\n with open(key_pair_manager.key_path) as f:\n key_content = f.read()\n\n # make sure the key is not being recreated\n key_pair_manager.maybe_create_key()\n with open(key_pair_manager.key_path) as f:\n same_key_content = f.read()\n self.assertEqual(key_content, same_key_content)\n\n # create the key and rewrite the key file\n ec2.delete_key_pair(KeyName=key_pair_manager.key_name)\n self.assertFalse(key_pair_manager._ec2_key_exists())\n self.assertTrue(os.path.isfile(key_pair_manager.key_path))\n key_pair_manager.maybe_create_key()\n self.assertTrue(key_pair_manager._ec2_key_exists())\n self.assertTrue(os.path.isfile(key_pair_manager.key_path))\n with open(key_pair_manager.key_path) as f:\n new_key_content = f.read()\n\n self.assertNotEqual(key_content, new_key_content)\n\n # recreate the key if the key file doesn't exist\n os.unlink(key_pair_manager.key_path)\n self.assertFalse(os.path.isfile(key_pair_manager.key_path))\n key_pair_manager.maybe_create_key()\n self.assertTrue(key_pair_manager._ec2_key_exists())\n self.assertTrue(os.path.isfile(key_pair_manager.key_path))\n\n # delete key\n key_pair_manager.delete_key()\n self.assertFalse(key_pair_manager._ec2_key_exists())\n self.assertFalse(os.path.isfile(key_pair_manager.key_path))\n\n\nif __name__ == '__main__':\n unittest.main()\n" }, { "path": "tests/providers/gcp/config/__init__.py", "content": "" }, { "path": "tests/providers/gcp/config/image_uri.py", "content": "import unittest\nfrom spotty.providers.gcp.config.image_uri import ImageUri\n\n\nclass TestImageUrl(unittest.TestCase):\n\n def test_image_url_parsing(self):\n pos_tests = [\n {\n 'uri': 'projects/debian-cloud/global/images/family/debian-9',\n 'expected': ('debian-cloud', True, 'debian-9'),\n },\n {\n 'uri': 'projects/debian-cloud/global/images/debian-9-stretch',\n 'expected': ('debian-cloud', False, 'debian-9-stretch'),\n },\n {\n 'uri': 'global/images/family/my-image-family',\n 'expected': (None, True, 'my-image-family'),\n },\n {\n 'uri': 'global/images/my-custom-image',\n 'expected': (None, False, 'my-custom-image'),\n },\n {\n 'uri': 'https://compute.googleapis.com/compute/v1/projects/debian-cloud/global/images/debian-9-stretch',\n 'expected': ('debian-cloud', False, 'debian-9-stretch'),\n },\n ]\n\n for pos_test in pos_tests:\n image_uri = ImageUri(pos_test['uri'])\n self.assertEqual(pos_test['expected'], (image_uri.project_id, image_uri.is_family, image_uri.name))\n\n neg_tests = [\n 'projects//global/images/family/debian-9', # no project\n 'projects/test1/test2/global/images/family/debian-9', # extra part\n 'projects/debian-cloud/global/image/family/debian-9', # \"image\" misspelling\n 'projects/debian-cloud/global/images/family/Debian-9', # capital letter\n 'projects/debian-cloud/global/images/' # no image name\n '/global/images/family/debian-9', # starts with a slash\n 'global/images/family/debian-9/', # ends with a slash\n 'global/images/-my-custom-image', # image name starts with a dash\n 'global/images/my-custom-image-', # image name ends with a dash\n 'https://compute.googleapis.com/compute/v1/global/images/debian-9-stretch', # no project name\n ]\n\n for neg_test in neg_tests:\n with self.assertRaises(ValueError):\n ImageUri(neg_test)\n\n\nif __name__ == '__main__':\n unittest.main()\n" }, { "path": "tests/providers/local/__init__.py", "content": "" }, { "path": "tests/providers/local/commands/__init__.py", "content": "" }, { "path": "tests/providers/local/commands/data/test-project/spotty.yaml", "content": "project:\n name: test-project\n\ncontainers:\n - projectDir: /workspace/project\n image: ubuntu:16.04\n\ninstances:\n - name: local-1\n provider: local\n" }, { "path": "tests/providers/local/commands/run.py", "content": "import os\nimport unittest\nfrom spotty.deployment.utils.commands import get_script_command\nfrom spotty.deployment.utils.user_scripts import render_script\nfrom tests.helpers.spotty_cli import SpottyCli\n\n\nclass TestInstanceRun(unittest.TestCase):\n\n spotty = SpottyCli('local-1')\n\n @classmethod\n def setUpClass(cls):\n # set local project directory\n project_dir = os.path.join(os.path.dirname(__file__), 'data', 'test-project')\n os.chdir(project_dir)\n\n # start AWS instance\n cls.spotty.start_instance()\n\n def test_script_arguments(self):\n script_name = 'echo'\n script_content = 'echo test $1 $2'\n\n # no arguments\n script_command = get_script_command(script_name, script_content, script_args=None, logging=False)\n output = self.spotty.exec(script_command)\n self.assertEqual(output.strip(), 'test')\n\n # custom arguments\n script_command = get_script_command(script_name, script_content, script_args=['arg 1', 'arg 2'], logging=False)\n output = self.spotty.exec(script_command)\n self.assertEqual(output.strip(), 'test arg 1 arg 2')\n\n def test_script_params(self):\n\n script_content = 'echo test {{PARAM_1}} {{PARAM_2}}'\n script_params = {\n 'PARAM_2': 'param 2',\n }\n\n script_content = render_script(script_content, script_params)\n\n self.assertEqual(script_content, '#!/usr/bin/env bash\\n\\n'\n 'set -xe\\n\\n'\n 'echo test param 2')\n\n def test_script_logging(self):\n script_name = 'echo'\n script_content = 'echo test'\n\n # run the script with logging\n script_command = get_script_command(script_name, script_content, script_args=None, logging=True)\n output = self.spotty.exec(script_command)\n self.assertEqual(output.strip(), 'test')\n\n # read the latest log file\n output = self.spotty.exec('bash -c \\'cat /var/log/spotty/run/$(ls -rt /var/log/spotty/run | tail -n1)\\'')\n self.assertEqual(output.splitlines()[0], 'test')\n\n\nif __name__ == '__main__':\n unittest.main()\n" }, { "path": "tests/providers/local/config/__init__.py", "content": "" }, { "path": "tests/providers/local/config/container_deployment.py", "content": "import os\nimport unittest\nfrom spotty.config.abstract_instance_config import VolumeMount\nfrom spotty.config.config_utils import _read_yaml\nfrom spotty.config.project_config import ProjectConfig\nfrom spotty.providers.local.config.instance_config import InstanceConfig\n\n\nclass TestContainerDeployment(unittest.TestCase):\n\n def test_instance_volume(self):\n local_project_dir = os.path.join(os.path.dirname(__file__), 'config', 'data')\n config = _read_yaml(os.path.join(local_project_dir, 'config1.yaml'))\n\n project_config = ProjectConfig(config, local_project_dir)\n instance_config = InstanceConfig(project_config.instances[0], project_config)\n\n self.assertEqual(instance_config.host_project_dir, local_project_dir)\n self.assertEqual(instance_config.dockerfile_path, os.path.join(local_project_dir, 'docker', 'Dockerfile'))\n self.assertEqual(instance_config.docker_context_path, os.path.join(local_project_dir, 'docker'))\n self.assertEqual(len(instance_config.volume_mounts), 2)\n self.assertEqual(instance_config.volume_mounts[0], VolumeMount(name='workspace',\n host_path='/mnt/test',\n mount_path='/workspace',\n mode='rw',\n hidden=False))\n self.assertEqual(instance_config.volume_mounts[1], VolumeMount(name=None,\n host_path=local_project_dir,\n mount_path='/workspace/project',\n mode='rw',\n hidden=True))\n\n\nif __name__ == '__main__':\n unittest.main()\n" }, { "path": "tests/providers/local/config/data/config1.yaml", "content": "project:\n name: my-project\n syncFilters:\n - exclude:\n - .git/*\n - .idea/*\n - '*/__pycache__/*'\n\ncontainers:\n - projectDir: /workspace/project\n file: docker/Dockerfile\n volumeMounts:\n - name: workspace\n mountPath: /workspace\n commands: |\n echo test\n\ninstances:\n - name: local-1\n provider: local\n parameters:\n volumes:\n - name: workspace\n parameters:\n path: /mnt/test\n" } ]