Showing preview only (420K chars total). Download the full file or copy to clipboard to get everything.
Repository: spotty-cloud/spotty
Branch: dev
Commit: bdbacd4e893b
Files: 250
Total size: 358.8 KB
Directory structure:
gitextract_scgum9gk/
├── .github/
│ ├── FUNDING.yml
│ └── workflows/
│ ├── generate-docs.yml
│ └── python-publish.yml
├── .gitignore
├── CONTRIBUTING.md
├── LICENSE
├── README.md
├── bin/
│ └── spotty
├── docs/
│ ├── Makefile
│ ├── make.bat
│ ├── requirements.txt
│ └── source/
│ ├── _static/
│ │ ├── favicon/
│ │ │ ├── browserconfig.xml
│ │ │ └── site.webmanifest
│ │ ├── scripts.js
│ │ └── styles.css
│ ├── conf.py
│ ├── docs/
│ │ ├── cli/
│ │ │ ├── spotty-aws.rst
│ │ │ ├── spotty-download.rst
│ │ │ ├── spotty-exec.rst
│ │ │ ├── spotty-run.rst
│ │ │ ├── spotty-sh.rst
│ │ │ ├── spotty-start.rst
│ │ │ ├── spotty-stop.rst
│ │ │ ├── spotty-sync.rst
│ │ │ └── spotty.rst
│ │ ├── providers/
│ │ │ ├── aws/
│ │ │ │ ├── caching-docker-image-on-an-ebs-volume.md
│ │ │ │ ├── ebs-volumes-and-deletion-policies.md
│ │ │ │ ├── faq.md
│ │ │ │ ├── instance-parameters.md
│ │ │ │ └── overview.rst
│ │ │ ├── gcp/
│ │ │ │ ├── account-preparation.md
│ │ │ │ ├── caching-docker-image-on-a-disk.md
│ │ │ │ ├── disks-and-deletion-policies.md
│ │ │ │ ├── instance-parameters.md
│ │ │ │ └── overview.rst
│ │ │ ├── local/
│ │ │ │ ├── instance-parameters.md
│ │ │ │ └── overview.rst
│ │ │ └── remote/
│ │ │ ├── instance-parameters.md
│ │ │ └── overview.rst
│ │ └── user-guide/
│ │ ├── configuration-file.md
│ │ ├── getting-started.md
│ │ └── installation.md
│ ├── index.rst
│ └── main.html
├── setup.cfg
├── setup.py
├── spotty/
│ ├── __init__.py
│ ├── cli.py
│ ├── commands/
│ │ ├── __init__.py
│ │ ├── abstract_command.py
│ │ ├── abstract_config_command.py
│ │ ├── abstract_provider_command.py
│ │ ├── aws.py
│ │ ├── download.py
│ │ ├── exec.py
│ │ ├── run.py
│ │ ├── sh.py
│ │ ├── start.py
│ │ ├── status.py
│ │ ├── stop.py
│ │ ├── sync.py
│ │ └── writers/
│ │ ├── __init__.py
│ │ ├── abstract_output_writrer.py
│ │ ├── null_output_writrer.py
│ │ └── output_writrer.py
│ ├── config/
│ │ ├── __init__.py
│ │ ├── abstract_instance_config.py
│ │ ├── abstract_instance_volume.py
│ │ ├── config_utils.py
│ │ ├── container_config.py
│ │ ├── host_path_volume.py
│ │ ├── project_config.py
│ │ ├── tmp_dir_volume.py
│ │ └── validation.py
│ ├── configuration.py
│ ├── deployment/
│ │ ├── __init__.py
│ │ ├── abstract_cloud_instance/
│ │ │ ├── __init__.py
│ │ │ ├── abstract_bucket_manager.py
│ │ │ ├── abstract_cloud_instance_manager.py
│ │ │ ├── abstract_data_transfer.py
│ │ │ ├── abstract_instance_deployment.py
│ │ │ ├── errors/
│ │ │ │ ├── __init__.py
│ │ │ │ └── bucket_not_found.py
│ │ │ ├── file_structure.py
│ │ │ └── resources/
│ │ │ ├── __init__.py
│ │ │ ├── abstract_bucket.py
│ │ │ └── abstract_instance.py
│ │ ├── abstract_docker_instance_manager.py
│ │ ├── abstract_instance_manager.py
│ │ ├── abstract_ssh_instance_manager.py
│ │ ├── container/
│ │ │ ├── __init__.py
│ │ │ ├── abstract_container_commands.py
│ │ │ ├── abstract_container_script.py
│ │ │ └── docker/
│ │ │ ├── __init__.py
│ │ │ ├── docker_commands.py
│ │ │ └── scripts/
│ │ │ ├── __init__.py
│ │ │ ├── abstract_docker_script.py
│ │ │ ├── container_bash_script.py
│ │ │ ├── data/
│ │ │ │ ├── container_bash.sh.tpl
│ │ │ │ ├── start_container.sh.tpl
│ │ │ │ └── stop_container.sh.tpl
│ │ │ ├── start_container_script.py
│ │ │ └── stop_container_script.py
│ │ └── utils/
│ │ ├── __init__.py
│ │ ├── cli.py
│ │ ├── commands.py
│ │ ├── print_info.py
│ │ └── user_scripts.py
│ ├── errors/
│ │ ├── __init__.py
│ │ ├── instance_not_running.py
│ │ └── nothing_to_do.py
│ ├── providers/
│ │ ├── __init__.py
│ │ ├── aws/
│ │ │ ├── __init__.py
│ │ │ ├── cfn_templates/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── instance/
│ │ │ │ │ ├── __init__.py
│ │ │ │ │ ├── data/
│ │ │ │ │ │ ├── files/
│ │ │ │ │ │ │ └── tmux.conf
│ │ │ │ │ │ ├── startup_scripts/
│ │ │ │ │ │ │ ├── 01_prepare_instance.sh
│ │ │ │ │ │ │ ├── 02_mount_volumes.sh
│ │ │ │ │ │ │ ├── 03_set_docker_root.sh
│ │ │ │ │ │ │ ├── 04_sync_project.sh
│ │ │ │ │ │ │ ├── 05_run_instance_startup_commands.sh
│ │ │ │ │ │ │ └── user_data.sh
│ │ │ │ │ │ └── template.yaml
│ │ │ │ │ ├── start_container_script.py
│ │ │ │ │ └── template.py
│ │ │ │ └── instance_profile/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── data/
│ │ │ │ │ └── template.yaml
│ │ │ │ └── template.py
│ │ │ ├── commands/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── clean_logs.py
│ │ │ │ └── spot_prices.py
│ │ │ ├── config/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── ebs_volume.py
│ │ │ │ ├── instance_config.py
│ │ │ │ └── validation.py
│ │ │ ├── data_transfer.py
│ │ │ ├── deletion_policies.py
│ │ │ ├── errors/
│ │ │ │ ├── __init__.py
│ │ │ │ └── volume_not_found.py
│ │ │ ├── helpers/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── ami.py
│ │ │ │ ├── availability_zone.py
│ │ │ │ ├── instance_prices.py
│ │ │ │ ├── logs.py
│ │ │ │ ├── s3_sync.py
│ │ │ │ ├── subnet.py
│ │ │ │ └── vpc.py
│ │ │ ├── instance_deployment.py
│ │ │ ├── instance_manager.py
│ │ │ ├── resource_managers/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── bucket_manager.py
│ │ │ │ ├── instance_profile_stack_manager.py
│ │ │ │ ├── instance_stack_manager.py
│ │ │ │ └── key_pair_manager.py
│ │ │ └── resources/
│ │ │ ├── __init__.py
│ │ │ ├── bucket.py
│ │ │ ├── image.py
│ │ │ ├── instance.py
│ │ │ ├── snapshot.py
│ │ │ ├── stack.py
│ │ │ ├── subnet.py
│ │ │ ├── volume.py
│ │ │ └── vpc.py
│ │ ├── gcp/
│ │ │ ├── __init__.py
│ │ │ ├── config/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── disk_volume.py
│ │ │ │ ├── image_uri.py
│ │ │ │ ├── instance_config.py
│ │ │ │ └── validation.py
│ │ │ ├── data_transfer.py
│ │ │ ├── dm_templates/
│ │ │ │ ├── __init__.py
│ │ │ │ └── instance/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── data/
│ │ │ │ │ ├── startup_script.sh.tpl
│ │ │ │ │ ├── startup_scripts/
│ │ │ │ │ │ ├── 01_prepare_instance.sh
│ │ │ │ │ │ ├── 02_mount_volumes.sh
│ │ │ │ │ │ ├── 03_set_docker_root.sh
│ │ │ │ │ │ ├── 04_sync_project.sh
│ │ │ │ │ │ └── 05_run_instance_startup_commands.sh
│ │ │ │ │ └── template.yaml
│ │ │ │ └── instance_template.py
│ │ │ ├── errors/
│ │ │ │ ├── __init__.py
│ │ │ │ └── image_not_found.py
│ │ │ ├── helpers/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── ce_client.py
│ │ │ │ ├── deployment.py
│ │ │ │ ├── dm_client.py
│ │ │ │ ├── dm_resource.py
│ │ │ │ ├── gcp_credentials.py
│ │ │ │ ├── gs_client.py
│ │ │ │ ├── gsutil_rsync.py
│ │ │ │ ├── image.py
│ │ │ │ ├── rtc_client.py
│ │ │ │ └── volumes.py
│ │ │ ├── instance_deployment.py
│ │ │ ├── instance_manager.py
│ │ │ ├── resource_managers/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── bucket_manager.py
│ │ │ │ ├── instance_stack_manager.py
│ │ │ │ └── ssh_key_manager.py
│ │ │ └── resources/
│ │ │ ├── __init__.py
│ │ │ ├── bucket.py
│ │ │ ├── disk.py
│ │ │ ├── image.py
│ │ │ ├── instance.py
│ │ │ ├── snapshot.py
│ │ │ └── stack.py
│ │ ├── instance_manager_factory.py
│ │ ├── local/
│ │ │ ├── __init__.py
│ │ │ ├── config/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── instance_config.py
│ │ │ │ └── validation.py
│ │ │ └── instance_manager.py
│ │ └── remote/
│ │ ├── __init__.py
│ │ ├── config/
│ │ │ ├── __init__.py
│ │ │ ├── instance_config.py
│ │ │ └── validation.py
│ │ ├── helpers/
│ │ │ └── rsync.py
│ │ └── instance_manager.py
│ └── utils.py
└── tests/
├── __init__.py
├── container_config.py
├── helpers/
│ ├── __init__.py
│ ├── cli.py
│ └── spotty_cli.py
└── providers/
├── __init__.py
├── aws/
│ ├── __init__.py
│ ├── commands/
│ │ ├── data/
│ │ │ └── test-project/
│ │ │ ├── ignored-dir/
│ │ │ │ ├── ignored-file
│ │ │ │ └── included-file
│ │ │ ├── ignored-file
│ │ │ ├── local-file
│ │ │ └── spotty.yaml
│ │ ├── download.py
│ │ └── sync.py
│ ├── config/
│ │ ├── __init__.py
│ │ ├── container_deployment.py
│ │ ├── data/
│ │ │ ├── config-wo-mounts.yaml
│ │ │ └── config1.yaml
│ │ └── instance_config_validation.py
│ └── project_resources/
│ ├── __init__.py
│ ├── bucket.py
│ └── key_pair.py
├── gcp/
│ └── config/
│ ├── __init__.py
│ └── image_uri.py
└── local/
├── __init__.py
├── commands/
│ ├── __init__.py
│ ├── data/
│ │ └── test-project/
│ │ └── spotty.yaml
│ └── run.py
└── config/
├── __init__.py
├── container_deployment.py
└── data/
└── config1.yaml
================================================
FILE CONTENTS
================================================
================================================
FILE: .github/FUNDING.yml
================================================
github: [apls777]
================================================
FILE: .github/workflows/generate-docs.yml
================================================
# This workflows will upload a Python Package using Twine when a release is created
# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
name: Generate Docs
on:
push:
branches:
- master
jobs:
update-doc:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions/setup-python@v1
with:
python-version: 3.6
- name: generate docs
run: |
cd docs
pip install -r requirements.txt
make html
cd build/html
touch .nojekyll
echo "spotty.cloud" > CNAME
git init
git config --local user.email "github-bot@spotty.cloud"
git config --local user.name "Spotty Dev Bot"
git add .
git commit -m "generated docs" -a
- uses: ad-m/github-push-action@master
with:
github_token: ${{ secrets.BOT_GITHUB_TOKEN }}
repository: spotty-cloud/website
force: true
directory: docs/build/html
================================================
FILE: .github/workflows/python-publish.yml
================================================
# This workflows will upload a Python Package using Twine when a release is created
# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
name: Upload Python Package
on:
release:
types: [created]
jobs:
deploy:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Set up Python
uses: actions/setup-python@v2
with:
python-version: '3.6'
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install setuptools wheel twine
- name: Build and publish
env:
TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
run: |
python setup.py sdist bdist_wheel
twine upload dist/*
================================================
FILE: .gitignore
================================================
.idea/
build/
dist/
*.egg-info/
__pycache__/
todo
================================================
FILE: CONTRIBUTING.md
================================================
# Contributing to Spotty
**Thank you for your interest in Spotty. Your contributions are highly welcome.**
There are multiple ways of getting involved:
- [Report a bug](#report-a-bug)
- [Suggest a feature](#suggest-a-feature)
- [Contribute code](#contribute-code)
Below are a few guidelines we would like you to follow.
If you need help, please reach out to us by opening an issue.
## Report a bug
Reporting bugs is one of the best ways to contribute. Before creating a bug report,
please check that an [issue](https://github.com/spotty-cloud/spotty/issues) reporting the same problem does not already
exist. If there is such an issue, you may add your information as a comment.
To report a new bug you should open an issue that summarizes the bug and set the label to "bug".
If you want to provide a fix along with your bug report: that is great! In this case please send us a pull request as
described in section [Contribute Code](#contribute-code).
## Suggest a feature
To request a new feature you should open an [issue](https://github.com/spotty-cloud/spotty/issues/new) and summarize
the desired functionality and its use case. Set the issue label to "feature".
## Contribute code
This is a rough outline of what the workflow for code contributions looks like:
- Check the list of open [issues](https://github.com/spotty-cloud/spotty/issues). Either assign an existing issue to
yourself or create a new one that you would like to work on and discuss your ideas and use cases. It is always best to
discuss your plans beforehand, to ensure that your contribution is in line with our goals for Spotty.
- Fork the repository on GitHub
- Create a topic branch from where you want to base your work. This is usually the master.
- Make commits of logical units
- Write good commit messages (see below)
- Push your changes to a topic branch in your fork of the repository
- Submit a pull request to [spotty-cloud/spotty](https://github.com/spotty-cloud/spotty)
Thanks for your contributions!
### Commit messages
Your commit messages ideally can answer two questions: what changed and why. The subject line should feature
the "what" and the body of the commit should describe the "why".
When creating a pull request, its comment should reference the corresponding issue ID.
**Have fun and enjoy hacking!**
================================================
FILE: LICENSE
================================================
The MIT License (MIT)
Copyright (c) 2017 Oleg Polosin
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit
persons to whom the Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the
Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
================================================
FILE: README.md
================================================
<img src="https://spotty.cloud/_static/images/logo_740x240.png" width="370" height="120" />
[](https://spotty.cloud)
[](https://pypi.org/project/spotty/)
Spotty drastically simplifies training of deep learning models on [AWS](https://aws.amazon.com/)
and [GCP](https://cloud.google.com/):
- it makes training on GPU instances as simple as training on your local machine
- it automatically manages all necessary cloud resources including images, volumes, snapshots and SSH keys
- it makes your model trainable in the cloud by everyone with a couple of commands
- it uses [tmux](https://en.wikipedia.org/wiki/Tmux) to easily detach remote processes from their terminals
- it saves you up to 70% of the costs by using [AWS Spot Instances](https://aws.amazon.com/ec2/spot/)
and [GCP Preemtible VMs](https://cloud.google.com/preemptible-vms/)
## Documentation
- See the [documentation page](https://spotty.cloud).
- Read [this](https://medium.com/@apls/how-to-train-deep-learning-models-on-aws-spot-instances-using-spotty-8d9e0543d365)
article on Medium for a real-world example.
## Installation
Requirements:
* Python >=3.6
* AWS CLI (see [Installing the AWS Command Line Interface](http://docs.aws.amazon.com/cli/latest/userguide/installing.html))
if you're using AWS
* Google Cloud SDK (see [Installing Google Cloud SDK](https://cloud.google.com/sdk/install))
if you're using GCP
Use [pip](http://www.pip-installer.org/en/latest/) to install or upgrade Spotty:
$ pip install -U spotty
## Get Started
1. Prepare a `spotty.yaml` file and put it to the root directory of your project:
- See the file specification [here](https://spotty.cloud/docs/user-guide/configuration-file.html).
- Read [this](https://medium.com/@apls/how-to-train-deep-learning-models-on-aws-spot-instances-using-spotty-8d9e0543d365)
article for a real-world example.
2. Start an instance:
```bash
$ spotty start
```
It will run a Spot Instance, restore snapshots if any, synchronize the project with the running instance
and start the Docker container with the environment.
3. Train a model or run notebooks.
To connect to the running container via SSH, use the following command:
```bash
$ spotty sh
```
It runs a [tmux](https://github.com/tmux/tmux/wiki) session, so you can always detach this session using
__`Ctrl + b`__, then __`d`__ combination of keys. To be attached to that session later, just use the
`spotty sh` command again.
Also, you can run your custom scripts inside the Docker container using the `spotty run <SCRIPT_NAME>` command. Read more
about custom scripts in the documentation:
[Configuration: "scripts" section](https://spotty.cloud/docs/configuration-file/#scripts-section-optional).
## Contributions
Any feedback or contributions are welcome! Please check out the [guidelines](CONTRIBUTING.md).
## License
[MIT License](LICENSE)
================================================
FILE: bin/spotty
================================================
#!/usr/bin/env python
import sys
import logging
import spotty
from spotty.cli import get_parser
from spotty.commands.writers.output_writrer import OutputWriter
parser = get_parser()
args = sys.argv[1:]
output = OutputWriter()
# display the version
if '-V' in args:
output.write(spotty.__version__)
sys.exit(0)
# separate Spotty arguments from custom arguments
custom_args = []
if '--' in args:
dd_idx = args.index('--')
custom_args = args[(dd_idx + 1):]
args = args[:dd_idx]
# parse arguments
args = parser.parse_args(args)
args.custom_args = custom_args
# logging
logging_level = logging.DEBUG if 'debug' in args and args.debug else logging.WARNING
logging.basicConfig(level=logging_level, format='[%(levelname)s] %(message)s')
if 'command' not in args:
parser.print_help()
sys.exit(1)
# run a command
try:
args.command.run(args, output)
except Exception as e:
output.write('Error:\n'
'------\n'
'%s' % str(e))
sys.exit(1)
================================================
FILE: docs/Makefile
================================================
# Minimal makefile for Sphinx documentation
#
# You can set these variables from the command line.
SPHINXOPTS =
SPHINXBUILD = python -msphinx
SPHINXPROJ = spotty
SOURCEDIR = source
BUILDDIR = build
# Put it first so that "make" without argument is like "make help".
help:
@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
.PHONY: help Makefile
# Catch-all target: route all unknown targets to Sphinx using the new
# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).
%: Makefile
@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
================================================
FILE: docs/make.bat
================================================
@ECHO OFF
pushd %~dp0
REM Command file for Sphinx documentation
if "%SPHINXBUILD%" == "" (
set SPHINXBUILD=python -msphinx
)
set SOURCEDIR=source
set BUILDDIR=build
set SPHINXPROJ=spotty
if "%1" == "" goto help
%SPHINXBUILD% >NUL 2>NUL
if errorlevel 9009 (
echo.
echo.The Sphinx module was not found. Make sure you have Sphinx installed,
echo.then set the SPHINXBUILD environment variable to point to the full
echo.path of the 'sphinx-build' executable. Alternatively you may add the
echo.Sphinx directory to PATH.
echo.
echo.If you don't have Sphinx installed, grab it from
echo.http://sphinx-doc.org/
exit /b 1
)
%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%
goto end
:help
%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%
:end
popd
================================================
FILE: docs/requirements.txt
================================================
sphinx==3.1.2
recommonmark==0.6.0
sphinx-argparse==0.2.5
sphinx-rtd-theme==0.5.0
PyYAML
schema
chevron
boto3
================================================
FILE: docs/source/_static/favicon/browserconfig.xml
================================================
<?xml version="1.0" encoding="utf-8"?>
<browserconfig>
<msapplication>
<tile>
<square150x150logo src="/mstile-150x150.png"/>
<TileColor>#00aba9</TileColor>
</tile>
</msapplication>
</browserconfig>
================================================
FILE: docs/source/_static/favicon/site.webmanifest
================================================
{
"name": "",
"short_name": "",
"icons": [
{
"src": "/android-chrome-192x192.png",
"sizes": "192x192",
"type": "image/png"
},
{
"src": "/android-chrome-512x512.png",
"sizes": "512x512",
"type": "image/png"
}
],
"theme_color": "#ffffff",
"background_color": "#ffffff",
"display": "standalone"
}
================================================
FILE: docs/source/_static/scripts.js
================================================
window.onload = function() {
var links = document.querySelectorAll('a.external');
for(var i = 0; i < links.length; i++) {
links[i].target = '_blank';
}
}
================================================
FILE: docs/source/_static/styles.css
================================================
.wy-nav-content {
max-width: 1280px;
}
.wy-side-nav-search {
background: none;
}
.wy-menu-vertical header, .wy-menu-vertical p.caption {
color: #e44859;
}
.wy-side-nav-search > a img.logo, .wy-side-nav-search .wy-dropdown > a img.logo {
max-width: 75%;
}
.wy-breadcrumbs a.icon-home {
color: #e44859;
}
.section#welcome-to-spotty-documentation {
display: none;
}
================================================
FILE: docs/source/conf.py
================================================
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# spotty documentation build configuration file, created by
# sphinx-quickstart on Fri Jul 17 16:00:08 2020.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.
# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#
# import os
# import sys
# sys.path.insert(0, os.path.abspath('.'))
import os
import sys
sys.path.insert(0, os.path.abspath('../..'))
import spotty
# -- General configuration ------------------------------------------------
# If your documentation needs a minimal Sphinx version, state it here.
#
# needs_sphinx = '1.0'
# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
'recommonmark',
'sphinxarg.ext',
'sphinx_rtd_theme',
'sphinx.ext.autosectionlabel',
]
# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']
# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
#
# source_suffix = ['.rst', '.md']
source_suffix = {
'.rst': 'restructuredtext',
'.md': 'markdown',
}
# The master toctree document.
master_doc = 'index'
# General information about the project.
project = 'spotty'
copyright = '2020, Oleg Polosin'
author = 'Oleg Polosin'
# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = spotty.__version__
# The full version, including alpha/beta/rc tags.
release = spotty.__version__
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
language = None
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This patterns also effect to html_static_path and html_extra_path
exclude_patterns = []
# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'
# If true, `todo` and `todoList` produce output, else they produce nothing.
todo_include_todos = False
# to display double-dash (--) in epilogs of some Spotty commands
smartquotes = False
# Prefix document path to section labels, otherwise autogenerated labels would look like 'heading'
# rather than 'path/to/file:heading'
autosectionlabel_prefix_document = True
# -- Options for HTML output ----------------------------------------------
# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
#
html_theme = 'sphinx_rtd_theme'
# Theme options are theme-specific and customize the look and feel of a theme
# further. For a list of options available for each theme, see the
# documentation.
#
html_theme_options = {
'logo_only': True,
'style_external_links': True,
'collapse_navigation': False,
'titles_only': True,
}
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']
# Custom sidebar templates, must be a dictionary that maps document names
# to template names.
#
# This is required for the alabaster theme
# refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars
html_sidebars = {
'**': [
'about.html',
'navigation.html',
'relations.html', # needs 'show_related': True theme option to display
'searchbox.html',
'donate.html',
]
}
html_show_copyright = False
html_show_sphinx = False
html_show_sourcelink = False
html_logo = '_static/images/logo_400x130_grey.png'
html_favicon = '_static/favicon/favicon.ico'
html_css_files = [
'styles.css',
]
html_js_files = [
'scripts.js',
]
# -- Options for HTMLHelp output ------------------------------------------
# Output file base name for HTML help builder.
htmlhelp_basename = 'spottydoc'
# -- Options for LaTeX output ---------------------------------------------
latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#
# 'papersize': 'letterpaper',
# The font size ('10pt', '11pt' or '12pt').
#
# 'pointsize': '10pt',
# Additional stuff for the LaTeX preamble.
#
# 'preamble': '',
# Latex figure (float) alignment
#
# 'figure_align': 'htbp',
}
# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
# author, documentclass [howto, manual, or own class]).
latex_documents = [
(master_doc, 'spotty.tex', 'spotty Documentation',
'Oleg Polosin', 'manual'),
]
# -- Options for manual page output ---------------------------------------
# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
(master_doc, 'spotty', 'spotty Documentation',
[author], 1)
]
# -- Options for Texinfo output -------------------------------------------
# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
# dir menu entry, description, category)
texinfo_documents = [
(master_doc, 'spotty', 'spotty Documentation',
author, 'spotty', 'One line description of project.',
'Miscellaneous'),
]
================================================
FILE: docs/source/docs/cli/spotty-aws.rst
================================================
spotty aws
==========
.. argparse::
:nodefaultconst:
:ref: spotty.cli.get_parser
:prog: spotty
:path: aws
================================================
FILE: docs/source/docs/cli/spotty-download.rst
================================================
spotty download
===============
.. argparse::
:nodefaultconst:
:ref: spotty.cli.get_parser
:prog: spotty
:path: download
================================================
FILE: docs/source/docs/cli/spotty-exec.rst
================================================
spotty exec
===========
.. argparse::
:nodefaultconst:
:ref: spotty.cli.get_parser
:prog: spotty
:path: exec
================================================
FILE: docs/source/docs/cli/spotty-run.rst
================================================
spotty run
==========
.. argparse::
:nodefaultconst:
:ref: spotty.cli.get_parser
:prog: spotty
:path: run
================================================
FILE: docs/source/docs/cli/spotty-sh.rst
================================================
spotty sh
=========
.. argparse::
:nodefaultconst:
:ref: spotty.cli.get_parser
:prog: spotty
:path: sh
================================================
FILE: docs/source/docs/cli/spotty-start.rst
================================================
spotty start
============
.. argparse::
:nodefaultconst:
:ref: spotty.cli.get_parser
:prog: spotty
:path: start
================================================
FILE: docs/source/docs/cli/spotty-stop.rst
================================================
spotty stop
===========
.. argparse::
:nodefaultconst:
:ref: spotty.cli.get_parser
:prog: spotty
:path: stop
================================================
FILE: docs/source/docs/cli/spotty-sync.rst
================================================
spotty sync
===========
.. argparse::
:nodefaultconst:
:ref: spotty.cli.get_parser
:prog: spotty
:path: sync
================================================
FILE: docs/source/docs/cli/spotty.rst
================================================
Spotty Command-line Interface
=============================
.. argparse::
:nosubcommands:
:nodefaultconst:
:noepilog:
:nodescription:
:ref: spotty.cli.get_parser
:prog: spotty
.. toctree::
:maxdepth: 1
:caption: Sub-commands
spotty-start
spotty-stop
spotty-sh
spotty-sync
spotty-download
spotty-run
spotty-exec
.. toctree::
:maxdepth: 1
:caption: Custom provider sub-commands
spotty-aws
================================================
FILE: docs/source/docs/providers/aws/caching-docker-image-on-an-ebs-volume.md
================================================
# Caching Docker Image on an EBS Volume
You can cache images that you've built or downloaded from the internet on an EBS volume or in a snapshot.
A configuration file has the "__dockerDataRoot__" parameter. It's a directory on the host OS where the Docker
daemon will save all the images.
Specify the `mountDir` directory for one of the instance volumes and set the `dockerDataRoot` parameter
to the same value (or to a subdirectory of the `mountDir` directory). Also, consider changing a deletion policy
for that volume to "__retain__", then the volume with the cache will be retained and the next time it just will be
attached to the instance.
Example:
```yaml
# ...
instances:
- name: aws-1
provider: aws
parameters:
# ...
dockerDataRoot: /docker
volumes:
# ...
- name: docker
parameters:
size: 10
mountDir: /docker
```
================================================
FILE: docs/source/docs/providers/aws/ebs-volumes-and-deletion-policies.md
================================================
# EBS Volumes and Deletion Policies
By default, EBS volumes have names in the following format: `<PROJECT_NAME>-<INSTANCE_NAME>-<VOLUME_NAME>`.
But you can specify a custom name using the `volumeName` parameter.
When you're starting an instance:
1. Spotty is looking for existing EBS volumes using their names. If a volume exists, it will be attached to the
instance.
2. If not - Spotty will be looking for a snapshot with the same name. If the snapshot exists, the volume will be
restored from that snapshot.
3. If neither snapshot nor volume with this name exists, new EBS volume will be created.
When you're stopping the instance Spotty applies deletion policies for the volumes. There are 4 deletion policies that
can be specified using the `deletionPolicy` parameter:
- __Retain__: this is the default deletion policy. The volume will retain, a snapshot won't be created.
- __CreateSnapshot__: Spotty will create a new snapshot every time you're stopping an instance, the old snapshot
will be renamed. AWS uses incremental snapshots, so each new snapshot keeps only the data that was changed since
the last snapshot made (see:
[How Incremental Snapshots Work](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSSnapshots.html#how_snapshots_work)).
- __UpdateSnapshot__: a new snapshot will be created and the old one will be deleted.
- __Delete__: the volume will be deleted without creating a snapshot. All data on this volume will be lost.
================================================
FILE: docs/source/docs/providers/aws/faq.md
================================================
# FAQ
## How does Spotty choose the AWS Availability Zone where to run the instance?
1. If the AZ is specified in the configuration file, this AZ will be used to run the instance.
2. If the instance already has some EBS volumes created, Spotty will pick up the volumes' AZ.
3. Otherwise Spotty will let AWS choose an AZ. Automatically chosen AZ might not have the
lowest Spot price, but in practice, it usually does.
Spotty will raise an error if the AZ in the configuration file doesn't match AZs of the volumes
or AZs of the volumes are different.
## Why an instance is launching too long?
Most likely the instance cannot be launched because you're trying to launch a Spot instance
and it cannot be fulfilled. You can try to change the region or availability zone, choose another
type of the instance, or run an On-demand Instance by removing the `spotInstance` parameter or setting it to `false`.
## The instance is failed to start. Where can I find the logs?
1. If the CloudFormation stack failed when it was launching the instance itself, then you need to log in to
your AWS Console and check CloudFormation logs there.
2. If the stack is failed after the instance is launched, then most likely the container is
failed to start because of the startup commands. In this case, Spotty usually automatically downloads necessary
logs to your local machine and shows where to find them. If that didn't happen, you can connect to the
host OS using the following command:
```bash
spotty sh -H
```
Then you can check the `cfn-init` logs to find out why the container is failed:
```bash
sudo tail /var/log/cfn-init-cmd.log
```
## How to ssh to a Spotty instance from a different machine?
When you start an instance, Spotty creates an EC2 Key Pair and downloads a private key to the
`~/.spotty/keys/aws` directory. If you want to have access to the instance from a different machine using
the `spotty sh` or the `spotty run` commands, you need to copy the private key to that machine to the same directory.
__Note:__ if you already have an EC2 Key Pair created for the project and the private key was
saved on the machine A (where from an instance was launched the first time) and then you're running
an instance for the same project from the machine B that doesn't have a private key in the `~/.spotty/keys/aws`
directory, then the EC2 Key Pair will be recreated and the machine A will not be able to connect to instances
because its private key doesn't match the EC2 Key Pair anymore.
================================================
FILE: docs/source/docs/providers/aws/instance-parameters.md
================================================
# Instance Parameters
Instance parameters are part of the [configuration file], but for each provider they are different.
Here you can find parameters for an AWS instance:
- __`containerName`__ _(optional)_ - a name of the container from the `containers` section.
Default value: `default`.
- __`region`__ - AWS region where to run an instance (you can use command `spotty aws spot-prices` to find the
cheapest region).
- __`availabilityZone`__ _(optional)_ - AWS availability zone where to run an instance. If a zone is not specified, it
will be chosen automatically.
- __`subnetId`__ _(optional)_ - AWS subnet ID. If this parameter is set, the "availabilityZone" parameter should be
set as well. If it's not specified, a default subnet will be used.
- __`instanceType`__ - a type of the instance to run. You can find more information about
types of GPU instances here:
[Recommended GPU Instances](https://docs.aws.amazon.com/dlami/latest/devguide/gpu.html).
- __`spotInstance`__ _(optional)_ - if set to `true`, runs a Spot instance instead of an On-demand instance,
- __`amiName`__ _(optional)_ - a name of the AMI with NVIDIA Docker (default value is "SpottyAMI"). Use the
`spotty aws create-ami` command to create it. This AMI will be used to run your application inside the Docker container.
- __`amiId`__ _(optional)_ - ID of the AMI with NVIDIA Docker. This parameter can be used to run an instance using a
shared Spotty AMI.
- __`maxPrice`__ _(optional)_ - the maximum price per hour that you are willing to pay for a Spot Instance. By default,
it's the On-demand price for the chosen instance type. Read more here:
[Spot Instances](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-spot-instances.html).
- __`rootVolumeSize`__ _(optional)_ - size of the root volume in GB. The root volume will be destroyed once
the instance is terminated. Use attached volumes to store the data you need to keep (see "volumes" parameter below).
- __`dockerDataRoot`__ _(optional)_ - directory where Docker will store all downloaded and built images.
Read more: [Caching Docker Image on an EBS Volume].
- __`volumes`__ _(optional)_ - the list of volumes to attach to the instance:
- __`name`__ - a name of the volume. This name should match one of the container's `volumeMounts` to have this
volume attached to the container's filesystem.
- __`parameters`__ _(optional)_ - parameters of the volume:
- __`type`__ _(optional)_ - the volume type. Supported types: "__gp2__", "__sc1__", "__st1__"
and "__standard__". The default value is "gp2". Read more here:
[Amazon EBS Volume Types](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSVolumeTypes.html).
- __`size`__ _(optional)_ - size of the volume in GB. Size of the volume cannot be less than the size of
the existing snapshot but can be increased.
- __`deletionPolicy`__ _(optional)_ - what to do with the volume once the instance is terminated using the
`spotty stop` command. Possible values include: "__Retain__" _(value by default)_, "__CreateSnapshot__",
"__UpdateSnapshot__" and "__Delete__". Read more here: [EBS Volumes and Deletion Policies].
- __`volumeName`__ _(optional)_ - name of the EBS volume. The default name is
"{project_name}-{instance_name}-{volume_name}".
- __`mountDir`__ _(optional)_ - directory where the volume will be mounted on the instance. The default
directory is "/mnt/{ebs_volume_name}".
- __`ports`__ _(optional)_ - list of ports to open on the instance. For example:
```yaml
ports: [6006, 8888]
```
It will open ports 6006 for TensorBoard and 8888 for Jupyter Notebook.
- __`localSshPort`__ _(optional)_ - if this parameter is set, all the Spotty commands will create SSH connections
with the instance using the IP address __127.0.0.1__ and the specified port. This can be useful in case when an
instance doesn't have a public IP address and a jump-server is used for tunneling.
- __`managedPolicyArns`__ _(optional)_ - a list of Amazon Resource Names (ARNs) of the IAM managed policies that
you want to attach to the instance role. Read more about Managed Policies
[here](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_managed-vs-inline.html).
- __`instanceProfileArn`__ _(optional)_ - an Amazon Resource Name (ARN) of the IAM Instance Profile that you'd like
to attach to the instance. Read more about Instance Profiles
[here](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2_instance-profiles.html).
- __`commands`__ _(optional)_ - commands that should be run on the host OS before the container is started.
For example, you could login to Amazon ECR to pull a Docker image from there
([Deep Learning Containers Images](https://docs.aws.amazon.com/dlami/latest/devguide/deep-learning-containers-images.html)):
```yaml
commands: |
$(aws ecr get-login --no-include-email --region us-east-2 --registry-ids 763104351884)
```
[configuration file]: </docs/user-guide/configuration-file>
[Caching Docker Image on an EBS Volume]: </docs/providers/aws/caching-docker-image-on-an-ebs-volume>
[EBS Volumes and Deletion Policies]: </docs/providers/aws/ebs-volumes-and-deletion-policies>
================================================
FILE: docs/source/docs/providers/aws/overview.rst
================================================
AWS Provider Overview
=====================
.. toctree::
instance-parameters
ebs-volumes-and-deletion-policies
caching-docker-image-on-an-ebs-volume
faq
================================================
FILE: docs/source/docs/providers/gcp/account-preparation.md
================================================
# GCP Account Preparation
1. [Create a project](https://console.cloud.google.com/projectcreate)
if you don't have one already.
2. Enable the [Deployment Manager API](https://console.cloud.google.com/apis/library/deploymentmanager.googleapis.com)
for the created project.
3. Enable the [Runtime Configuration API](https://console.developers.google.com/apis/library/runtimeconfig.googleapis.com)
for the created project.
4. [Create a service account](https://console.cloud.google.com/iam-admin/serviceaccounts/create).
5. Go to the [IAM page](https://console.cloud.google.com/iam-admin/iam) and add the following
roles to the created service account:
1. _Compute Admin_
2. _Storage Admin_
3. _Deployment Manager Editor_
4. _Cloud RuntimeConfig Admin_
6. Make sure you have a quota to run GPU instances:
1. Go to the quotas page in "IAM & admin" and filter the list of services by setting the Metric
field to __"GPUs (all regions)"__: [https://console.cloud.google.com/iam-admin/quotas?metric=GPUs%20(all%20regions)](https://console.cloud.google.com/iam-admin/quotas?metric=GPUs%20(all%20regions)).
2. Check the limit for the __"Compute Engine API"__ service. If it's a zero, select the service and
click the __"[+] EDIT QUOTAS"__ button at the top of the page.
3. Set a new quota limit to 1 or more and submit the request.
7. [Install Google Cloud SDK](https://cloud.google.com/sdk/install).
8. Before using Spotty commands like `spotty start`, `spotty run` and others, make sure that the
`GOOGLE_APPLICATION_CREDENTIALS` environmental variable is set up and contains the path to your service
account key file:
```bash
export GOOGLE_APPLICATION_CREDENTIALS="/path/to/the/service/account/key/file.json"
```
================================================
FILE: docs/source/docs/providers/gcp/caching-docker-image-on-a-disk.md
================================================
# Caching Docker Image on a Disk
You can cache images that you've built or downloaded from the internet on a disk that you attach to the instance.
A configuration file has the "__dockerDataRoot__" parameter. It's a directory on the host OS where the Docker
daemon will save all the images.
Specify the `moundDir` directory for one of the instance volumes and set the `dockerDataRoot` parameter
to the same value (or to a subdirectory of the `moundDir` directory).
Example:
```yaml
# ...
instances:
- name: gcp-1
provider: gcp
parameters:
# ...
dockerDataRoot: /docker
volumes:
# ...
- name: docker
parameters:
size: 10
mountDir: /docker
```
================================================
FILE: docs/source/docs/providers/gcp/disks-and-deletion-policies.md
================================================
# Disks and Deletion Policies
By default, disks have names in the following format: `<PROJECT_NAME>-<INSTANCE_NAME>-<VOLUME_NAME>`.
But you can specify a custom name using the `diskName` parameter.
When you're starting an instance:
1. Spotty is looking for existing disks using their names. If a disk exists, it will be attached to the
instance.
2. If not - Spotty will be looking for a snapshot with the same name. If the snapshot exists, the disk will be
restored from that snapshot.
3. If neither snapshot nor disk with this name exists, a new disk will be created.
__Note:__ Deletion Policies for the GCP provider are not implemented yet, so, regardless of the `deletionPolicy`
parameter value, created disks will retain when the instance is terminated.
================================================
FILE: docs/source/docs/providers/gcp/instance-parameters.md
================================================
# Instance Parameters
Instance parameters are part of the [configuration file], but for each provider they are different.
Here you can find parameters for a GCP instance:
- __`containerName`__ _(optional)_ - a name of the container from the `containers` section.
Default value: `default`.
- __`zone`__ - GCP zone where to run an instance.
- __`machineType`__ - a type of the instance to run. You can find a list of predefined machine types
here: [Machine Types](https://cloud.google.com/compute/docs/machine-types). If you in doubt what to use,
just go for `n1-standard-1`. To attach GPUs to the selected machine type, use the `gpu` parameter (see
the details below).
- __`gpu`__ _(optional)_ - _a dictionary with keys `type` and `count`_:
- __`type`__ - a type of GPU to attach to the instance. Read more about GPUs and their availabily
in different zones here: [GPUs on Compute Engine](https://cloud.google.com/compute/docs/gpus/).
- __`count`__ _(optional)_ - a number of GPUs that should be attached to the instance. The default
value is 1. See here a number of GPUs that you can attach to different machine types:
[Valid numbers of GPUs for each machine type](https://cloud.google.com/ml-engine/docs/tensorflow/using-gpus#gpu-compatibility-table).
- __`preemptibleInstance`__ _(optional)_ - if set to `true`, runs a preemptible instance instead of an on-demand
instance. __Note:__ be aware that GCP terminates preemptible instances in 24 hours. Read more about Preemptible VMs
[here](https://cloud.google.com/compute/docs/instances/preemptible).
- __`imageName`__ _(optional)_ - a name of the image with NVIDIA Docker in the current GCP project. You can use
the `spotty gcp create-image` command to create it. By default, the command will create an image with the name
"spotty". This image will be used to run your application inside the Docker container. If you didn't create your own
image, see the behaviour of the `imageUrl` parameter.
- __`imageUrl`__ _(optional)_ - a URL of the image with NVIDIA Docker. You can use this parameter to work with an image
from another GCP project. If this parameter is not specified and you didn't create your own image (see the `imageName`
parameter), Spotty will be using the `projects/spotty-cloud/global/images/family/spotty` image provided by the Spotty
project.
- __`bootDiskSize`__ _(optional)_ - size of the root volume in GB. The root volume will be destroyed once
the instance is terminated. Use attached volumes to store the data that you need to keep (see the `volumes`
parameter below).
- __`dockerDataRoot`__ _(optional)_ - directory where Docker will store all downloaded and built images.
Read more: [Caching Docker Image on a Disk].
- __`volumes`__ _(optional)_ - the list of volumes to attach to the instance:
- __`name`__ - a name of the volume. This name should match one of the container's `volumeMounts` to have this
volume attached to the container's filesystem.
- __`parameters`__ _(optional)_ - parameters of the volume:
- __`size`__ _(optional)_ - size of the disk in GB. Size of the disk cannot be less than the size of
the existing snapshot but can be increased.
- __`deletionPolicy`__ _(optional)_ - what to do with the disk once the instance is terminated using the
`spotty stop` command. Possible values include: "__Retain__" _(value by default)_, "__CreateSnapshot__",
"__UpdateSnapshot__" and "__Delete__". Read more: [Disks and Deletion Policies].
__(!) Note:__ Deletion Policies are not implemented yet, so created disks will always retain.
- __`diskName`__ _(optional)_ - name of the disk. The default name is
"{project_name}-{instance_name}-{volume_name}".
- __`mountDir`__ _(optional)_ - directory where the disk will be mounted on the instance. The default
directory is "/mnt/{disk_name}".
- __`ports`__ _(optional)_ - list of ports to open on the instance. For example:
```yaml
ports: [6006, 8888]
```
It will open ports 6006 for TensorBoard and 8888 for Jupyter Notebook.
- __`localSshPort`__ _(optional)_ - if this parameter is set, all the Spotty commands will create SSH connections
with the instance using the IP address __127.0.0.1__ and the specified port. This can be useful in case when an
instance doesn't have a public IP address and a jump-server is used for tunneling.
- __`commands`__ _(optional)_ - commands that should be run on the host OS before the container is started.
[configuration file]: </docs/user-guide/configuration-file>
[Caching Docker Image on a Disk]: </docs/providers/gcp/caching-docker-image-on-a-disk>
[Disks and Deletion Policies]: </docs/providers/gcp/disks-and-deletion-policies>
================================================
FILE: docs/source/docs/providers/gcp/overview.rst
================================================
GCP Provider Overview
=====================
.. toctree::
account-preparation
instance-parameters
disks-and-deletion-policies
caching-docker-image-on-a-disk
================================================
FILE: docs/source/docs/providers/local/instance-parameters.md
================================================
# Instance Parameters
Instance parameters are part of the [configuration file], but for each provider they are different.
Here you can find parameters for a local instance:
- __`containerName`__ _(optional)_ - a name of the container from the `containers` section.
Default value: `default`.
- __`volumes`__ _(optional)_ - the list of volumes to attach to the instance:
- __`name`__ - a name of the volume. This name should match one of the container's `volumeMounts` to have this
volume attached to the container's filesystem.
- __`parameters`__ _(optional)_ - parameters of the volume:
- __`path`__ _(optional)_ - a path on a local instance that should be mounted to the container.
[configuration file]: </docs/user-guide/configuration-file>
================================================
FILE: docs/source/docs/providers/local/overview.rst
================================================
Local Provider Overview
=======================
.. toctree::
instance-parameters
================================================
FILE: docs/source/docs/providers/remote/instance-parameters.md
================================================
# Instance Parameters
Instance parameters are part of the [configuration file], but for each provider they are different.
Here you can find parameters for a remote instance:
- __`containerName`__ _(optional)_ - a name of the container from the `containers` section.
Default value: `default`.
- __`volumes`__ _(optional)_ - the list of volumes to attach to the instance:
- __`name`__ - a name of the volume. This name should match one of the container's `volumeMounts` to have this
volume attached to the container's filesystem.
- __`parameters`__ _(optional)_ - parameters of the volume:
- __`path`__ _(optional)_ - a path on a remote instance that should be mounted to the container.
[configuration file]: </docs/user-guide/configuration-file>
================================================
FILE: docs/source/docs/providers/remote/overview.rst
================================================
Remote Provider Overview
========================
.. toctree::
instance-parameters
================================================
FILE: docs/source/docs/user-guide/configuration-file.md
================================================
# Spotty Configuration File
By default, Spotty is looking for a `spotty.yaml` file in the root directory of the project. This file describes
parameters of a remote instance and an environment for the project. Here is a basic example of such file for AWS:
```yaml
project:
name: my-project-name
syncFilters:
- exclude:
- .git/*
- .idea/*
- '*/__pycache__/*'
containers:
- projectDir: /workspace/project
image: tensorflow/tensorflow:latest-gpu-py3-jupyter
env:
PYTHONPATH: /workspace/project
ports:
# TensorBoard
- containerPort: 6006
hostPort: 6006
# Jupyter
- containerPort: 8888
hostPort: 8888
volumeMounts:
- name: workspace
mountPath: /workspace
instances:
- name: aws-1
provider: aws
parameters:
region: eu-west-1
instanceType: p2.xlarge
ports: [6006, 8888]
volumes:
- name: workspace
parameters:
size: 50
scripts:
tensorboard: |
tensorboard --bind_all --port 6006 --logdir /workspace/project/training
jupyter: |
jupyter notebook --allow-root --ip 0.0.0.0 --notebook-dir=/workspace/project
```
Instance parameters are different for each provider:
- [Local Provider Instance Parameters]
- [Remote Provider Instance Parameters]
- [AWS Provider Instance Parameters]
- [GCP Provider Instance Parameters]
## Available Parameters
Configuration file consists of 4 sections: `project`, `containers`, `instances` and `scripts`.
### __`project`__ section
The `project` section contains the following parameters:
- __`name`__ - the name of your project. It will be used to create an S3 bucket and a CloudFormation stack to run
an instance.
- __`syncFilters`__ _(optional)_ - filters to skip some directories or files during synchronization. By default, all
project files will be synced with the instance. Example:
```yaml
syncFilters:
- exclude:
- .idea/*
- .git/*
- data/*
- include:
- data/test/*
- exclude:
- data/test/dump.json
```
It will skip ".idea/", ".git/" and "data/" directories except the "data/test/" directory. All files from
the "data/test/" directory will be synced with the instance except the "data/test/dump.json" file.
You can read more about filters
here: [Use of Exclude and Include Filter](https://docs.aws.amazon.com/cli/latest/reference/s3/index.html#use-of-exclude-and-include-filters).
### __`containers`__ section
The `containers` section contains a list of containers where each container is described
with the following parameters:
- __`name`__ - a name of the container. You can associate containers with the instances using the `containerName`
parameter in the instance configuration. Default value: `default`.
- __`projectDir`__ - a directory inside the container where the local project will be copied. If
it's a subdirectory of a container volume, the project will be located on that volume,
otherwise, the data will be lost once the instance is terminated.
- __`image`__ _(optional)_ - the name of the Docker image that contains the environment for your project. For example,
you could use [TensorFlow image for GPU](https://hub.docker.com/r/tensorflow/tensorflow/)
(`tensorflow/tensorflow:latest-gpu-py3-jupyter`). It already contains NumPy, SciPy, scikit-learn, pandas, Jupyter Notebook and
TensorFlow itself. If you need to use your own image, you can specify the path to your Dockerfile in the
__`file`__ parameter (see below), or push your image to the [Docker Hub](https://hub.docker.com/).
- __`file`__ _(optional)_ - relative path to your custom Dockerfile.
__Note:__ Spotty uses the directory with the Dockerfile as its build context, so make sure it doesn't contain
gigabytes of irrelevant data (keep the Dockerfile in a separate directory or use the `.dockerignore` file).
Otherwise, you may get an out-of-space error because Docker copies the entire build context to the Docker daemon
during the build. Read more here: ["docker build" command](https://docs.docker.com/engine/reference/commandline/build/).
__Example:__ if you use TensorFlow and need to download your dataset from S3, you could install
[AWS CLI](https://github.com/aws/aws-cli) on top of the original TensorFlow image. Just create a
`Dockerfile` in the `docker/` directory of your project:
```dockerfile
FROM tensorflow/tensorflow:latest-gpu-py3-jupyter
RUN pip install awscli
```
Then set the `file` parameter to `docker/Dockerfile`.
- __`runAsHostUser`__ _(optional)_ - if set to `true`, the container will be run with the host user ID and group ID,
- __`volumeMounts`__ _(optional)_ - where to mount instance volumes into the container's filesystem. Each element
of a list has the following parameters:
- __`name`__ - this must match the name of an instance volume.
- __`mountPath`__ - a path within the container at which the volume should be mounted.
- __`workingDir`__ _(optional)_ - working directory for your custom scripts (see "scripts" section below),
- __`env`__ _(optional)_ - a dictionary with environmental variables that will be available in the container,
- __`hostNetwork`__ _(optional)_ - if set to `true`, the Docker container will be run with the host network,
- __`ports`__ _(optional)_ - container ports that should be published to the host. Each element of a list
contains the following parameters:
- __`containerPort`__ - a container port,
- __`hostPort`__ _(optional)_ - a host port. By default, the container port will be published on a random
host port.
- __`commands`__ _(optional)_ - commands which should be performed once your container is started. For example, you
could download your datasets from an S3 bucket to the project directory (see "project" section):
```yaml
commands: |
aws s3 sync s3://my-bucket/datasets/my-dataset /workspace/project/data
```
- __`runtimeParameters`__ _(optional)_ - a list of additional parameters for the container runtime. For example:
```yaml
runtimeParameters: ['--privileged', '--shm-size', '2G']
```
### __`instances`__ section
The `instances` section contains a list of instances where each instance is described
with the following parameters:
- __`name`__ - a name of the instance. Use this name to manage the instance with the commands like
"spotty start" or "spotty stop". Also Spotty uses this name in the names of AWS and GCP resources.
- __`provider`__ - a provider for the instance. At the moment Spotty supports 4 providers:
- "__local__" - runs containers using the Docker installed on the local machine,
- "__remote__" - runs containers on a remote machine through SSH,
- "__aws__" - Amazon Web Services EC2 instances,
- "__gcp__" - Google Cloud Platform VMs.
- __`parameters`__ - parameters of the instance. These parameters are different for each provider:
- [Local Provider Instance Parameters]
- [Remote Provider Instance Parameters]
- [AWS Provider Instance Parameters]
- [GCP Provider Instance Parameters]
### __`scripts`__ section
The `scripts` section contains custom scripts which can be run with the `spotty run <SCRIPT_NAME>`
command. The following example defines 2 scripts: `jupyter` - to run Jupyter server and `train` -
to start training a model:
```yaml
scripts:
jupyter: |
jupyter notebook --allow-root --ip 0.0.0.0 --notebook-dir=/workspace/project
train: |
if [ -n "{{MODEL}}" ]; then
python /workspace/project/model/train.py --model-name {{MODEL}}
else
echo "The MODEL parameter is required."
fi
```
To start Jupyter simply run:
```bash
spotty run jupyter
```
It will start Jupyter server on the remote instance inside a tmux session. Jupyter will be available on the port
specified in the container configuration (see the example on top of the page).
Copy an authentication token from the command output and use the __`Ctrl + b`__, then __`d`__ combination of keys
to detach the tmux session - Jupyter will keep running.
You also can write parametrized scripts. For example, the `train` script contains the `MODEL` parameter. So you
could run your training script with different model names:
```bash
spotty run train -p MODEL=my-model
```
Use the __`Ctrl + b`__, then __`d`__ combination of keys to detached tmux session - the script will keep running.
You can come back to the running script the following ways:
- either use the same command again - you will be reattached to the existing tmux session,
- or connect to the instance using the `spotty sh` command and then use the __`Ctrl + b`__,
then __`s`__ combination of keys to switch into the right tmux session.
__Note:__ don't forget to use the "|" character for multi-line scripts, otherwise the YAML parser
will merge multiple lines together.
[Local Provider Instance Parameters]: </docs/providers/local/instance-parameters>
[Remote Provider Instance Parameters]: </docs/providers/remote/instance-parameters>
[AWS Provider Instance Parameters]: </docs/providers/aws/instance-parameters>
[GCP Provider Instance Parameters]: </docs/providers/gcp/instance-parameters>
================================================
FILE: docs/source/docs/user-guide/getting-started.md
================================================
# Getting Started
## Installation
Use [pip](http://www.pip-installer.org/en/latest/) to install or upgrade Spotty:
```bash
pip install -U spotty
```
Python >=3.6 is required.
Also, depending on the use case, some additional software is needed:
* __Docker__ if you want to run containers locally: [Get Docker](https://docs.docker.com/get-docker/)
* __AWS CLI__ if you're going to use AWS: [Installing the AWS Command Line Interface](http://docs.aws.amazon.com/cli/latest/userguide/installing.html)
* __Google Cloud SDK__ if you're going to use GCP: [Installing Google Cloud SDK](https://cloud.google.com/sdk/install)
## Prepare a configuration file
Prepare a `spotty.yaml` file and put it to the root directory of your project:
- See the file specification and an example here: [Spotty Configuration File].
- Read [this](https://medium.com/@apls/how-to-train-deep-learning-models-on-aws-spot-instances-using-spotty-8d9e0543d365)
article for a real-world example.
## Start an instance
Use the following command to launch an instance with the Docker container:
```bash
spotty start
```
If you're using AWS, it will create EBS volumes if needed, start an instance, upload project files and start
the Docker container with the environment for your project.
## Train your models or run notebooks
To connect to the running container via SSH, use the following command:
```bash
spotty sh
```
It runs a [tmux](https://github.com/tmux/tmux/wiki) session, so you can always detach this session using
__`Ctrl + b`__, then __`d`__ combination of keys. To be attached to that session later, just use the
`spotty sh` command again.
Also, you can run custom scripts inside the Docker container using the `spotty run <SCRIPT_NAME>` command. Read more
about custom scripts in the documentation: [Configuration File: "scripts" section].
[Spotty Configuration File]: </docs/user-guide/configuration-file>
[Configuration File: "scripts" section]: <docs/user-guide/configuration-file:scripts section>
================================================
FILE: docs/source/docs/user-guide/installation.md
================================================
# Installation
Use [pip](http://www.pip-installer.org/en/latest/) to install or upgrade Spotty:
```bash
pip install -U spotty
```
Python >=3.6 is required.
Also, depending on the use case, some additional software is needed:
* __Docker__ if you want to run containers locally: [Get Docker](https://docs.docker.com/get-docker/)
* __AWS CLI__ if you're going to use AWS: [Installing the AWS Command Line Interface](http://docs.aws.amazon.com/cli/latest/userguide/installing.html)
* __Google Cloud SDK__ if you're going to use GCP: [Installing Google Cloud SDK](https://cloud.google.com/sdk/install)
================================================
FILE: docs/source/index.rst
================================================
.. raw:: html
:file: main.html
Welcome to Spotty Documentation
===============================
.. toctree::
:hidden:
:maxdepth: 2
:caption: User Guide
docs/user-guide/getting-started
docs/user-guide/installation
docs/user-guide/configuration-file
docs/cli/spotty
.. toctree::
:hidden:
:maxdepth: 2
:caption: Providers
Local Provider <docs/providers/local/overview>
Remote Provider <docs/providers/remote/overview>
AWS Provider <docs/providers/aws/overview>
GCP Provider <docs/providers/gcp/overview>
================================================
FILE: docs/source/main.html
================================================
<style>
#main-page-title {
text-align: center;
font-family: Lato,proxima-nova,Helvetica Neue,Arial,sans-serif;
font-weight: normal;
margin-bottom: 30px;
}
#main-page-buttons {
text-align: center;
margin-bottom: 70px;
}
#main-page-buttons .btn.btn-get-started {
background-color: #e64759 !important;
}
#main-page-buttons .btn.btn-get-started:hover {
background-color: #c64b60 !important;
}
#main-page-features td.block {
width: 50%;
padding-bottom: 50px;
}
#main-page-features td.block .icon {
width: 100px;
text-align: center;
height: 100%;
}
#main-page-features td.block .title {
font-weight: bold;
margin-bottom: 10px;
font-size: 18px;
}
#main-page-features td.block .description {
color: #888;
}
</style>
<div class="page-content">
<div style="text-align: center; margin-bottom: 20px;">
<img src="_static/images/logo_740x240.png" style="height: 120px;"/>
</div>
<h1 id="main-page-title">
An Open-source Tool for Training<br />Deep Learning Models in the Cloud
</h1>
<div id="main-page-buttons">
<a href="docs/user-guide/getting-started.html" class="btn btn-get-started">
<i class="fa fa-terminal"></i> Get started now
</a>
<a href="https://github.com/spotty-cloud/spotty" target="_blank" class="btn btn-neutral">
<i class="fa fa-github"></i> View it on GitHub
</a>
</div>
<div id="main-page-features">
<table>
<tr>
<td class="block">
<table>
<tr>
<td class="icon">
<i class="fa fa-cloud-upload fa-4x" style="color: #a35063;"></i>
</td>
<td class="text">
<div class="title">Run Training on AWS and GCP Instances</div>
<div class="description">Spotty makes training of deep learning models on AWS and GCP instances as simple as training on your local machine.</div>
</td>
</tr>
</table>
</td>
<td class="block">
<table>
<tr>
<td class="icon">
<i class="fa fa-dollar fa-4x" style="color: #2d9681;"></i>
</td>
<td class="text">
<div class="title">Reduce Training Costs</div>
<div class="description">Spotty can save you up to 70% of the costs by using <a href="https://aws.amazon.com/ec2/spot/">AWS Spot Instances</a>
or <a href="https://cloud.google.com/preemptible-vms/">GCP Preemtible VMs</a>.</div>
</td>
</tr>
</table>
</td>
</tr>
<tr>
<td class="block">
<table>
<tr>
<td class="icon">
<i class="fa fa-github fa-4x"></i>
</td>
<td class="text">
<div class="title">Share Your Model with Everyone</div>
<div class="description">Spotty makes your model trainable locally or in the cloud by everyone with a couple of commands.</div>
</td>
</tr>
</table>
</td>
<td class="block">
<table>
<tr>
<td class="icon">
<i class="fa fa-cube fa-4x" style="color: #1f89c4;"></i>
</td>
<td class="text">
<div class="title">Develop with Docker</div>
<div class="description">Spotty helps you to develop your model locally using a Docker container, so the environment can be set up by anyone and anywhere with a single command.</div>
</td>
</tr>
</table>
</td>
</tr>
</table>
</div>
</div>
================================================
FILE: setup.cfg
================================================
[metadata]
description-file = README.md
================================================
FILE: setup.py
================================================
#!/usr/bin/env python
import os
import re
from setuptools import setup, find_packages
def get_version():
root_dir = os.path.abspath(os.path.dirname(__file__))
with open(os.path.join(root_dir, 'spotty', '__init__.py')) as f:
content = f.read()
version_match = re.search(r'^__version__ = [\'"]([^\'"]*)[\'"]', content, re.M)
if not version_match:
raise RuntimeError('Unable to find version string.')
return version_match.group(1)
def get_description():
readme_path = os.path.abspath(os.path.join(os.path.dirname(__file__), 'README.md'))
with open(readme_path, encoding='utf-8') as f:
description = f.read()
return description
setup(name='spotty',
version=get_version(),
description='Training deep learning models on AWS and GCP instances',
url='https://github.com/spotty-cloud/spotty',
author='Oleg Polosin',
author_email='apls777@gmail.com',
license='MIT',
long_description=get_description(),
long_description_content_type='text/markdown',
packages=find_packages(exclude=['tests*']),
package_data={
'spotty.deployment.container.docker.scripts': ['data/*', 'data/*/*'],
'spotty.providers.aws.cfn_templates.instance': ['data/*', 'data/*/*'],
'spotty.providers.aws.cfn_templates.instance_profile': ['data/*', 'data/*/*'],
'spotty.providers.gcp.dm_templates.instance': ['data/*', 'data/*/*'],
},
scripts=['bin/spotty'],
install_requires=[
'boto3>=1.9.0',
'google-api-python-client>=1.7.8',
'google-cloud-storage>=1.15.0',
'cfn_flip', # to work with CloudFormation templates
'schema',
'chevron',
],
tests_require=['moto'],
test_suite='tests',
classifiers=[
'Development Status :: 5 - Production/Stable',
'Intended Audience :: Science/Research',
'Intended Audience :: Developers',
'Intended Audience :: System Administrators',
'Natural Language :: English',
'License :: OSI Approved :: MIT License',
'Programming Language :: Python',
'Programming Language :: Python :: 3',
'Programming Language :: Python :: 3.6',
'Programming Language :: Python :: 3.7',
])
================================================
FILE: spotty/__init__.py
================================================
__version__ = '1.3.4'
================================================
FILE: spotty/cli.py
================================================
import argparse
from typing import List, Type
import pkg_resources
from spotty.commands.abstract_command import AbstractCommand
from spotty.commands.aws import AwsCommand
from spotty.commands.download import DownloadCommand
from spotty.commands.exec import ExecCommand
from spotty.commands.run import RunCommand
from spotty.commands.sh import ShCommand
from spotty.commands.start import StartCommand
from spotty.commands.status import StatusCommand
from spotty.commands.stop import StopCommand
from spotty.commands.sync import SyncCommand
def get_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser()
parser.add_argument('-V', '--version', action='store_true', help='Display the version of the Spotty')
command_classes = [
StartCommand,
StopCommand,
StatusCommand,
ShCommand,
RunCommand,
ExecCommand,
SyncCommand,
DownloadCommand,
AwsCommand,
] + _get_custom_commands()
# add commands to the parser
add_subparsers(parser, command_classes)
return parser
def add_subparsers(parser: argparse.ArgumentParser, command_classes: List[Type[AbstractCommand]]):
"""Adds commands to the parser."""
subparsers = parser.add_subparsers()
for command_class in command_classes:
command = command_class()
subparser = subparsers.add_parser(command.name, help=command.description, description=command.description)
subparser.set_defaults(command=command, parser=subparser)
command.configure(subparser)
def _get_custom_commands() -> List[Type[AbstractCommand]]:
"""Returns custom commands that integrated through entry points."""
return [entry_point.load() for entry_point in pkg_resources.iter_entry_points('spotty.commands')]
================================================
FILE: spotty/commands/__init__.py
================================================
================================================
FILE: spotty/commands/abstract_command.py
================================================
from abc import ABC, abstractmethod
from argparse import Namespace, ArgumentParser
from spotty.commands.writers.abstract_output_writrer import AbstractOutputWriter
class AbstractCommand(ABC):
"""Abstract class for a Spotty sub-command."""
@property
@abstractmethod
def name(self) -> str:
"""The sub-command name."""
raise NotImplementedError
@property
def description(self) -> str:
"""The sub-command description. It will be displayed in the help text."""
return ''
def configure(self, parser: ArgumentParser):
"""Adds arguments for the sub-command."""
parser.add_argument('-d', '--debug', action='store_true', help='Show debug messages')
@abstractmethod
def run(self, args: Namespace, output: AbstractOutputWriter):
"""Runs the sub-command.
Args:
args: Arguments provided by argparse.
output: Output writer.
Raises:
ValueError: If command's arguments can't be processed.
"""
raise NotImplementedError
================================================
FILE: spotty/commands/abstract_config_command.py
================================================
from abc import abstractmethod
from typing import List
from argparse import Namespace, ArgumentParser
from spotty.config.config_utils import load_config
from spotty.deployment.abstract_instance_manager import AbstractInstanceManager
from spotty.providers.instance_manager_factory import InstanceManagerFactory
from spotty.commands.abstract_command import AbstractCommand
from spotty.commands.writers.abstract_output_writrer import AbstractOutputWriter
class AbstractConfigCommand(AbstractCommand):
"""Abstract class for a Spotty sub-command that needs to use a project's configuration."""
@abstractmethod
def _run(self, instance_manager: AbstractInstanceManager, args: Namespace, output: AbstractOutputWriter):
raise NotImplementedError
def configure(self, parser: ArgumentParser):
super().configure(parser)
parser.add_argument('-c', '--config', type=str, default=None, help='Path to the configuration file')
parser.add_argument('instance_name', metavar='INSTANCE_NAME', nargs='?', type=str, help='Instance name')
def run(self, args: Namespace, output: AbstractOutputWriter):
# get project configuration
project_config = load_config(args.config)
# get instance configuration
instance_id = self._get_instance_id(project_config.instances, args.instance_name, output)
instance_config = project_config.instances[instance_id]
# create an instance manger
instance_manager = InstanceManagerFactory.get_instance(project_config, instance_config)
# run the command
self._run(instance_manager, args, output)
@staticmethod
def _get_instance_id(instances: List[dict], instance_name: str, output: AbstractOutputWriter):
if not instance_name:
if len(instances) > 1:
# ask user to choose the instance
output.write('Select the instance:\n')
with output.prefix(' '):
for i, instance_config in enumerate(instances):
output.write('[%d] %s' % (i + 1, instance_config['name']))
output.write()
try:
num = int(input('Enter number: '))
output.write()
except ValueError:
num = 0
if num < 1 or num > len(instances):
raise ValueError('The value from 1 to %d was expected.' % len(instances))
instance_id = num - 1
else:
instance_id = 0
else:
# get instance ID by name
instance_ids = [i for i, instance in enumerate(instances) if instance['name'] == instance_name]
if not instance_ids:
raise ValueError('Instance "%s" not found in the configuration file' % instance_name)
instance_id = instance_ids[0]
return instance_id
================================================
FILE: spotty/commands/abstract_provider_command.py
================================================
from abc import abstractmethod
from argparse import Namespace, ArgumentParser
import sys
from spotty.commands.abstract_command import AbstractCommand
from spotty.commands.writers.abstract_output_writrer import AbstractOutputWriter
class AbstractProviderCommand(AbstractCommand):
"""Abstract class for a provider sub-command."""
@property
@abstractmethod
def commands(self) -> list:
"""Returns a list of the provider sub-commands."""
raise NotImplementedError
def configure(self, parser: ArgumentParser):
from spotty.cli import add_subparsers
add_subparsers(parser, self.commands)
def run(self, args: Namespace, output: AbstractOutputWriter):
"""If the command is called, it just displays a list of available sub-commands."""
args.parser.print_help()
sys.exit(1)
================================================
FILE: spotty/commands/aws.py
================================================
from spotty.commands.abstract_provider_command import AbstractProviderCommand
from spotty.providers.aws.commands.clean_logs import CleanLogsCommand
from spotty.providers.aws.commands.spot_prices import SpotPricesCommand
class AwsCommand(AbstractProviderCommand):
name = 'aws'
description = 'AWS commands'
commands = [
SpotPricesCommand,
CleanLogsCommand,
]
================================================
FILE: spotty/commands/download.py
================================================
from argparse import Namespace, ArgumentParser
from spotty.commands.abstract_config_command import AbstractConfigCommand
from spotty.commands.writers.abstract_output_writrer import AbstractOutputWriter
from spotty.errors.instance_not_running import InstanceNotRunningError
from spotty.errors.nothing_to_do import NothingToDoError
from spotty.deployment.abstract_instance_manager import AbstractInstanceManager
class DownloadCommand(AbstractConfigCommand):
name = 'download'
description = 'Download files from the running instance'
def configure(self, parser: ArgumentParser):
super().configure(parser)
parser.add_argument('-i', '--include', metavar='PATTERN', action='append', type=str, required=True,
help='Download all files that matches the specified pattern (see Include Filters '
'for the "aws s3 sync" command). Paths must be relative to your project directory, '
'they cannot be absolute.')
parser.add_argument('--dry-run', action='store_true', help='Show files to be downloaded')
def _run(self, instance_manager: AbstractInstanceManager, args: Namespace, output: AbstractOutputWriter):
# check that the instance is started
if not instance_manager.is_running():
raise InstanceNotRunningError(instance_manager.instance_config.name)
filters = [
{'exclude': ['*']},
{'include': args.include}
]
dry_run = args.dry_run
with output.prefix('[dry-run] ' if dry_run else ''):
try:
instance_manager.download(filters, output, dry_run)
except NothingToDoError as e:
output.write(str(e))
return
output.write('Done')
================================================
FILE: spotty/commands/exec.py
================================================
import sys
from argparse import ArgumentParser, Namespace
from spotty.commands.abstract_config_command import AbstractConfigCommand
from spotty.commands.writers.abstract_output_writrer import AbstractOutputWriter
from spotty.deployment.utils.cli import shlex_join
from spotty.errors.instance_not_running import InstanceNotRunningError
from spotty.errors.nothing_to_do import NothingToDoError
from spotty.deployment.abstract_instance_manager import AbstractInstanceManager
class ExecCommand(AbstractConfigCommand):
name = 'exec'
description = 'Execute a command in the container'
def configure(self, parser: ArgumentParser):
super().configure(parser)
parser.add_argument('-i', '--interactive', action='store_true', help='Pass STDIN to the container')
parser.add_argument('-t', '--tty', action='store_true', help='Allocate a pseudo-TTY')
parser.add_argument('-u', '--user', type=str, default=None,
help='Container username or UID (format: <name|uid>[:<group|gid>])')
parser.add_argument('--no-sync', action='store_true', help='Don\'t sync the project before running the script')
# add the "double-dash" argument to the usage message
parser.prog = 'spotty exec'
parser.usage = parser.format_usage()[7:-1] + ' -- COMMAND [args...]\n'
parser.epilog = 'The double dash (--) separates the command that you want to execute inside the container ' \
'from the Spotty arguments.'
def _run(self, instance_manager: AbstractInstanceManager, args: Namespace, output: AbstractOutputWriter):
# check that the command is provided
if not args.custom_args:
raise ValueError('Use the double-dash ("--") to split Spotty arguments from the command that should be '
'executed inside the container.')
# check that the instance is started
if not instance_manager.is_running():
raise InstanceNotRunningError(instance_manager.instance_config.name)
# sync the project with the instance
if not args.no_sync:
try:
instance_manager.sync(output)
except NothingToDoError:
pass
# generate a "docker exec" command
command = shlex_join(args.custom_args)
command = instance_manager.container_commands.exec(command, interactive=args.interactive, tty=args.tty,
user=args.user)
# execute the command on the host OS
exit_code = instance_manager.exec(command, tty=args.tty)
sys.exit(exit_code)
================================================
FILE: spotty/commands/run.py
================================================
from argparse import ArgumentParser, Namespace
from spotty.commands.abstract_config_command import AbstractConfigCommand
from spotty.commands.writers.abstract_output_writrer import AbstractOutputWriter
from spotty.deployment.utils.commands import get_script_command, get_log_command, get_tmux_session_command, get_bash_command
from spotty.errors.instance_not_running import InstanceNotRunningError
from spotty.errors.nothing_to_do import NothingToDoError
from spotty.deployment.utils.user_scripts import parse_script_parameters, render_script
from spotty.deployment.abstract_instance_manager import AbstractInstanceManager
class RunCommand(AbstractConfigCommand):
name = 'run'
description = 'Run a custom script from the configuration file inside the container'
def configure(self, parser: ArgumentParser):
super().configure(parser)
parser.add_argument('script_name', metavar='SCRIPT_NAME', type=str, help='Script name')
parser.add_argument('-u', '--user', type=str, default=None,
help='Container username or UID (format: <name|uid>[:<group|gid>])')
parser.add_argument('-s', '--session-name', type=str, default=None, help='tmux session name')
parser.add_argument('-l', '--logging', action='store_true', help='Log the script outputs to a file')
parser.add_argument('-p', '--parameter', metavar='PARAMETER=VALUE', action='append', type=str, default=[],
help='Set a value for the script parameter (format: PARAMETER=VALUE). This '
'argument can be used multiple times to set several parameters. Parameters can be '
'used in the script as Mustache variables (for example: {{PARAMETER}}).')
parser.add_argument('--no-sync', action='store_true', help='Don\'t sync the project before running the script')
# add the "double-dash" argument to the usage message
parser.prog = 'spotty run'
parser.usage = parser.format_usage()[7:-1] + ' [-- args...]\n'
parser.epilog = 'The double dash (--) separates custom arguments that you can pass to the script ' \
'from the Spotty arguments.'
def _run(self, instance_manager: AbstractInstanceManager, args: Namespace, output: AbstractOutputWriter):
# check that the script exists
script_name = args.script_name
scripts = instance_manager.project_config.scripts
if script_name not in scripts:
raise ValueError('Script "%s" is not defined in the configuration file.' % script_name)
# replace script parameters
params = parse_script_parameters(args.parameter)
script_content = render_script(scripts[script_name], params)
# check that the instance is started
if not instance_manager.is_running():
raise InstanceNotRunningError(instance_manager.instance_config.name)
# sync the project with the instance
if not args.no_sync:
try:
instance_manager.sync(output)
except NothingToDoError:
pass
# get a command to run the script with "docker exec"
script_command = get_script_command(script_name, script_content, script_args=args.custom_args,
logging=args.logging)
command = instance_manager.container_commands.exec(script_command, interactive=True, tty=True,
user=args.user)
# wrap the command with the tmux session
if instance_manager.use_tmux:
session_name = args.session_name if args.session_name else 'spotty-script-%s' % script_name
default_command = instance_manager.container_commands.exec(get_bash_command(), interactive=True, tty=True,
user=args.user)
command = get_tmux_session_command(command, session_name, script_name, default_command=default_command,
keep_pane=True)
# execute command on the host OS
instance_manager.exec(command)
================================================
FILE: spotty/commands/sh.py
================================================
from argparse import ArgumentParser, Namespace
from spotty.commands.abstract_config_command import AbstractConfigCommand
from spotty.commands.writers.abstract_output_writrer import AbstractOutputWriter
from spotty.deployment.utils.commands import get_bash_command, get_tmux_session_command
from spotty.errors.instance_not_running import InstanceNotRunningError
from spotty.deployment.abstract_instance_manager import AbstractInstanceManager
class ShCommand(AbstractConfigCommand):
name = 'sh'
description = 'Get a shell to the container or to the instance itself'
def configure(self, parser: ArgumentParser):
super().configure(parser)
parser.add_argument('-u', '--user', type=str, default=None,
help='Container username or UID (format: <name|uid>[:<group|gid>])')
parser.add_argument('-H', '--host-os', action='store_true', help='Connect to the host OS instead of the Docker '
'container')
parser.add_argument('-s', '--session-name', type=str, default=None, help='tmux session name')
parser.add_argument('-l', '--list-sessions', action='store_true', help='List all tmux sessions managed by the '
'instance')
def _run(self, instance_manager: AbstractInstanceManager, args: Namespace, output: AbstractOutputWriter):
# check that the instance is started
if not instance_manager.is_running():
raise InstanceNotRunningError(instance_manager.instance_config.name)
if args.list_sessions:
if not instance_manager.use_tmux:
raise ValueError('The "%s" provider doesn\'t support tmux.'
% instance_manager.instance_config.provider_name)
# a command to list existing tmux session on the host OS
command = 'tmux ls; echo ""'
else:
if args.host_os:
# get a command to open a login shell on the host OS
session_name = args.session_name if args.session_name else 'spotty-sh-host'
shell_command = '$SHELL'
command = get_tmux_session_command(shell_command, session_name, keep_pane=False) \
if instance_manager.use_tmux else shell_command
else:
# get a command to run bash inside the docker container
command = instance_manager.container_commands.exec(get_bash_command(), interactive=True, tty=True,
user=args.user)
# wrap the command with the tmux session
if instance_manager.use_tmux:
session_name = args.session_name if args.session_name else 'spotty-sh-container'
command = get_tmux_session_command(command, session_name, default_command=command, keep_pane=False)
# execute command on the host OS
instance_manager.exec(command)
================================================
FILE: spotty/commands/start.py
================================================
from argparse import Namespace, ArgumentParser
from spotty.commands.abstract_config_command import AbstractConfigCommand
from spotty.commands.writers.abstract_output_writrer import AbstractOutputWriter
from spotty.errors.instance_not_running import InstanceNotRunningError
from spotty.deployment.abstract_instance_manager import AbstractInstanceManager
class StartCommand(AbstractConfigCommand):
name = 'start'
description = 'Start an instance with a container'
def configure(self, parser: ArgumentParser):
super().configure(parser)
parser.add_argument('-C', '--container', action='store_true', help='Starts or restarts container on the '
'running instance')
parser.add_argument('--dry-run', action='store_true', help='Displays the steps that would be performed '
'using the specified command without actually '
'running them')
def _run(self, instance_manager: AbstractInstanceManager, args: Namespace, output: AbstractOutputWriter):
dry_run = args.dry_run
if args.container:
# check that the instance is started
if not instance_manager.is_running():
raise InstanceNotRunningError(instance_manager.instance_config.name)
# start a container on the running instance
instance_manager.start_container(output, dry_run=dry_run)
if not dry_run:
instance_name = ''
if len(instance_manager.project_config.instances) > 1:
instance_name = ' ' + instance_manager.instance_config.name
output.write('\nContainer was successfully started.\n'
'Use the "spotty sh%s" command to connect to the container.\n'
% instance_name)
else:
# start the instance
with output.prefix('[dry-run] ' if dry_run else ''):
instance_manager.start(output, dry_run)
if not dry_run:
instance_name = ''
if len(instance_manager.project_config.instances) > 1:
instance_name = ' ' + instance_manager.instance_config.name
output.write('\n%s\n'
'\nUse the "spotty sh%s" command to connect to the container.\n'
% (instance_manager.get_status_text(), instance_name))
================================================
FILE: spotty/commands/status.py
================================================
from argparse import Namespace
from spotty.commands.abstract_config_command import AbstractConfigCommand
from spotty.commands.writers.abstract_output_writrer import AbstractOutputWriter
from spotty.deployment.abstract_instance_manager import AbstractInstanceManager
class StatusCommand(AbstractConfigCommand):
name = 'status'
description = 'Print information about the instance'
def _run(self, instance_manager: AbstractInstanceManager, args: Namespace, output: AbstractOutputWriter):
output.write(instance_manager.get_status_text())
================================================
FILE: spotty/commands/stop.py
================================================
from argparse import Namespace
from spotty.commands.abstract_config_command import AbstractConfigCommand
from spotty.commands.writers.abstract_output_writrer import AbstractOutputWriter
from spotty.deployment.abstract_instance_manager import AbstractInstanceManager
class StopCommand(AbstractConfigCommand):
name = 'stop'
description = 'Terminate running instance and apply deletion policies for the volumes'
# TODO: the "spotty start" command should restart the instance and the container if the instance was shutdown
# def configure(self, parser: ArgumentParser):
# super().configure(parser)
# parser.add_argument('-s', '--shutdown', action='store_true',
# help='Shutdown the instance without terminating it. Deletion policies for the volumes '
# 'won\'t be applied.')
def _run(self, instance_manager: AbstractInstanceManager, args: Namespace, output: AbstractOutputWriter):
instance_manager.stop(only_shutdown=False, output=output)
================================================
FILE: spotty/commands/sync.py
================================================
from argparse import Namespace, ArgumentParser
from spotty.commands.abstract_config_command import AbstractConfigCommand
from spotty.commands.writers.abstract_output_writrer import AbstractOutputWriter
from spotty.errors.instance_not_running import InstanceNotRunningError
from spotty.errors.nothing_to_do import NothingToDoError
from spotty.deployment.abstract_instance_manager import AbstractInstanceManager
class SyncCommand(AbstractConfigCommand):
name = 'sync'
description = 'Synchronize the project with the running instance'
def configure(self, parser: ArgumentParser):
super().configure(parser)
parser.add_argument('--dry-run', action='store_true', help='Show files to be synced')
def _run(self, instance_manager: AbstractInstanceManager, args: Namespace, output: AbstractOutputWriter):
# check that the instance is started
if not instance_manager.is_running():
raise InstanceNotRunningError(instance_manager.instance_config.name)
dry_run = args.dry_run
with output.prefix('[dry-run] ' if dry_run else ''):
try:
instance_manager.sync(output, dry_run)
except NothingToDoError as e:
output.write(str(e))
return
output.write('Done')
================================================
FILE: spotty/commands/writers/__init__.py
================================================
================================================
FILE: spotty/commands/writers/abstract_output_writrer.py
================================================
from abc import ABC, abstractmethod
from contextlib import contextmanager
class AbstractOutputWriter(ABC):
def __init__(self):
self._prefix = ''
self._ignore_prefix = False
@abstractmethod
def _write(self, msg: str, newline: bool = True):
raise NotImplementedError
def write(self, msg: str = '', newline: bool = True):
if not self._ignore_prefix:
msg = '\n'.join([self._prefix + line for line in msg.split('\n')])
self._write(msg, newline=newline)
self._ignore_prefix = not newline
@contextmanager
def prefix(self, prefix):
self._prefix += prefix
yield
self._prefix = self._prefix[:-len(prefix)]
================================================
FILE: spotty/commands/writers/null_output_writrer.py
================================================
from spotty.commands.writers.abstract_output_writrer import AbstractOutputWriter
class NullOutputWriter(AbstractOutputWriter):
def _write(self, msg: str, newline: bool = True):
"""Does nothing."""
pass
================================================
FILE: spotty/commands/writers/output_writrer.py
================================================
from spotty.commands.writers.abstract_output_writrer import AbstractOutputWriter
class OutputWriter(AbstractOutputWriter):
def _write(self, msg: str, newline: bool = True):
"""Prints messages to STDOUT."""
print(msg, end=('\n' if newline else ''), flush=True)
================================================
FILE: spotty/config/__init__.py
================================================
================================================
FILE: spotty/config/abstract_instance_config.py
================================================
import os
from abc import ABC, abstractmethod
from collections import OrderedDict, namedtuple
from typing import List
from spotty.config.container_config import ContainerConfig
from spotty.config.project_config import ProjectConfig
from spotty.config.tmp_dir_volume import TmpDirVolume
from spotty.config.validation import DEFAULT_CONTAINER_NAME, is_subdir
from spotty.config.abstract_instance_volume import AbstractInstanceVolume
from spotty.deployment.abstract_cloud_instance.file_structure import INSTANCE_SPOTTY_TMP_DIR, CONTAINERS_TMP_DIR
from spotty.utils import filter_list
VolumeMount = namedtuple('VolumeMount', ['name', 'host_path', 'mount_path', 'mode', 'hidden'])
class AbstractInstanceConfig(ABC):
def __init__(self, instance_config: dict, project_config: ProjectConfig):
self._project_config = project_config
# set instance parameters
self._name = instance_config['name']
self._provider_name = instance_config['provider']
self._params = self._validate_instance_params(instance_config['parameters'])
# get container config
container_configs = filter_list(project_config.containers, 'name', self.container_name)
if not container_configs:
raise ValueError('Container configuration with the name "%s" not found.' % self.container_name)
self._container_config = ContainerConfig(container_configs[0])
# get volumes
self._volumes = self._get_volumes()
# get container volume mounts
self._volume_mounts = self._get_volume_mounts(self._volumes)
# get the host project directory
self._host_project_dir = self._get_host_project_dir(self._volume_mounts)
@abstractmethod
def _validate_instance_params(self, params: dict) -> dict:
"""Validates instance parameters and fill missing ones with the default values."""
raise NotImplementedError
@abstractmethod
def _get_instance_volumes(self) -> List[AbstractInstanceVolume]:
"""Returns specific to the provider volumes that should be mounted on the host OS."""
raise NotImplementedError
@property
def project_config(self) -> ProjectConfig:
return self._project_config
@property
def container_config(self) -> ContainerConfig:
return self._container_config
@property
@abstractmethod
def user(self) -> str:
raise NotImplementedError
@property
def name(self) -> str:
"""Name of the instance."""
return self._name
@property
def provider_name(self):
"""Provider name."""
return self._provider_name
@property
def container_name(self) -> str:
return self._params['containerName'] if self._params['containerName'] else DEFAULT_CONTAINER_NAME
@property
def full_container_name(self) -> str:
"""A container name that is used in the "docker run" command."""
return ('spotty-%s-%s-%s' % (self.project_config.project_name, self.name, self.container_name)).lower()
@property
def docker_data_root(self) -> str:
"""Data root directory for Docker daemon."""
return self._params['dockerDataRoot']
@property
def local_ssh_port(self) -> int:
"""Local SSH port to connect to the instance (in case of a tunnel)."""
return self._params['localSshPort']
@property
def commands(self) -> str:
"""Commands that should be run once an instance is started."""
return self._params['commands']
@property
def host_project_dir(self):
"""Project directory on the host OS."""
return self._host_project_dir
@property
def volumes(self) -> List[AbstractInstanceVolume]:
return self._volumes
@property
def volume_mounts(self) -> List[VolumeMount]:
return self._volume_mounts
@property
def dockerfile_path(self):
"""Dockerfile path on the host OS."""
dockerfile_path = self.container_config.file
if dockerfile_path:
dockerfile_path = self.host_project_dir + '/' + dockerfile_path
return dockerfile_path
@property
def docker_context_path(self):
"""Docker build's context path on the host OS."""
dockerfile_path = self.dockerfile_path
if not dockerfile_path:
return ''
return os.path.dirname(dockerfile_path)
@property
def host_container_dir(self):
"""A temporary directory on the host OS that contains container-related files and directories."""
return '%s/%s' % (CONTAINERS_TMP_DIR, self.full_container_name)
@property
def host_logs_dir(self):
"""A directory mainly for the "spotty run" command logs."""
return self.host_container_dir + '/logs'
@property
def host_volumes_dir(self):
"""A directory with temporary volumes. If there is a Volume Mount in the configuration file
that doesn't have a corresponding instance volume, a temporary directory will be created
and attached to the container.
"""
return self.host_container_dir + '/volumes'
def _get_volumes(self) -> List[AbstractInstanceVolume]:
"""Returns volumes that should be mounted on the host OS."""
volumes = self._get_instance_volumes()
# create temporary volumes for the volume mounts that don't have corresponding
# volumes in the instance configuration
instance_volume_names = set(volume.name for volume in volumes)
for container_volume in self.container_config.volume_mounts:
if container_volume['name'] not in instance_volume_names:
volumes.append(TmpDirVolume(volume_config={
'name': container_volume['name'],
'parameters': {'path': '%s/%s' % (self.host_volumes_dir, container_volume['name'])}
}))
return volumes
def _get_volume_mounts(self, volumes: List[AbstractInstanceVolume]) \
-> List[VolumeMount]:
"""Returns container volume mounts and a path to the project directory on the host OS."""
# get mount directories for the volumes
host_paths = OrderedDict([(volume.name, volume.host_path) for volume in volumes])
# get container volumes mapping
volume_mounts = []
for container_volume in self.container_config.volume_mounts:
volume_mounts.append(VolumeMount(
name=container_volume['name'],
host_path=host_paths[container_volume['name']],
mount_path=container_volume['mountPath'],
mode='rw',
hidden=False,
))
return volume_mounts
def _get_host_project_dir(self, volume_mounts: List[VolumeMount]) -> str:
"""Returns the host project directory."""
host_project_dir = None
for volume_mount in sorted(volume_mounts, key=lambda x: len(x.mount_path), reverse=True):
if is_subdir(self.container_config.project_dir, volume_mount.mount_path):
# the project directory is a subdirectory of a Volume Mount directory
project_subdir = os.path.relpath(self.container_config.project_dir, volume_mount.mount_path)
host_project_dir = os.path.normpath(volume_mount.host_path + '/' + project_subdir)
break
# this should not be the case as the volume mount for the project directory should be added automatically
# if it doesn't exist in the configuration
assert host_project_dir is not None, 'A volume mount that contains the project directory not found.'
return host_project_dir
================================================
FILE: spotty/config/abstract_instance_volume.py
================================================
from abc import ABC, abstractmethod
class AbstractInstanceVolume(ABC):
def __init__(self, volume_config: dict):
self._name = volume_config['name']
self._params = self._validate_volume_parameters(volume_config['parameters'])
@abstractmethod
def _validate_volume_parameters(self, params: dict) -> dict:
raise NotImplementedError
@property
def name(self) -> str:
"""Unique name of the volume that will be used for the deployment."""
return self._name
@property
@abstractmethod
def host_path(self) -> str:
"""A path on the host OS that will be mounted to the container."""
raise NotImplementedError
@property
@abstractmethod
def title(self) -> str:
"""A title for the volume type.
It will be used to display information about the volumes during the deployment.
"""
raise NotImplementedError
@property
def deletion_policy_title(self) -> str:
"""A title for the volume's deletion policy.
It will be used to display information about the volumes during the deployment.
"""
return ''
================================================
FILE: spotty/config/config_utils.py
================================================
import os
from collections import namedtuple
import yaml
from spotty.config.project_config import ProjectConfig
from spotty.config.validation import DEFAULT_CONTAINER_NAME
DEFAULT_CONFIG_FILENAME = 'spotty.yaml'
OVERRIDE_CONFIG_FILENAME = 'spotty.override.yaml'
def load_config(config_path: str = None) -> ProjectConfig:
# get project directory
if not config_path:
config_path = DEFAULT_CONFIG_FILENAME
if os.path.isabs(config_path):
config_abs_path = config_path
else:
config_abs_path = os.path.abspath(os.path.join(os.getcwd(), config_path))
if not os.path.exists(config_abs_path):
raise ValueError('Configuration file "%s" not found.' % config_path)
# get the project directory
project_dir = os.path.dirname(config_abs_path)
# read the config
config = _read_yaml(config_abs_path)
# update the config if an override config exists
if os.path.basename(config_abs_path) == DEFAULT_CONFIG_FILENAME:
override_config_abs_path = os.path.join(project_dir, OVERRIDE_CONFIG_FILENAME)
if os.path.isfile(override_config_abs_path):
override_config = _read_yaml(override_config_abs_path)
config = _merge_configs(config, override_config)
# get project configuration
project_config = ProjectConfig(config, project_dir)
return project_config
def _read_yaml(file_path: str):
"""Returns content of the YAML file."""
with open(file_path, 'r') as f:
res = yaml.safe_load(f)
return res
def _merge_configs(orig_config, override_config):
"""Merges original config with the override config."""
MergeRule = namedtuple('MergeRule', ['key', 'merge_key', 'default_value', 'has_default_value'])
merge_rules = [MergeRule(
key='containers',
merge_key='name',
default_value=DEFAULT_CONTAINER_NAME,
has_default_value=True,
), MergeRule(
key='instances',
merge_key='name',
default_value=None,
has_default_value=False,
)]
# validate and merge lists by keys
for rule in merge_rules:
if override_config and (rule.key in orig_config) and (rule.key in override_config):
if not isinstance(orig_config[rule.key], list):
raise ValueError('The "%s" key in the config must be a list.' % rule.key)
if not isinstance(override_config[rule.key], list):
raise ValueError('The "%s" key in the override config must be a list.' % rule.key)
# convert lists to dictionaries
dicts_to_merge = []
for list_to_merge in [orig_config[rule.key], override_config[rule.key]]:
dict_to_merge = {}
for item in list_to_merge:
if not isinstance(item, dict):
raise ValueError('Each item of the "%s" list must be a dictionary.' % rule.key)
if rule.merge_key in item:
merge_value = item[rule.merge_key]
elif rule.has_default_value:
merge_value = rule.default_value
else:
raise ValueError('Each item of the "%s" list must contain the "%s" field.'
% (rule.key, rule.merge_key))
if merge_value in dict_to_merge:
raise ValueError('Each item of the "%s" list must have a unique "%s" value.'
% (rule.key, rule.merge_key))
dict_to_merge[merge_value] = item
dicts_to_merge.append(dict_to_merge)
# merge lists
merged_dict = _update_dict(*dicts_to_merge)
orig_config[rule.key] = [{**item, rule.merge_key: key} for key, item in merged_dict.items()]
del override_config[rule.key]
# merge the rest of the override config
config = _update_dict(orig_config, override_config)
return config
def _update_dict(d, u):
if not isinstance(u, dict):
return d
if not isinstance(d, dict):
return u
for k, v in u.items():
if isinstance(d, dict):
if isinstance(v, dict):
d[k] = _update_dict(d.get(k, {}), v)
else:
d[k] = u[k]
else:
d = {k: u[k]}
return d
================================================
FILE: spotty/config/container_config.py
================================================
from typing import List
from spotty.config.validation import is_subdir
PROJECT_VOLUME_MOUNT_NAME = '.project'
class ContainerConfig(object):
def __init__(self, container_config: dict):
self._config = container_config
self._volume_mounts = self._get_volume_mounts()
@property
def name(self) -> str:
return self._config['name']
@property
def project_dir(self) -> str:
return self._config['projectDir']
@property
def image(self) -> str:
return self._config['image']
@property
def file(self) -> str:
return self._config['file']
@property
def run_as_host_user(self) -> str:
return self._config['runAsHostUser']
@property
def volume_mounts(self) -> list:
return self._volume_mounts
@property
def commands(self) -> str:
return self._config['commands']
@property
def working_dir(self) -> str:
"""Working directory for the Docker container."""
working_dir = self._config['workingDir']
if not working_dir:
working_dir = self._config['projectDir']
return working_dir
@property
def env(self) -> dict:
return self._config['env']
@property
def host_network(self) -> bool:
return self._config['hostNetwork']
@property
def ports(self) -> List[dict]:
return self._config['ports']
@property
def runtime_parameters(self) -> list:
return self._config['runtimeParameters']
def _get_volume_mounts(self):
"""Returns container volume mounts from the configuration and
adds the project volume mount if necessary."""
volume_mounts = self._config['volumeMounts']
# check if the project directory is a sub-directory of one of the volume mounts
project_has_volume = False
for volume_mount in volume_mounts:
if is_subdir(self.project_dir, volume_mount['mountPath']):
project_has_volume = True
break
# if it's not, then add new volume mount
if not project_has_volume:
volume_mounts.insert(0, {
'name': PROJECT_VOLUME_MOUNT_NAME,
'mountPath': self.project_dir,
})
return volume_mounts
================================================
FILE: spotty/config/host_path_volume.py
================================================
import os
from spotty.config.abstract_instance_volume import AbstractInstanceVolume
from spotty.config.validation import validate_host_path_volume_parameters
class HostPathVolume(AbstractInstanceVolume):
TYPE_NAME = 'HostPath'
def __init__(self, volume_config: dict, base_dir: str = None):
super().__init__(volume_config)
self._base_dir = base_dir
def _validate_volume_parameters(self, params: dict) -> dict:
return validate_host_path_volume_parameters(params)
@property
def title(self):
return 'HostPath volume'
@property
def name(self):
return self._name
@property
def deletion_policy_title(self) -> str:
return ''
@property
def host_path(self) -> str:
"""A path on the host OS that will be mounted to the container."""
path = os.path.expanduser(self._params['path'])
if not os.path.isabs(path):
if self._base_dir is not None:
path = os.path.join(self._base_dir, path)
else:
raise ValueError('Use absolute path for the "%s" volume.' % self.name)
return path
================================================
FILE: spotty/config/project_config.py
================================================
from spotty.config.validation import validate_basic_config
class ProjectConfig(object):
def __init__(self, config: dict, project_dir: str):
# validate the config
config = validate_basic_config(config)
self._project_dir = project_dir
self._config = config
@property
def project_dir(self) -> str:
return self._project_dir
@property
def project_name(self) -> str:
return self._config['project']['name']
@property
def sync_filters(self) -> list:
return self._config['project']['syncFilters']
@property
def containers(self) -> list:
return self._config['containers']
@property
def instances(self) -> list:
return self._config['instances']
@property
def scripts(self) -> dict:
return self._config['scripts']
================================================
FILE: spotty/config/tmp_dir_volume.py
================================================
from spotty.config.host_path_volume import HostPathVolume
class TmpDirVolume(HostPathVolume):
@property
def title(self):
return 'temporary directory'
@property
def deletion_policy_title(self) -> str:
return ''
================================================
FILE: spotty/config/validation.py
================================================
import os
from typing import List
from schema import Schema, And, Use, Optional, Or, Regex, Hook, SchemaError, SchemaForbiddenKeyError
DEFAULT_CONTAINER_NAME = 'default'
def validate_basic_config(data):
container = And(
{
Optional('name', default=DEFAULT_CONTAINER_NAME): And(str, Regex(r'^[\w-]+$')),
'projectDir': And(str,
And(os.path.isabs,
error='Use an absolute path when specifying the project directory'),
Use(lambda x: x.rstrip('/'))
),
Optional('image', default=''): And(str, len),
Optional('file', default=''): And(str, # TODO: a proper regex that the filename is valid
Regex(r'^[\w\.\/@-]*$',
error='Invalid name for a Dockerfile'),
And(lambda x: not x.endswith('/'),
error='Invalid name for a Dockerfile'),
And(lambda x: not os.path.isabs(x),
error='Path to the Dockerfile should be relative to the '
'project\'s root directory.'),
),
Optional('runAsHostUser', default=False): bool,
Optional('volumeMounts', default=[]): (And(
[{
'name': And(Or(int, str), Use(str), Regex(r'^[\w-]+$')),
'mountPath': And(
str,
And(os.path.isabs, error='Use an absolute path when specifying a mount directory'),
Use(lambda x: x.rstrip('/')),
),
}],
And(lambda x: is_unique_value(x, 'name'),
error='Each volume mount must have a unique name.'),
And(lambda x: not has_prefix([(volume['mountPath'] + '/') for volume in x]),
error='Volume mount paths cannot be prefixes for each other.'),
)),
Optional('workingDir', default=''): And(str,
And(os.path.isabs,
error='Use an absolute path when specifying a '
'working directory'),
Use(lambda x: x.rstrip('/'))
),
Optional('env', default={}): {
And(str, Regex(r'^[a-zA-Z_]+[a-zA-Z0-9_]*$')): str,
},
Optional('hostNetwork', default=False): bool,
Optional('ports', default=[]): [{
'containerPort': And(int, lambda x: 0 < x < 65536),
Optional('hostPort', default=None): And(int, lambda x: 0 < x < 65536),
}],
Optional('commands', default=''): str,
# TODO: allow to use only certain runtime parameters
Optional('runtimeParameters', default=[]): And([str], Use(lambda x: [p.strip() for p in x])),
},
And(lambda x: x['image'] or x['file'], error='Either "image" or "file" should be specified.'),
And(lambda x: not (x['image'] and x['file']), error='"image" and "file" cannot be specified together.'),
And(lambda x: not (x['hostNetwork'] and x['ports']),
error='Published ports and the host network mode cannot be used together.'),
)
schema = Schema({
'project': {
'name': And(str, Regex(r'^[a-zA-Z0-9][a-zA-Z0-9-]{,26}[a-zA-Z0-9]$')),
Optional('syncFilters', default=[]): And(
[And(
{
Optional('exclude'): [And(str, len, And(lambda x: '**' not in x,
error='Use single asterisks ("*") in sync filters'))],
Optional('include'): [And(str, len, And(lambda x: '**' not in x,
error='Use single asterisks ("*") in sync filters'))],
},
And(lambda x: x, error='Either "exclude" or "include" filter should be specified.'),
And(lambda x: not ('exclude' in x and 'include' in x),
error='"exclude" and "include" filters should be specified as different list items.'),
)],
error='"project.syncFilters" field must be a list.',
)
},
WrongKey('container', error='Use "containers" field instead of "container".'): object,
Optional('containers', default=[]): And(
[container],
And(lambda x: is_unique_value(x, 'name'), error='Each container must have a unique name.'),
error='"containers" field must be a list.',
),
WrongKey('instance', error='Use "instances" field instead of "instance".'): object,
'instances': And(
[{
'name': And(Or(int, str), Use(str), Regex(r'^[\w-]+$')),
'provider': str,
Optional('parameters', default={}): {
And(str, Regex(r'^[\w]+$')): object,
}
}],
And(lambda x: len(x), error='At least one instance must be specified in the configuration file.'),
And(lambda x: is_unique_value(x, 'name'), error='Each instance must have a unique name.'),
),
Optional('scripts', default={}): {
And(str, Regex(r'^[\w-]+$')): And(str, len),
},
})
return validate_config(schema, data)
def validate_host_path_volume_parameters(params: dict):
schema = Schema({
'path': And(str, Use(lambda x: x.rstrip('/'))),
})
return validate_config(schema, params)
def get_instance_parameters_schema(instance_parameters: dict, default_volume_type: str,
instance_checks: list = None, volumes_checks: list = None):
if not instance_checks:
instance_checks = []
if not volumes_checks:
volumes_checks = []
schema = Schema(And(
{
**instance_parameters,
Optional('containerName', default=None): And(str, Regex(r'^[\w-]+$')),
Optional('dockerDataRoot', default=''): And(
str,
And(os.path.isabs, error='Use an absolute path when specifying a Docker data root directory'),
Use(lambda x: x.rstrip('/')),
),
Optional('volumes', default=[]): And(
[{
'name': And(Or(int, str), Use(str), Regex(r'^[\w-]+$')),
Optional('type', default=default_volume_type): str,
Optional('parameters', default={}): {
And(str, Regex(r'^[\w]+$')): object,
},
}],
And(lambda x: is_unique_value(x, 'name'), error='Each instance volume must have a unique name.'),
*volumes_checks,
),
Optional('localSshPort', default=None): Or(None, And(int, lambda x: 0 < x < 65536)),
Optional('commands', default=''): str,
},
And(lambda x: not x['dockerDataRoot'] or any([True for v in x['volumes'] if v['parameters'].get('mountDir') and
is_subdir(x['dockerDataRoot'], v['parameters']['mountDir'])]),
error='The "mountDir" of one of the volumes must be a prefix for the "dockerDataRoot" path.'),
*instance_checks
))
return schema
def is_unique_value(x: List[dict], key):
"""Returns "True" if all values of the key in the list of dictionaries are unique."""
return len(x) == len(set([v[key] for v in x]))
def has_prefix(x: list):
"""Returns "True" if some value in the list is a prefix for another value in this list."""
for val in x:
if len(list(filter(val.startswith, x))) > 1:
return True
return False
def is_subdir(subdir_path, dir_path):
"""Returns "True" if it's the second path parameter is a subdirectory of the first path parameter."""
return (subdir_path.rstrip('/') + '/').startswith(dir_path.rstrip('/') + '/')
def validate_config(schema: Schema, config):
try:
validated = schema.validate(config)
except SchemaError as e:
raise ValueError('Validation error: ' + (e.errors[-1] if e.errors[-1] else e.autos[-1]))
return validated
class WrongKey(Hook):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs, handler=self.raise_error)
def raise_error(self, key, *args):
raise SchemaForbiddenKeyError(self._error)
================================================
FILE: spotty/configuration.py
================================================
import os
def get_spotty_config_dir():
"""Spotty configuration directory."""
path = os.path.join(os.path.expanduser('~'), '.spotty')
if not os.path.isdir(path):
os.makedirs(path, mode=0o755, exist_ok=True)
return path
def get_spotty_keys_dir(provider_name: str):
""""Spotty keys directory."""
path = os.path.join(get_spotty_config_dir(), 'keys', provider_name)
if not os.path.isdir(path):
os.makedirs(path, mode=0o755, exist_ok=True)
return path
================================================
FILE: spotty/deployment/__init__.py
================================================
================================================
FILE: spotty/deployment/abstract_cloud_instance/__init__.py
================================================
================================================
FILE: spotty/deployment/abstract_cloud_instance/abstract_bucket_manager.py
================================================
from abc import ABC
from spotty.deployment.abstract_cloud_instance.resources.abstract_bucket import AbstractBucket
class AbstractBucketManager(ABC):
def __init__(self, project_name: str):
self._project_name = project_name
@property
def project_name(self) -> str:
return self._project_name
def get_bucket(self) -> AbstractBucket:
raise NotImplementedError
def create_bucket(self) -> AbstractBucket:
raise NotImplementedError
================================================
FILE: spotty/deployment/abstract_cloud_instance/abstract_cloud_instance_manager.py
================================================
import logging
from abc import ABC, abstractmethod
from spotty.commands.writers.abstract_output_writrer import AbstractOutputWriter
from spotty.config.project_config import ProjectConfig
from spotty.deployment.abstract_cloud_instance.abstract_data_transfer import AbstractDataTransfer
from spotty.deployment.abstract_cloud_instance.abstract_instance_deployment import AbstractInstanceDeployment
from spotty.deployment.abstract_cloud_instance.abstract_bucket_manager import AbstractBucketManager
from spotty.deployment.abstract_cloud_instance.errors.bucket_not_found import BucketNotFoundError
from spotty.errors.instance_not_running import InstanceNotRunningError
from spotty.deployment.abstract_ssh_instance_manager import AbstractSshInstanceManager
class AbstractCloudInstanceManager(AbstractSshInstanceManager, ABC):
def __init__(self, project_config: ProjectConfig, instance_config: dict):
super().__init__(project_config, instance_config)
self._bucket_manager = self._get_bucket_manager()
self._data_transfer = self._get_data_transfer()
self._instance_deployment = self._get_instance_deployment()
@abstractmethod
def _get_bucket_manager(self) -> AbstractBucketManager:
"""Returns an bucket manager."""
raise NotImplementedError
@abstractmethod
def _get_data_transfer(self) -> AbstractDataTransfer:
"""Returns a data transfer object."""
raise NotImplementedError
@abstractmethod
def _get_instance_deployment(self) -> AbstractInstanceDeployment:
"""Returns an instance deployment manager."""
raise NotImplementedError
@property
def bucket_manager(self) -> AbstractBucketManager:
"""Returns a bucket manager."""
return self._bucket_manager
@property
def data_transfer(self) -> AbstractDataTransfer:
"""Returns a data transfer object."""
return self._data_transfer
@property
def instance_deployment(self) -> AbstractInstanceDeployment:
"""Returns an instance deployment manager."""
return self._instance_deployment
def is_running(self) -> bool:
"""Checks if the instance is running."""
instance = self.instance_deployment.get_instance()
return instance and instance.is_running
def start(self, output: AbstractOutputWriter, dry_run=False):
# make sure the Dockerfile exists
self._check_dockerfile_exists()
if not dry_run:
# check if the instance is already running
instance = self.instance_deployment.get_instance()
if instance:
if instance.is_running:
print('Instance is already running. Are you sure you want to restart it?')
res = input('Type "y" to confirm: ')
if res != 'y':
raise ValueError('The operation was cancelled.')
# terminating the instance to make EBS volumes available (the stack will be deleted later)
output.write('Terminating the instance... ', newline=False)
instance.terminate()
output.write('DONE')
elif instance.is_stopped:
# TODO: restart the instance if it stopped
pass
# create or get existing bucket for the project
bucket_name = None
try:
bucket_name = self.bucket_manager.get_bucket().name
except BucketNotFoundError:
if not dry_run:
bucket_name = self.bucket_manager.create_bucket().name
output.write('Bucket "%s" was created.' % bucket_name)
# deploy the instance
self.instance_deployment.deploy(
container_commands=self.container_commands,
bucket_name=bucket_name,
data_transfer=self.data_transfer,
output=output,
dry_run=dry_run,
)
def stop(self, only_shutdown: bool, output: AbstractOutputWriter):
if only_shutdown:
output.write('Shutting down the instance... ', newline=False)
self.instance_deployment.get_instance().stop()
output.write('DONE')
else:
# delete the stack and apply deletion policies
self.instance_deployment.delete(output)
def clean(self, output: AbstractOutputWriter):
pass
def sync(self, output: AbstractOutputWriter, dry_run=False):
# get the project bucket name
bucket_name = self.bucket_manager.get_bucket().name
# sync the project with the S3 bucket
output.write('Syncing the project with the bucket...')
self.data_transfer.upload_local_to_bucket(bucket_name, dry_run=dry_run)
if not dry_run:
# sync the S3 bucket with the instance
output.write('Syncing the bucket with the instance...')
remote_cmd = self.data_transfer.get_download_bucket_to_instance_command(
bucket_name=bucket_name,
use_sudo=(not self.instance_config.container_config.run_as_host_user),
)
logging.debug('Remote sync command: ' + remote_cmd)
# execute the command on the host OS
exit_code = self.exec(remote_cmd)
if exit_code != 0:
raise ValueError('Failed to download files from the bucket to the instance')
def download(self, download_filters: list, output: AbstractOutputWriter, dry_run=False):
# get the project bucket name
bucket_name = self.bucket_manager.get_bucket().name
# sync files from the instance to a temporary S3 directory
output.write('Uploading files from the instance to the bucket...')
remote_cmd = self.data_transfer.get_upload_instance_to_bucket_command(
bucket_name=bucket_name,
download_filters=download_filters,
use_sudo=(not self.instance_config.container_config.run_as_host_user),
dry_run=dry_run,
)
logging.debug('Remote sync command: ' + remote_cmd)
# execute the command on the host OS
exit_code = self.exec(remote_cmd)
if exit_code != 0:
raise ValueError('Failed to upload files from the instance to the bucket')
if not dry_run:
# sync the project with the S3 bucket
output.write('Downloading files from the bucket to local...')
self.data_transfer.download_bucket_to_local(bucket_name=bucket_name, download_filters=download_filters)
@property
def ssh_host(self):
"""Returns an IP address that will be used for SSH connections."""
if self._instance_config.local_ssh_port:
return '127.0.0.1'
# get a public IP address of the running instance
instance = self.instance_deployment.get_instance()
if not instance or not instance.is_running:
raise InstanceNotRunningError(self.instance_config.name)
instance_ip_address = instance.public_ip_address if instance.public_ip_address else instance.private_ip_address
if not instance_ip_address:
raise ValueError('Instance IP address not found')
return instance_ip_address
@property
def ssh_port(self) -> int:
if self._instance_config.local_ssh_port:
return self._instance_config.local_ssh_port
return 22
@property
def use_tmux(self) -> bool:
return True
================================================
FILE: spotty/deployment/abstract_cloud_instance/abstract_data_transfer.py
================================================
from abc import ABC, abstractmethod
class AbstractDataTransfer(ABC):
def __init__(self, local_project_dir: str, host_project_dir: str, sync_filters: list, instance_name: str):
self._instance_name = instance_name
self._local_project_dir = local_project_dir
self._host_project_dir = host_project_dir
self._sync_filters = sync_filters
@property
def instance_name(self):
return self._instance_name
@property
@abstractmethod
def scheme_name(self) -> str:
raise NotImplementedError
def _get_bucket_project_path(self, bucket_name: str) -> str:
"""A bucket path where the project files are located."""
return '%s://%s/project' % (self.scheme_name, bucket_name)
def _get_bucket_downloads_path(self, bucket_name: str) -> str:
"""A bucket path where the downloaded files are located."""
return '%s://%s/download/instance-%s' % (self.scheme_name, bucket_name, self.instance_name)
@abstractmethod
def upload_local_to_bucket(self, bucket_name: str, dry_run: bool = False):
"""Uploads files from local to the bucket."""
raise NotImplementedError
@abstractmethod
def download_bucket_to_local(self, bucket_name: str, download_filters: list):
"""Downloads files from the bucket to local."""
raise NotImplementedError
@abstractmethod
def get_download_bucket_to_instance_command(self, bucket_name: str, use_sudo: bool = False) -> str:
"""A remote command to download files from the bucket to the instance."""
raise NotImplementedError
@abstractmethod
def get_upload_instance_to_bucket_command(self, bucket_name: str, download_filters: list, use_sudo: bool = False,
dry_run: bool = False) -> str:
"""A remote command to upload files from the instance to the bucket."""
raise NotImplementedError
================================================
FILE: spotty/deployment/abstract_cloud_instance/abstract_instance_deployment.py
================================================
from abc import abstractmethod, ABC
from spotty.commands.writers.abstract_output_writrer import AbstractOutputWriter
from spotty.config.abstract_instance_config import AbstractInstanceConfig
from spotty.deployment.abstract_cloud_instance.abstract_data_transfer import AbstractDataTransfer
from spotty.deployment.abstract_cloud_instance.resources.abstract_instance import AbstractInstance
from spotty.deployment.container.abstract_container_commands import AbstractContainerCommands
class AbstractInstanceDeployment(ABC):
def __init__(self, instance_config: AbstractInstanceConfig):
self._instance_config = instance_config
@property
def instance_config(self) -> AbstractInstanceConfig:
return self._instance_config
@abstractmethod
def get_instance(self) -> AbstractInstance:
"""Returns information about the instance it it exists."""
raise NotImplementedError
@abstractmethod
def deploy(self, container_commands: AbstractContainerCommands, bucket_name: str,
data_transfer: AbstractDataTransfer, output: AbstractOutputWriter, dry_run: bool = False):
"""Deploys or redeploys the instance."""
raise NotImplementedError
@abstractmethod
def delete(self, output: AbstractOutputWriter):
"""Deletes the stack with the instance and applies deletion policies for the volumes."""
raise NotImplementedError
================================================
FILE: spotty/deployment/abstract_cloud_instance/errors/__init__.py
================================================
================================================
FILE: spotty/deployment/abstract_cloud_instance/errors/bucket_not_found.py
================================================
class BucketNotFoundError(Exception):
def __init__(self):
super().__init__('Bucket for the project not found.')
================================================
FILE: spotty/deployment/abstract_cloud_instance/file_structure.py
================================================
"""
INSTANCE FILE STRUCTURE
"""
# a base temporary directory on an instance
INSTANCE_SPOTTY_TMP_DIR = '/tmp/spotty'
# a base directory for container-related files and directories
CONTAINERS_TMP_DIR = INSTANCE_SPOTTY_TMP_DIR + '/containers'
# a base directory for instance-related files and directories
INSTANCE_DIR = INSTANCE_SPOTTY_TMP_DIR + '/instance'
# helper scripts
INSTANCE_SCRIPTS_DIR = INSTANCE_DIR + '/scripts'
# instance startup scripts
INSTANCE_STARTUP_SCRIPTS_DIR = INSTANCE_SCRIPTS_DIR + '/startup'
# a path to the script that attaches user to the container
CONTAINER_BASH_SCRIPT_PATH = INSTANCE_SCRIPTS_DIR + '/container_bash.sh'
================================================
FILE: spotty/deployment/abstract_cloud_instance/resources/__init__.py
================================================
================================================
FILE: spotty/deployment/abstract_cloud_instance/resources/abstract_bucket.py
================================================
from abc import ABC
class AbstractBucket(ABC):
@property
def name(self):
raise NotImplementedError
================================================
FILE: spotty/deployment/abstract_cloud_instance/resources/abstract_instance.py
================================================
from abc import ABC
class AbstractInstance(ABC):
@property
def public_ip_address(self):
raise NotImplementedError
@property
def private_ip_address(self):
raise NotImplementedError
@property
def is_running(self):
"""Returns true if the instance is running."""
raise NotImplementedError
@property
def is_stopped(self):
"""Returns true if the instance is stopped, so it can be restarted."""
raise NotImplementedError
def terminate(self, wait: bool = True):
raise NotImplementedError
def stop(self, wait: bool = True):
raise NotImplementedError
================================================
FILE: spotty/deployment/abstract_docker_instance_manager.py
================================================
import os
from abc import ABC
from spotty.commands.writers.abstract_output_writrer import AbstractOutputWriter
from spotty.deployment.utils.commands import get_script_command
from spotty.deployment.container.docker.docker_commands import DockerCommands
from spotty.deployment.container.docker.scripts.start_container_script import StartContainerScript
from spotty.deployment.container.docker.scripts.stop_container_script import StopContainerScript
from spotty.deployment.abstract_instance_manager import AbstractInstanceManager
from spotty.errors.nothing_to_do import NothingToDoError
from spotty.utils import render_table
class AbstractDockerInstanceManager(AbstractInstanceManager, ABC):
@property
def container_commands(self) -> DockerCommands:
"""A collection of commands to manage a container from the host OS."""
return DockerCommands(self.instance_config)
def is_container_running(self) -> bool:
"""Checks if the container is running."""
is_running_cmd = self.container_commands.is_created(is_running=True)
exit_code = self.exec(is_running_cmd)
return exit_code == 0
def start_container(self, output: AbstractOutputWriter, dry_run=False):
"""Starts or restarts container on the host OS."""
# make sure the Dockerfile exists
self._check_dockerfile_exists()
# sync the project with the instance
try:
self.sync(output, dry_run=dry_run)
except NothingToDoError:
pass
# generate a script that starts container
start_container_script = StartContainerScript(self.container_commands).render()
start_container_command = get_script_command('start-container', start_container_script)
# start the container
exit_code = self.exec(start_container_command)
if exit_code != 0:
raise ValueError('Failed to start the container')
def start(self, output: AbstractOutputWriter, dry_run=False):
# start or restart container
self.start_container(output, dry_run=dry_run)
def stop(self, only_shutdown: bool, output: AbstractOutputWriter):
# stop container
stop_container_script = StopContainerScript(self.container_commands).render()
stop_container_command = get_script_command('stop-container', stop_container_script)
exit_code = self.exec(stop_container_command)
if exit_code != 0:
raise ValueError('Failed to stop the container')
def get_status_text(self):
if self.is_container_running():
msg = 'Container is running.'
else:
msg = 'Container is not running.'
return render_table([(msg,)])
def _check_dockerfile_exists(self):
"""Raises an error if a Dockerfile specified in the configuration file but doesn't exist."""
if self.instance_config.container_config.file:
dockerfile_path = os.path.join(self.project_config.project_dir, self.instance_config.container_config.file)
if not os.path.isfile(dockerfile_path):
raise FileNotFoundError('A Dockerfile specified in the container configuration doesn\'t exist:\n ' +
dockerfile_path)
================================================
FILE: spotty/deployment/abstract_instance_manager.py
================================================
import subprocess
from abc import ABC, abstractmethod
from spotty.commands.writers.abstract_output_writrer import AbstractOutputWriter
from spotty.config.abstract_instance_config import AbstractInstanceConfig
from spotty.config.project_config import ProjectConfig
from spotty.deployment.container.abstract_container_commands import AbstractContainerCommands
class AbstractInstanceManager(ABC):
def __init__(self, project_config: ProjectConfig, instance_config: dict):
self._project_config = project_config
self._instance_config = self._get_instance_config(instance_config)
@property
def project_config(self) -> ProjectConfig:
return self._project_config
@property
def instance_config(self) -> AbstractInstanceConfig:
return self._instance_config
@abstractmethod
def _get_instance_config(self, instance_config: dict) -> AbstractInstanceConfig:
"""A factory method to create a provider's instance config."""
raise NotImplementedError
@property
@abstractmethod
def container_commands(self) -> AbstractContainerCommands:
"""A collection of commands to manage a container from the host OS."""
raise NotImplementedError
@abstractmethod
def is_running(self) -> bool:
"""Checks if the instance is running."""
raise NotImplementedError
@abstractmethod
def start(self, output: AbstractOutputWriter, dry_run=False):
"""Creates a stack with the instance."""
raise NotImplementedError
@abstractmethod
def start_container(self, output: AbstractOutputWriter, dry_run=False):
"""Starts or restarts container on the host OS."""
raise NotImplementedError
@abstractmethod
def stop(self, only_shutdown: bool, output: AbstractOutputWriter):
"""Deletes the stack."""
raise NotImplementedError
def exec(self, command: str, tty: bool = True) -> int:
"""Executes a command on the host OS."""
return subprocess.call(command, shell=True)
@abstractmethod
def clean(self, output: AbstractOutputWriter):
"""Deletes the stack."""
raise NotImplementedError
@abstractmethod
def sync(self, output: AbstractOutputWriter, dry_run=False):
"""Synchronizes the project code with the instance."""
raise NotImplementedError
@abstractmethod
def download(self, download_filters: list, output: AbstractOutputWriter, dry_run=False):
"""Downloads files from the instance."""
raise NotImplementedError
@abstractmethod
def get_status_text(self) -> str:
"""Returns information about the started instance.
It will be shown to the user once the instance is started and by using the "status" command.
"""
raise NotImplementedError
@property
def use_tmux(self) -> bool:
"""Use tmux when running a custom script or connecting to the instance."""
return False
================================================
FILE: spotty/deployment/abstract_ssh_instance_manager.py
================================================
import logging
import os
from abc import abstractmethod
from spotty.deployment.utils.commands import get_ssh_command
from spotty.deployment.abstract_docker_instance_manager import AbstractDockerInstanceManager
class AbstractSshInstanceManager(AbstractDockerInstanceManager):
def exec(self, command: str, tty: bool = True) -> int:
"""Executes a command on the host OS."""
if not os.path.isfile(self.ssh_key_path):
raise ValueError('SSH key doesn\'t exist: ' + self.ssh_key_path)
ssh_command = get_ssh_command(self.ssh_host, self.ssh_port, self.ssh_user, self.ssh_key_path,
command, env_vars=self.ssh_env_vars, tty=tty)
logging.debug('SSH command: ' + ssh_command)
return super().exec(ssh_command)
@property
@abstractmethod
def ssh_host(self):
raise NotImplementedError
@property
@abstractmethod
def ssh_port(self) -> int:
raise NotImplementedError
@property
@abstractmethod
def ssh_key_path(self) -> str:
raise NotImplementedError
@property
def ssh_user(self) -> str:
return self.instance_config.user
@property
def ssh_env_vars(self) -> dict:
"""Environmental variables that will be set when ssh to the instance."""
return {
'SPOTTY_CONTAINER_NAME': self.instance_config.full_container_name,
'SPOTTY_CONTAINER_WORKING_DIR': self.instance_config.container_config.working_dir,
}
@property
def use_tmux(self) -> bool:
"""Use tmux when running a custom script or connecting to the instance."""
return True
================================================
FILE: spotty/deployment/container/__init__.py
================================================
================================================
FILE: spotty/deployment/container/abstract_container_commands.py
================================================
from abc import ABC, abstractmethod
from spotty.config.abstract_instance_config import AbstractInstanceConfig
class AbstractContainerCommands(ABC):
def __init__(self, instance_config: AbstractInstanceConfig):
self._instance_config = instance_config
@property
def instance_config(self) -> AbstractInstanceConfig:
return self._instance_config
@abstractmethod
def exec(self, command: str, interactive: bool = False, tty: bool = False, user: str = None,
container_name: str = None, working_dir: str = None) -> str:
raise NotImplementedError
================================================
FILE: spotty/deployment/container/abstract_container_script.py
================================================
from abc import ABC, abstractmethod
from spotty.deployment.container.abstract_container_commands import AbstractContainerCommands
class AbstractContainerScript(ABC):
def __init__(self, container_commands: AbstractContainerCommands):
self._commands = container_commands
@property
def commands(self) -> AbstractContainerCommands:
return self._commands
@abstractmethod
def render(self) -> str:
raise NotImplementedError
================================================
FILE: spotty/deployment/container/docker/__init__.py
================================================
================================================
FILE: spotty/deployment/container/docker/docker_commands.py
================================================
import shlex
from spotty.deployment.container.abstract_container_commands import AbstractContainerCommands
from spotty.deployment.utils.cli import shlex_join
class DockerCommands(AbstractContainerCommands):
def build(self, image_name: str) -> str:
if not self._instance_config.dockerfile_path:
raise ValueError('Cannot generate the "build" command as Dockerfile path is not specified')
if not self._instance_config.docker_context_path:
raise ValueError('Cannot generate the "build" command as Docker context path is not set')
build_cmd = 'docker build -t %s -f %s %s' % (image_name, shlex.quote(self._instance_config.dockerfile_path),
shlex.quote(self._instance_config.docker_context_path))
if self._instance_config.container_config.run_as_host_user:
build_cmd += ' --build-arg USER_ID=$(id -u %s) --build-arg GROUP_ID=$(id -g %s)' \
% (self._instance_config.user, self._instance_config.user)
return build_cmd
def pull(self) -> str:
return 'docker pull ' + self._instance_config.container_config.image
def run(self, image_name: str = None) -> str:
image_name = image_name if image_name else self._instance_config.container_config.image
# prepare "docker run" arguments
args = ['-td'] + self._instance_config.container_config.runtime_parameters
if self._instance_config.container_config.host_network:
args += ['--net=host']
for port in self._instance_config.container_config.ports:
host_port = port['hostPort']
container_port = port['containerPort']
args += ['-p', ('%d:%d' % (host_port, container_port)) if host_port else str(container_port)]
for volume_mount in self._instance_config.volume_mounts:
args += ['-v', '%s:%s:%s' % (volume_mount.host_path, volume_mount.mount_path, volume_mount.mode)]
for env_name, env_value in self._instance_config.container_config.env.items():
args += ['-e', '%s=%s' % (env_name, env_value)]
args += ['--name', self._instance_config.full_container_name]
run_cmd = 'docker run $(nvidia-smi &> /dev/null && echo "--gpus all")'
if self._instance_config.container_config.run_as_host_user:
run_cmd += ' -u $(id -u %s):$(id -g %s) -e HOST_USER_ID=$(id -u %s) -e HOST_GROUP_ID=$(id -g %s)' \
% tuple([self._instance_config.user] * 4)
run_cmd += ' %s %s /bin/sh > /dev/null' % (shlex_join(args), image_name)
return run_cmd
def is_created(self, container_name: str = None, is_running: bool = False) -> str:
container_name = container_name if container_name else self._instance_config.full_container_name
show_all = '' if is_running else 'a'
test_cmd = '[ $(docker ps -q%s --filter name="%s" | wc -c) -ne 0 ]' % (show_all, container_name)
return test_cmd
def remove(self):
return 'docker rm -f "%s" > /dev/null' % self._instance_config.full_container_name
def exec(self, command: str, interactive: bool = False, tty: bool = False, user: str = None,
container_name: str = None, working_dir: str = None) -> str:
container_name = container_name if container_name else self._instance_config.full_container_name
working_dir = working_dir if working_dir else self._instance_config.container_config.working_dir
exec_cmd = 'docker exec'
if interactive:
exec_cmd += ' -i'
if tty:
exec_cmd += ' -t'
if user:
exec_cmd += ' -u ' + shlex.quote(user)
if working_dir:
# no quoting, it can be environmental variable
exec_cmd += ' -w ' + working_dir
exec_cmd += ' %s %s' % (container_name, command)
# run "exec" command only if the container is running
test_cmd = self.is_created(container_name, is_running=True)
error_msg = 'Container is not running.\\nUse the "spotty start -C" command to start it.\\n'
cond_exec_cmd = 'if %s; then %s; else printf %s; exit 1; fi' % (test_cmd, exec_cmd, shlex.quote(error_msg))
return cond_exec_cmd
================================================
FILE: spotty/deployment/container/docker/scripts/__init__.py
================================================
================================================
FILE: spotty/deployment/container/docker/scripts/abstract_docker_script.py
================================================
from abc import ABC
from spotty.deployment.container.docker.docker_commands import DockerCommands
from spotty.deployment.container.abstract_container_script import AbstractContainerScript
class AbstractDockerScript(AbstractContainerScript, ABC):
@property
def commands(self) -> DockerCommands:
return self._commands
================================================
FILE: spotty/deployment/container/docker/scripts/container_bash_script.py
================================================
import os
import chevron
from spotty.deployment.utils.commands import get_bash_command
from spotty.deployment.container.docker.scripts.abstract_docker_script import AbstractDockerScript
class ContainerBashScript(AbstractDockerScript):
def render(self) -> str:
# read template file
template_path = os.path.join(os.path.dirname(__file__), 'data', 'container_bash.sh.tpl')
with open(template_path) as f:
template = f.read()
# render the script
content = chevron.render(template, data={
'docker_exec_bash': self.commands.exec(get_bash_command(), interactive=True, tty=True,
container_name='$SPOTTY_CONTAINER_NAME',
working_dir='$SPOTTY_CONTAINER_WORKING_DIR'),
})
return content
================================================
FILE: spotty/deployment/container/docker/scripts/data/container_bash.sh.tpl
================================================
#!/bin/bash -e
if [ -z "$SPOTTY_CONTAINER_NAME" ]; then
echo -e "\nSPOTTY_CONTAINER_NAME environmental variable is not set.\n"
exit 1
fi
SPOTTY_CONTAINER_WORKING_DIR=${SPOTTY_CONTAINER_WORKING_DIR:-/}
{{{docker_exec_bash}}}
================================================
FILE: spotty/deployment/container/docker/scripts/data/start_container.sh.tpl
================================================
#!/usr/bin/env bash
{{bash_flags}}
if {{{is_created_cmd}}}; then
printf 'Removing existing container... '
{{{remove_cmd}}}
echo 'DONE'
fi
{{> before_image_build}}
{{#build_image_cmd}}
echo 'Building Docker image...'
{{{build_image_cmd}}}
{{/build_image_cmd}}
{{> before_container_run}}
{{#pull_image_cmd}}
i=0
until [ "$i" -ge 3 ]
do
if [ "$i" -ne 0 ]; then
echo "Retrying to pull the image $i..."
fi
PULL_EXIT_CODE=0
{{{pull_image_cmd}}} || PULL_EXIT_CODE=$?
if [ "$PULL_EXIT_CODE" -ne 125 ]; then
break
fi
i=$((i+1))
sleep 10
done
if [ "$PULL_EXIT_CODE" -ne 0 ]; then
exit $PULL_EXIT_CODE
fi
{{/pull_image_cmd}}
printf 'Starting container... '
{{{start_container_cmd}}}
echo 'DONE'
{{> before_startup_commands}}
{{#docker_exec_startup_script_cmd}}
echo 'Running startup commands...'
{{{docker_exec_startup_script_cmd}}}
{{/docker_exec_startup_script_cmd}}
================================================
FILE: spotty/deployment/container/docker/scripts/data/stop_container.sh.tpl
================================================
#!/usr/bin/env bash
set -e
if {{{is_created_cmd}}}; then
printf 'Removing the container... '
{{{remove_cmd}}}
echo 'DONE'
else
echo 'Container is not running.'
fi
================================================
FILE: spotty/deployment/container/docker/scripts/start_container_script.py
================================================
import os
import time
import chevron
from spotty.deployment.utils.commands import get_script_command
from spotty.deployment.container.docker.scripts.abstract_docker_script import AbstractDockerScript
class StartContainerScript(AbstractDockerScript):
def _partials(self) -> dict:
return {
'before_image_build': '',
'before_container_run': '',
'before_startup_commands': '',
}
def render(self, print_trace: bool = False) -> str:
# read template file
template_path = os.path.join(os.path.dirname(__file__), 'data', 'start_container.sh.tpl')
with open(template_path) as f:
template = f.read()
# generate "docker build" command if necessary
if self.commands.instance_config.dockerfile_path:
image_name = '%s:%d' % (self.commands.instance_config.full_container_name, time.time())
build_image_cmd = self.commands.build(image_name)
pull_image_cmd = ''
else:
image_name = self.commands.instance_config.container_config.image
build_image_cmd = ''
pull_image_cmd = self.commands.pull()
# generate a command to run the startup script
exec_script_cmd = ''
if self.commands.instance_config.container_config.commands:
startup_script_cmd = get_script_command('container-startup-commands',
self.commands.instance_config.container_config.commands)
exec_script_cmd = self.commands.exec(startup_script_cmd, user='root')
# render the script
content = chevron.render(template, data={
'bash_flags': 'set -xe' if print_trace else 'set -e',
'is_created_cmd': self.commands.is_created(),
'remove_cmd': self.commands.remove(),
'build_image_cmd': build_image_cmd,
'pull_image_cmd': pull_image_cmd,
'tmp_container_dir': self.commands.instance_config.host_container_dir,
'start_container_cmd': self.commands.run(image_name),
'docker_exec_startup_script_cmd': exec_script_cmd,
}, partials_dict=self._partials())
return content
================================================
FILE: spotty/deployment/container/docker/scripts/stop_container_script.py
================================================
import os
import chevron
from spotty.deployment.container.docker.scripts.abstract_docker_script import AbstractDockerScript
class StopContainerScript(AbstractDockerScript):
def render(self) -> str:
# read template file
template_path = os.path.join(os.path.dirname(__file__), 'data', 'stop_container.sh.tpl')
with open(template_path) as f:
template = f.read()
# render the script
content = chevron.render(template, data={
'is_created_cmd': self.commands.is_created(),
'remove_cmd': self.commands.remove(),
})
return content
================================================
FILE: spotty/deployment/utils/__init__.py
================================================
================================================
FILE: spotty/deployment/utils/cli.py
================================================
import shlex
def shlex_join(split_command: list):
"""Return a shell-escaped string from *split_command*.
Copy-pasted from the Python 3.8 code.
"""
return ' '.join(shlex.quote(arg) for arg in split_command)
================================================
FILE: spotty/deployment/utils/commands.py
================================================
import base64
import os
import shlex
import time
from spotty.deployment.utils.cli import shlex_join
def get_bash_command() -> str:
return '/usr/bin/env bash'
def get_script_command(script_name: str, script_content: str, script_args: list = None,
logging: bool = False) -> str:
"""Encodes a multi-line script into base64 and returns a one-line command
that unpacks the script to a temporary file and runs it."""
# encode the script content to base64
script_base64 = base64.b64encode(script_content.encode('utf-8')).decode('utf-8')
# command to decode the script, save it to a temporary file and run inside the container
script_args = shlex_join(script_args) if script_args else ''
script_cmd = ' && '.join([
'TMPDIR=${TMPDIR%/}',
'TMP_SCRIPT_PATH=$(mktemp ${TMPDIR:-/tmp}/spotty-%s.XXXXXXXX)' % script_name,
'chmod +x $TMP_SCRIPT_PATH',
'echo %s | base64 -d > $TMP_SCRIPT_PATH' % script_base64,
'$TMP_SCRIPT_PATH ' + script_args,
])
# log the command output to a file
if logging:
log_file_path = '/var/log/spotty/run/%s-%d.log' % (script_name, time.time())
script_cmd = get_log_command(script_cmd, log_file_path)
# execute the command with bash
script_cmd = '%s -c %s' % (get_bash_command(), shlex.quote(script_cmd))
return script_cmd
def get_log_command(command: str, log_file_path: str) -> str:
# log the command outputs to a file on the host OS
log_dir = os.path.dirname(log_file_path)
log_cmd = '; '.join([
'set -o pipefail',
' && '.join([
'mkdir -pm 777 ' + shlex.quote(log_dir),
'(%s) 2>&1 | tee %s' % (command, shlex.quote(log_file_path)),
]),
])
return log_cmd
def get_tmux_session_command(command: str, session_name: str, window_name: str = None, default_command: str = None,
keep_pane: bool = False) -> str:
session_cmd = 'tmux new -A -s ' + session_name
if window_name:
session_cmd += ' -n ' + window_name
if command:
# keep the pane alive when the script is finished
keep_pane_cmd = 'tmux set -w remain-on-exit on; ' if keep_pane else ''
# set the default command (to automatically run bash inside the container when a new window is created)
default_command_cmd = ('tmux set default-command %s; ' % shlex.quote(default_command)) \
if default_command else ''
# keep the pane alive if the script is failed
tmux_cmd = '%s%s(%s) || tmux set -w remain-on-exit on' % (keep_pane_cmd, default_command_cmd, command)
# run the command inside the tmux session
session_cmd += ' ' + shlex.quote(tmux_cmd)
# use tmux only if it's installed
session_cmd = 'if command -v tmux &> /dev/null; then %s; else %s; fi' % (session_cmd, command)
return session_cmd
def get_ssh_command(host: str, port: int, user: str, key_path: str, command: str, env_vars: dict = None,
tty: bool = True, quiet: bool = False) -> str:
ssh_command = 'ssh -i %s -o StrictHostKeyChecking=no -o ConnectTimeout=10' % shlex.quote(key_path)
if tty:
ssh_command += ' -t'
if port != 22:
ssh_command += ' -p %d' % port
if quiet:
ssh_command += ' -q'
# export environmental variables
if env_vars:
export_cmd = '; '.join(['export %s=%s' % (name, shlex.quote(val)) for name, val in env_vars.items()])
command = '%s; %s' % (export_cmd, command)
# final SSH command
ssh_command += ' %s@%s %s' % (user, host, shlex.quote(command))
return ssh_command
================================================
FILE: spotty/deployment/utils/print_info.py
================================================
from typing import List
from spotty.config.abstract_instance_config import VolumeMount
from spotty.config.abstract_instance_volume import AbstractInstanceVolume
from spotty.config.container_config import PROJECT_VOLUME_MOUNT_NAME
from spotty.utils import render_table
def render_volumes_info_table(volume_mounts: List[VolumeMount], volumes: List[AbstractInstanceVolume]):
table = [('Name', 'Mount Path', 'Type', 'Deletion Policy')]
# add volume mounts to the info table
volumes_dict = {volume.name: volume for volume in volumes}
for volume_mount in volume_mounts:
if not volume_mount.hidden:
# the volume will be mounted to the container
volume = volumes_dict[volume_mount.name]
vol_mount_name = '-' if volume_mount.name == PROJECT_VOLUME_MOUNT_NAME else volume_mount.name
deletion_policy = volume.deletion_policy_title if volume.deletion_policy_title else '-'
table.append((vol_mount_name, volume_mount.mount_path, volume.title, deletion_policy))
# add volumes that were not mounted to the container to the info table
volume_mounts_dict = {volume_mount.name for volume_mount in volume_mounts}
for volume in volumes:
if volume.name not in volume_mounts_dict:
deletion_policy = volume.deletion_policy_title if volume.deletion_policy_title else '-'
table.append((volume.name, '-', volume.title, deletion_policy))
return render_table(table, separate_title=True)
================================================
FILE: spotty/deployment/utils/user_scripts.py
================================================
import re
import chevron
from spotty.deployment.utils.commands import get_bash_command
def parse_script_parameters(script_params: str):
"""Parses script parameters."""
params = {}
for param in script_params:
match = re.match('(\\w+)=(.*)', param)
if not match:
raise ValueError('Invalid format for the script parameter: "%s" (the "PARAMETER=VALUE" format is expected).'
% param)
param_name, param_value = match.groups()
if param_name in params:
raise ValueError('Parameter "%s" was defined twice.' % param_name)
params[param_name] = param_value
return params
def render_script(template: str, params: dict):
"""Renders a script template.
It based on the Mustache templates, but only
variables and delimiter changes are allowed.
Raises an exception if one of the provided parameters doesn't
exist in the template.
"""
tokens = list(chevron.tokenizer.tokenize(template))
template_keys = set()
for tag, key in tokens:
if tag not in ['literal', 'no escape', 'variable', 'set delimiter']:
raise ValueError('Script templates support only variables and delimiter changes.')
template_keys.add(key)
# check that the script contains keys for all provided parameters
for key in params:
if key not in template_keys:
raise ValueError('Parameter "%s" doesn\'t exist in the script.' % key)
content = chevron.render(tokens, params)
if content[:2] != '#!':
content = ('#!%s\n\nset -xe\n\n' % get_bash_command()) + content
return content
================================================
FILE: spotty/errors/__init__.py
================================================
================================================
FILE: spotty/errors/instance_not_running.py
================================================
class InstanceNotRunningError(Exception):
def __init__(self, instance_name: str):
super().__init__('Instance "%s" is not running.\n'
'Use the "spotty start %s" command to start the instance.'
% (instance_name, instance_name))
================================================
FILE: spotty/errors/nothing_to_do.py
================================================
class NothingToDoError(Exception):
pass
================================================
FILE: spotty/providers/__init__.py
================================================
================================================
FILE: spotty/providers/aws/__init__.py
================================================
================================================
FILE: spotty/providers/aws/cfn_templates/__init__.py
================================================
================================================
FILE: spotty/providers/aws/cfn_templates/instance/__init__.py
================================================
================================================
FILE: spotty/providers/aws/cfn_templates/instance/data/files/tmux.conf
================================================
bind-key x kill-pane
================================================
FILE: spotty/providers/aws/cfn_templates/instance/data/startup_scripts/01_prepare_instance.sh
================================================
#!/bin/bash -xe
cfn-signal -e 0 --stack ${AWS::StackName} --region ${AWS::Region} --resource PreparingInstanceSignal
# install AWS CLI
update-locale LANG=en_US.UTF-8 LC_ALL=en_US.UTF-8
apt-get update && apt-get install -y python3-pip
pip3 install -U awscli
aws configure set default.region ${AWS::Region}
# install jq
apt-get install -y jq
# create an alias to connect to the docker container
CONTAINER_BASH_ALIAS=container
echo "alias $CONTAINER_BASH_ALIAS=\"{{CONTAINER_BASH_SCRIPT_PATH}}\"" >> /home/ubuntu/.bashrc
echo "alias $CONTAINER_BASH_ALIAS=\"{{CONTAINER_BASH_SCRIPT_PATH}}\"" >> /root/.bashrc
# create common temporary directories
mkdir -pm 777 '{{SPOTTY_TMP_DIR}}'
mkdir -pm 777 '{{CONTAINERS_TMP_DIR}}'
================================================
FILE: spotty/providers/aws/cfn_templates/instance/data/startup_scripts/02_mount_volumes.sh
================================================
#!/bin/bash -xe
cfn-signal -e 0 --stack ${AWS::StackName} --region ${AWS::Region} --resource MountingVolumesSignal
# mount volumes
DEVICE_LETTERS=(f g h i j k l m n o p)
MOUNT_DIRS=({{{MOUNT_DIRS}}})
for i in ${!!MOUNT_DIRS[*]}
do
MOUNT_DIR=${!MOUNT_DIRS[$i]}
DEVICE=/dev/xvd${!DEVICE_LETTERS[$i]}
# NVMe EBS volume (see: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/nvme-ebs-volumes.html)
if [ ! -b $DEVICE ]; then
VOLUME_ID=$(cfn-get-metadata --stack ${AWS::StackName} --region ${AWS::Region} --resource VolumeAttachment${!DEVICE_LETTERS[$i]^} -k VolumeId)
DEVICE=$(lsblk -o NAME,SERIAL -dpJ | jq -rc ".blockdevices[] | select(.serial == \"${!VOLUME_ID//-}\") | .name")
if [ -z "$DEVICE" ]; then
echo "Device for the volume $VOLUME_ID not found"
exit 1
fi
fi
blkid -o value -s TYPE $DEVICE || mkfs -t ext4 $DEVICE
mkdir -p $MOUNT_DIR
mount $DEVICE $MOUNT_DIR
chmod 777 $MOUNT_DIR
resize2fs $DEVICE
done
# create directories for temporary container volumes
{{#TMP_VOLUME_DIRS}}
mkdir -p {{PATH}}
chmod 777 {{PATH}}
{{/TMP_VOLUME_DIRS}}
================================================
FILE: spotty/providers/aws/cfn_templates/instance/data/startup_scripts/03_set_docker_root.sh
================================================
#!/bin/bash -xe
cfn-signal -e 0 --stack ${AWS::StackName} --region ${AWS::Region} --resource SettingDockerRootSignal
# change docker data root directory
if [ -n "${DockerDataRootDirectory}" ]; then
jq '. + { "data-root": "${DockerDataRootDirectory}" }' /etc/docker/daemon.json > /tmp/docker_daemon.json \
&& mv /tmp/docker_daemon.json /etc/docker/daemon.json
service docker restart
fi
================================================
FILE: spotty/providers/aws/cfn_templates/instance/data/startup_scripts/04_sync_project.sh
================================================
#!/bin/bash -xe
cfn-signal -e 0 --stack ${AWS::StackName} --region ${AWS::Region} --resource SyncingProjectSignal
# create a project directory
if [ -n "${HostProjectDirectory}" ]; then
mkdir -p 777 ${HostProjectDirectory}
chmod 777 ${HostProjectDirectory}
if [ -d '${HostProjectDirectory}/lost+found' ]; then
chmod 777 '${HostProjectDirectory}/lost+found'
fi
fi
# sync project files from S3 bucket to the instance
{{{SYNC_PROJECT_CMD}}}
================================================
FILE: spotty/providers/aws/cfn_templates/instance/data/startup_scripts/05_run_instance_startup_commands.sh
================================================
#!/bin/bash -xe
cfn-signal -e 0 --stack ${AWS::StackName} --region ${AWS::Region} --resource RunningInstanceStartupCommandsSignal
/bin/bash -xe {{INSTANCE_STARTUP_SCRIPTS_DIR}}/instance_startup_commands.sh
================================================
FILE: spotty/providers/aws/cfn_templates/instance/data/startup_scripts/user_data.sh
================================================
#!/bin/bash -x
cd /root || exit 1
# install CloudFormation tools if they are not installed yet
if [ ! -e /usr/local/bin/cfn-init ]; then
apt-get update
apt-get install -y python-setuptools
mkdir -p aws-cfn-bootstrap-latest
curl https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz | tar xz -C aws-cfn-bootstrap-latest --strip-components 1
python2 -m easy_install aws-cfn-bootstrap-latest
fi
# prepare the instance and run Docker container
cfn-init \
--stack ${AWS::StackName} \
--region ${AWS::Region} \
--resource InstanceLaunchTemplate \
-c init \
-v
STACK_CREATED=$?
# uplooad cfn-init logs to the bucket
if [ $STACK_CREATED -ne 0 ]; then
STACK_ID=${AWS::StackId}
STACK_UUID=${!STACK_ID##*/}
aws s3 cp /var/log/cfn-init-cmd.log ${LogsS3Path}/$STACK_UUID/cfn-init-cmd.log
fi
# send signal that the Docker container is ready or failed
cfn-signal \
-e $STACK_CREATED \
--stack ${AWS::StackName} \
--region ${AWS::Region} \
--resource DockerReadyWaitCondition
================================================
FILE: spotty/providers/aws/cfn_templates/instance/data/template.yaml
================================================
Description: Spotty EC2 Instance
Parameters:
VpcId:
Description: VPC ID
Type: AWS::EC2::VPC::Id
InstanceProfileArn:
Description: Instance Profile ARN
Type: String
InstanceType:
Description: Instance type
Type: String
KeyName:
Description: EC2 Key Pair name
Type: AWS::EC2::KeyPair::KeyName
ImageId:
Description: AMI ID
Type: AWS::EC2::Image::Id
RootVolumeSize:
Description: Root volume size
Type: String
DockerDataRootDirectory:
Description: Docker data root directory
Type: String
Default: ''
InstanceNameTag:
Description: Name for the instance
Type: String
Default: ''
HostProjectDirectory:
Description: Destination directory for the project
Type: String
Default: ''
LogsS3Path:
Description: An S3 path where logs will be uploaded in the case of a failure
Type: String
Default: ''
Resources:
Instance:
Type: AWS::EC2::Instance
Properties:
LaunchTemplate:
LaunchTemplateId: !Ref InstanceLaunchTemplate
Version: !GetAtt InstanceLaunchTemplate.LatestVersionNumber
InstanceLaunchTemplate:
Type: AWS::EC2::LaunchTemplate
Properties:
LaunchTemplateData:
InstanceType: !Ref InstanceType
ImageId: !Ref ImageId
KeyName: !Ref KeyName
EbsOptimized: 'false'
TagSpecifications:
- ResourceType: instance
Tags:
- Key: Name
Value: !Ref InstanceNameTag
IamInstanceProfile:
Arn: !Ref InstanceProfileArn
SecurityGroupIds:
- !Ref InstanceSecurityGroup
InstanceInitiatedShutdownBehavior: terminate
InstanceMarketOptions:
MarketType: spot
SpotOptions:
SpotInstanceType: one-time
InstanceInterruptionBehavior: terminate
BlockDeviceMappings:
- DeviceName: /dev/sda1
Ebs:
VolumeSize: !Ref RootVolumeSize
DeleteOnTermination: true
UserData: ''
Metadata:
'AWS::CloudFormation::Init': {}
InstanceSecurityGroup:
Type: AWS::EC2::SecurityGroup
Properties:
VpcId: !Ref VpcId
GroupDescription: Spotty security group
SecurityGroupEgress:
- CidrIp: 0.0.0.0/0
IpProtocol: -1
FromPort: 0
ToPort: 65535
- CidrIpv6: ::/0
IpProtocol: -1
FromPort: 0
ToPort: 65535
SecurityGroupIngress:
- CidrIp: 0.0.0.0/0
IpProtocol: tcp
FromPort: 22
ToPort: 22
- CidrIpv6: ::/0
IpProtocol: tcp
FromPort: 22
ToPort: 22
PreparingInstanceSignal:
Type: AWS::CloudFormation::WaitCondition
DependsOn: Instance
CreationPolicy:
ResourceSignal:
Timeout: PT30M
MountingVolumesSignal:
Type: AWS::CloudFormation::WaitCondition
DependsOn: Instance
CreationPolicy:
ResourceSignal:
Timeout: PT30M
SettingDockerRootSignal:
Type: AWS::CloudFormation::WaitCondition
DependsOn: Instance
CreationPolicy:
ResourceSignal:
Timeout: PT60M
SyncingProjectSignal:
Type: AWS::CloudFormation::WaitCondition
DependsOn: Instance
CreationPolicy:
ResourceSignal:
Timeout: PT60M
RunningInstanceStartupCommandsSignal:
Type: AWS::CloudFormation::WaitCondition
DependsOn: Instance
CreationPolicy:
ResourceSignal:
Timeout: PT60M
BuildingDockerImageSignal:
Type: AWS::CloudFormation::WaitCondition
DependsOn: Instance
CreationPolicy:
ResourceSignal:
Timeout: PT60M
StartingContainerSignal:
Type: AWS::CloudFormation::WaitCondition
DependsOn: Instance
CreationPolicy:
ResourceSignal:
Timeout: PT60M
RunningContainerStartupCommandsSignal:
Type: AWS::CloudFormation::WaitCondition
DependsOn: Instance
CreationPolicy:
ResourceSignal:
Timeout: PT60M
DockerReadyWaitCondition:
Type: AWS::CloudFormation::WaitCondition
DependsOn: Instance
CreationPolicy:
ResourceSignal:
Timeout: PT30M
Outputs:
InstanceId:
Value: !Ref Instance
AvailabilityZone:
Value: !GetAtt Instance.AvailabilityZone
================================================
FILE: spotty/providers/aws/cfn_templates/instance/start_container_script.py
================================================
from spotty.deployment.container.docker.scripts.start_container_script import StartContainerScript
class StartContainerScriptWithCfnSignals(StartContainerScript):
@staticmethod
def _get_signal_command(resource_name: str):
return 'cfn-signal -e 0 --stack $_{AWS::StackName} --region $_{AWS::Region} --resource ' + resource_name
def _partials(self) -> dict:
return {
'before_image_build': self._get_signal_command('BuildingDockerImageSignal'),
'before_container_run': self._get_signal_command('StartingContainerSignal'),
'before_startup_commands': self._get_signal_command('RunningContainerStartupCommandsSignal'),
}
def render(self, print_trace: bool = False) -> str:
content = super().render(print_trace=print_trace)
content = content.replace('${', '${!')
content = content.replace('$_{', '${')
return content
================================================
FILE: spotty/providers/aws/cfn_templates/instance/template.py
================================================
from typing import List
import os
import chevron
import yaml
from cfn_tools import CfnYamlLoader, CfnYamlDumper
from spotty.commands.writers.abstract_output_writrer import AbstractOutputWriter
from spotty.config.tmp_dir_volume import TmpDirVolume
from spotty.config.validation import is_subdir
from spotty.config.abstract_instance_volume import AbstractInstanceVolume
from spotty.deployment.container.docker.docker_commands import DockerCommands
from spotty.deployment.container.docker.scripts.container_bash_script import ContainerBashScript
from spotty.deployment.abstract_cloud_instance.file_structure import INSTANCE_SPOTTY_TMP_DIR, \
CONTAINER_BASH_SCRIPT_PATH, \
INSTANCE_STARTUP_SCRIPTS_DIR, CONTAINERS_TMP_DIR
from spotty.providers.aws.cfn_templates.instance.start_container_script import StartContainerScriptWithCfnSignals
from spotty.providers.aws.helpers.ami import get_ami
from spotty.providers.aws.helpers.vpc import get_vpc_id
from spotty.providers.aws.resources.snapshot import Snapshot
from spotty.providers.aws.resources.volume import Volume
from spotty.providers.aws.config.instance_config import InstanceConfig
from spotty.providers.aws.config.ebs_volume import EbsVolume
from spotty.providers.aws.helpers.logs import get_logs_s3_path
def prepare_instance_template(ec2, instance_config: InstanceConfig, docker_commands: DockerCommands,
availability_zone: str, sync_project_cmd: str, output: AbstractOutputWriter):
"""Prepares CloudFormation template to run a Spot Instance."""
# read and update CF template
with open(os.path.join(os.path.dirname(__file__), 'data', 'template.yaml')) as f:
template = yaml.load(f, Loader=CfnYamlLoader)
# get volume resources and updated availability zone
volume_resources = _get_volume_resources(ec2, instance_config.volumes, output)
# add volume resources to the template
template['Resources'].update(volume_resources)
# set availability zone
if availability_zone:
template['Resources']['InstanceLaunchTemplate']['Properties']['LaunchTemplateData']['Placement'] = {
'AvailabilityZone': availability_zone,
}
output.write('- availability zone: %s' % availability_zone)
else:
output.write('- availability zone: auto')
# set subnet
if instance_config.subnet_id:
template['Resources']['InstanceLaunchTemplate']['Properties']['LaunchTemplateData']['NetworkInterfaces'] = [
{
'SubnetId': instance_config.subnet_id,
'DeviceIndex': 0,
'Groups': template['Resources']['InstanceLaunchTemplate']['Properties']['LaunchTemplateData'][
'SecurityGroupIds'],
}]
del template['Resources']['InstanceLaunchTemplate']['Properties']['LaunchTemplateData']['SecurityGroupIds']
# add ports to the security group
for port in instance_config.ports:
if port != 22:
template['Resources']['InstanceSecurityGroup']['Properties']['SecurityGroupIngress'] += [{
'CidrIp': '0.0.0.0/0',
'IpProtocol': 'tcp',
'FromPort': port,
'ToPort': port,
}, {
'CidrIpv6': '::/0',
'IpProtocol': 'tcp',
'FromPort': port,
'ToPort': port,
}]
if instance_config.is_spot_instance:
# set maximum price
if instance_config.max_price:
template['Resources']['InstanceLaunchTemplate']['Properties']['LaunchTemplateData'] \
['InstanceMarketOptions']['SpotOptions']['MaxPrice'] = instance_config.max_price
output.write('- maximum Spot Instance price: %s'
% (('%.04f' % instance_config.max_price) if instance_config.max_price else 'on-demand'))
else:
# run on-demand instance
del template['Resources']['InstanceLaunchTemplate']['Properties']['LaunchTemplateData'][
'InstanceMarketOptions']
output.write('- on-demand instance')
# set the user data script
template['Resources']['InstanceLaunchTemplate']['Properties']['LaunchTemplateData']['UserData'] = {
'Fn::Base64': {
'Fn::Sub': _read_template_file(os.path.join('startup_scripts', 'user_data.sh')),
},
}
# run sync command as a non-root user
if instance_config.container_config.run_as_host_user:
sync_project_cmd = 'sudo -u %s %s' % (instance_config.user, sync_project_cmd)
# get mount directories
mount_dirs = [volume.mount_dir for volume in instance_config.volumes if isinstance(volume, EbsVolume)]
# set CloudFormation configs
cfn_init_configs = [
{
'name': 'prepare_instance',
'files': {
INSTANCE_STARTUP_SCRIPTS_DIR + '/01_prepare_instance.sh': {
'owner': 'ubuntu',
'group': 'ubuntu',
'mode': '000755',
'content': {
'Fn::Sub': _read_template_file(os.path.join('startup_scripts', '01_prepare_instance.sh'), {
'CONTAINER_BASH_SCRIPT_PATH': CONTAINER_BASH_SCRIPT_PATH,
'SPOTTY_TMP_DIR': INSTANCE_SPOTTY_TMP_DIR,
'CONTAINERS_TMP_DIR': CONTAINERS_TMP_DIR,
}),
},
},
CONTAINER_BASH_SCRIPT_PATH: {
'owner': 'ubuntu',
'group': 'ubuntu',
'mode': '000755',
'content': ContainerBashScript(docker_commands).render(),
},
'/home/ubuntu/.tmux.conf': {
'owner': 'ubuntu',
'group': 'ubuntu',
'mode': '000644',
'content': {
'Fn::Sub': _read_template_file(os.path.join('files', 'tmux.conf')),
},
},
},
'command': INSTANCE_STARTUP_SCRIPTS_DIR + '/01_prepare_instance.sh',
},
{
'name': 'mount_volumes',
'files': {
INSTANCE_STARTUP_SCRIPTS_DIR + '/02_mount_volumes.sh': {
'owner': 'ubuntu',
'group': 'ubuntu',
'mode': '000755',
'content': {
'Fn::Sub': _read_template_file(os.path.join('startup_scripts', '02_mount_volumes.sh'), {
'MOUNT_DIRS': ('"%s"' % '" "'.join(mount_dirs)) if mount_dirs else '',
'TMP_VOLUME_DIRS': [{'PATH': volume.host_path} for volume in instance_config.volumes
if isinstance(volume, TmpDirVolume)],
}),
},
},
},
'command': INSTANCE_STARTUP_SCRIPTS_DIR + '/02_mount_volumes.sh',
},
{
'name': 'set_docker_root',
'files': {
INSTANCE_STARTUP_SCRIPTS_DIR + '/03_set_docker_root.sh': {
'owner': 'ubuntu',
'group': 'ubuntu',
'mode': '000755',
'content': {
'Fn::Sub': _read_template_file(os.path.join('startup_scripts', '03_set_docker_root.sh')),
},
},
},
'command': INSTANCE_STARTUP_SCRIPTS_DIR + '/03_set_docker_root.sh',
},
{
'name': 'sync_project',
'files': {
INSTANCE_STARTUP_SCRIPTS_DIR + '/04_sync_project.sh': {
'owner': 'ubuntu',
'group': 'ubuntu',
'mode': '000755',
'content': {
'Fn::Sub': _read_template_file(os.path.join('startup_scripts', '04_sync_project.sh'), {
'SYNC_PROJECT_CMD': sync_project_cmd,
}),
},
},
},
'command': INSTANCE_STARTUP_SCRIPTS_DIR + '/04_sync_project.sh',
},
{
'name': 'run_instance_startup_commands',
'files': {
INSTANCE_STARTUP_SCRIPTS_DIR + '/05_run_instance_startup_commands.sh': {
'owner': 'ubuntu',
'group': 'ubuntu',
'mode': '000755',
'content': {
'Fn::Sub': _read_template_file(
os.path.join('startup_scripts', '05_run_instance_startup_commands.sh'), {
'INSTANCE_STARTUP_SCRIPTS_DIR': INSTANCE_STARTUP_SCRIPTS_DIR,
}),
},
},
INSTANCE_STARTUP_SCRIPTS_DIR + '/instance_startup_commands.sh': {
'owner': 'ubuntu',
'group': 'ubuntu',
'mode': '000644',
'content': instance_config.commands or '#',
},
},
'command': INSTANCE_STARTUP_SCRIPTS_DIR + '/05_run_instance_startup_commands.sh',
},
{
'name': 'start_container',
'files': {
INSTANCE_STARTUP_SCRIPTS_DIR + '/06_start_container.sh': {
'owner': 'ubuntu',
'group': 'ubuntu',
'mode': '000755',
'content': {
'Fn::Sub': StartContainerScriptWithCfnSignals(docker_commands).render(print_trace=True),
},
},
},
'command': INSTANCE_STARTUP_SCRIPTS_DIR + '/06_start_container.sh',
},
]
template['Resources']['InstanceLaunchTemplate']['Metadata']['AWS::CloudFormation::Init']['configSets'] = {
'init': [config['name'] for config in cfn_init_configs],
}
for config in cfn_init_configs:
template['Resources']['InstanceLaunchTemplate']['Metadata']['AWS::CloudFormation::Init'][config['name']] = {
'files': config.get('files', {}),
'commands': {
config['name']: {
'command': config['command'],
}
},
}
return yaml.dump(template, Dumper=CfnYamlDumper)
def _read_template_file(filename: str, params: dict = None):
with open(os.path.join(os.path.dirname(__file__), 'data', filename)) as f:
content = f.read()
if params:
content = chevron.render(content, params)
return content
def _get_volume_attachment_resource(volume_id, device_name):
attachment_resource = {
'Type': 'AWS::EC2::VolumeAttachment',
'Properties': {
'Device': device_name,
'InstanceId': {'Ref': 'Instance'},
'VolumeId': volume_id if isinstance(volume_id, str) else dict(volume_id), # avoid YAML aliases
},
'Metadata': {
'Device': device_name,
'VolumeId': volume_id if isinstance(volume_id, str) else dict(volume_id), # avoid YAML aliases
},
}
return attachment_resource
def _get_volume_resource(ec2, volume: EbsVolume, output: AbstractOutputWriter):
# new volume will be created
volume_resource = {
'Type': 'AWS::EC2::Volume',
'DeletionPolicy': 'Retain',
'Properties': {
'AvailabilityZone': {'Fn::GetAtt': ['Instance', 'AvailabilityZone']},
'Tags': [{
'Key': 'Name',
'Value': volume.ec2_volume_name,
}],
'VolumeType': volume.type,
},
}
# check if the snapshot exists and restore the volume from it
snapshot = Snapshot.get_by_name(ec2, volume.ec2_volume_name)
if snapshot:
# volume will be restored from the snapshot
# check size of the volume
if volume.size and (volume.size < snapshot.size):
raise ValueError('Specified size for the "%s" volume (%dGB) is less than size of the '
'snapshot (%dGB).'
% (volume.name, volume.size, snapshot.size))
# set snapshot ID
volume_resource['Properties']['SnapshotId'] = snapshot.snapshot_id
output.write('- volume "%s" will be restored from the snapshot' % volume.ec2_volume_name)
else:
# empty volume will be created, check that the size is specified
if not volume.size:
raise ValueError('Size for the new volume is required.')
output.write('- volume "%s" will be created' % volume.ec2_volume_name)
# set size of the volume
if volume.size:
volume_resource['Properties']['Size'] = volume.size
# set a name for the new volume
volume_resource['Properties']['Tags'] = [{'Key': 'Name', 'Value': volume.ec2_volume_name}]
return volume_resource
def _get_volume_resources(ec2, volumes: List[AbstractInstanceVolume], output: AbstractOutputWriter):
resources = {}
# ending letters for the devices (see: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/device_naming.html)
device_letters = 'fghijklmnop'
# create and attach volumes
for i, volume in enumerate(volumes):
if isinstance(volume, EbsVolume):
device_letter = device_letters[i]
ec2_volume = Volume.get_by_name(ec2, volume.ec2_volume_name)
if ec2_volume:
# check if the volume is available
if not ec2_volume.is_available():
raise ValueError('EBS volume "%s" is not available (state: %s).'
% (volume.ec2_volume_name, ec2_volume.state))
# check size of the volume
if volume.size and (volume.size != ec2_volume.size):
raise ValueError('Specified size for the "%s" volume (%dGB) doesn\'t match the size of the '
'existing volume (%dGB).' % (volume.name, volume.size, ec2_volume.size))
output.write('- volume "%s" (%s) will be attached' % (ec2_volume.name, ec2_volume.volume_id))
volume_id = ec2_volume.volume_id
else:
# create Volume resource
vol_resource_name = 'Volume' + device_letter.upper()
vol_resource = _get_volume_resource(ec2, volume, output)
resources[vol_resource_name] = vol_resource
volume_id = {'Ref': vol_resource_name}
# create VolumeAttachment resource
vol_attachment_resource_name = 'VolumeAttachment' + device_letter.upper()
device_name = '/dev/sd' + device_letter
vol_attachment_resource = _get_volume_attachment_resource(volume_id, device_name)
resources[vol_attachment_resource_name] = vol_attachment_resource
return resources
def get_template_parameters(ec2, instance_config: InstanceConfig, instance_profile_arn: str, bucket_name: str,
key_pair_name: str, output: AbstractOutputWriter):
# get AMI
ami = get_ami(ec2, instance_config.ami_id, instance_config.ami_name)
output.write('- AMI: "%s" (%s)' % (ami.name, ami.image_id))
# check root volume size
root_volume_size = instance_config.root_volume_size
if root_volume_size and root_volume_size < ami.size:
raise ValueError('Root volume size cannot be less than the size of AMI (%dGB).' % ami.size)
elif not root_volume_size:
# if a root volume size is not specified, make it 5GB larger than the AMI size
root_volume_size = ami.size + 5
# print info about the Docker data root
ebs_volumes = [volume for volume in instance_config.volumes if isinstance(volume, EbsVolume)]
if instance_config.docker_data_root:
docker_data_volume_name = [volume.name for volume in ebs_volumes
if is_subdir(instance_config.docker_data_root, volume.mount_dir)][0]
output.write('- Docker data will be stored on the "%s" volume' % docker_data_volume_name)
# create stack
parameters = {
'VpcId': get_vpc_id(ec2, instance_config.subnet_id),
'InstanceProfileArn': instance_profile_arn,
'InstanceType': instance_config.instance_type,
'KeyName': key_pair_name,
'ImageId': ami.image_id,
'RootVolumeSize': str(root_volume_size),
'DockerDataRootDirectory': instance_config.docker_data_root,
'InstanceNameTag': instance_config.ec2_instance_name,
'HostProjectDirectory': instance_config.host_project_dir,
'LogsS3Path': get_logs_s3_path(bucket_name, instance_config.name),
}
return parameters
================================================
FILE: spotty/providers/aws/cfn_templates/instance_profile/__init__.py
================================================
================================================
FILE: spotty/providers/aws/cfn_templates/instance_profile/data/template.yaml
================================================
Description: Spotty EC2 Instance Profile
Resources:
InstanceProfile:
Type: AWS::IAM::InstanceProfile
Properties:
Roles:
- Ref: InstanceRole
InstanceRole:
Type: AWS::IAM::Role
Properties:
AssumeRolePolicyDocument:
Version: '2012-10-17'
Statement:
- Effect: Allow
Principal:
Service:
- ec2.amazonaws.com
Action:
- sts:AssumeRole
{{#HAS_MANAGED_POLICIES}}
ManagedPolicyArns:
{{#MANAGED_POLICY_ARNS}}
- {{MANAGED_POLICY_ARN}}
{{/MANAGED_POLICY_ARNS}}
{{/HAS_MANAGED_POLICIES}}
Policies:
- PolicyName: S3AccessPolicy
PolicyDocument:
Version: '2012-10-17'
Statement:
- Effect: Allow
Action:
- s3:ListAllMyBuckets
- s3:GetBucketLocation
- s3:ListBucket
- s3:GetObject
- s3:PutObject
- s3:DeleteObject
Resource:
- arn:aws:s3:::*
Outputs:
ProfileArn:
Value: !GetAtt InstanceProfile.Arn
================================================
FILE: spotty/providers/aws/cfn_templates/instance_profile/template.py
================================================
import os
import chevron
def prepare_instance_profile_template(managed_policy_arns: list):
with open(os.path.join(os.path.dirname(__file__), 'data', 'template.yaml')) as f:
content = f.read()
parameters = {
'HAS_MANAGED_POLICIES': len(managed_policy_arns),
'MANAGED_POLICY_ARNS': [{'MANAGED_POLICY_ARN': arn} for arn in managed_policy_arns]
}
template = chevron.render(content, parameters)
return template
================================================
FILE: spotty/providers/aws/commands/__init__.py
================================================
================================================
FILE: spotty/providers/aws/commands/clean_logs.py
================================================
from argparse import ArgumentParser, Namespace
from time import time
import boto3
from spotty.commands.abstract_command import AbstractCommand
from spotty.commands.writers.abstract_output_writrer import AbstractOutputWriter
class CleanLogsCommand(AbstractCommand):
name = 'clean-logs'
description = 'Delete expired CloudFormation log groups with Spotty prefixes'
def configure(self, parser: ArgumentParser):
super().configure(parser)
parser.add_argument('-r', '--region', type=str, required=True, help='AWS region')
parser.add_argument('-a', '--delete-all', action='store_true', help='Delete all Spotty log groups, '
'not just expired ones')
def run(self, args: Namespace, output: AbstractOutputWriter):
region = args.region
logs = boto3.client('logs', region_name=region)
prefixes = ['spotty-', '/aws/lambda/spotty-']
only_empty = not args.delete_all
output.write('Deleting %s Spotty log groups...' % ('empty' if only_empty else 'all'))
res = logs.describe_log_groups()
self._delete_log_groups(logs, res['logGroups'], prefixes, only_empty, output)
while 'nextToken' in res:
res = logs.describe_log_groups(nextToken=res['nextToken'])
self._delete_log_groups(logs, res['logGroups'], prefixes, only_empty, output)
output.write('Done')
@staticmethod
def _delete_log_groups(logs, log_groups: list, prefixes: list, only_empty: bool, output: AbstractOutputWriter):
for log_group in log_groups:
for prefix in prefixes:
if log_group['logGroupName'].s
gitextract_scgum9gk/
├── .github/
│ ├── FUNDING.yml
│ └── workflows/
│ ├── generate-docs.yml
│ └── python-publish.yml
├── .gitignore
├── CONTRIBUTING.md
├── LICENSE
├── README.md
├── bin/
│ └── spotty
├── docs/
│ ├── Makefile
│ ├── make.bat
│ ├── requirements.txt
│ └── source/
│ ├── _static/
│ │ ├── favicon/
│ │ │ ├── browserconfig.xml
│ │ │ └── site.webmanifest
│ │ ├── scripts.js
│ │ └── styles.css
│ ├── conf.py
│ ├── docs/
│ │ ├── cli/
│ │ │ ├── spotty-aws.rst
│ │ │ ├── spotty-download.rst
│ │ │ ├── spotty-exec.rst
│ │ │ ├── spotty-run.rst
│ │ │ ├── spotty-sh.rst
│ │ │ ├── spotty-start.rst
│ │ │ ├── spotty-stop.rst
│ │ │ ├── spotty-sync.rst
│ │ │ └── spotty.rst
│ │ ├── providers/
│ │ │ ├── aws/
│ │ │ │ ├── caching-docker-image-on-an-ebs-volume.md
│ │ │ │ ├── ebs-volumes-and-deletion-policies.md
│ │ │ │ ├── faq.md
│ │ │ │ ├── instance-parameters.md
│ │ │ │ └── overview.rst
│ │ │ ├── gcp/
│ │ │ │ ├── account-preparation.md
│ │ │ │ ├── caching-docker-image-on-a-disk.md
│ │ │ │ ├── disks-and-deletion-policies.md
│ │ │ │ ├── instance-parameters.md
│ │ │ │ └── overview.rst
│ │ │ ├── local/
│ │ │ │ ├── instance-parameters.md
│ │ │ │ └── overview.rst
│ │ │ └── remote/
│ │ │ ├── instance-parameters.md
│ │ │ └── overview.rst
│ │ └── user-guide/
│ │ ├── configuration-file.md
│ │ ├── getting-started.md
│ │ └── installation.md
│ ├── index.rst
│ └── main.html
├── setup.cfg
├── setup.py
├── spotty/
│ ├── __init__.py
│ ├── cli.py
│ ├── commands/
│ │ ├── __init__.py
│ │ ├── abstract_command.py
│ │ ├── abstract_config_command.py
│ │ ├── abstract_provider_command.py
│ │ ├── aws.py
│ │ ├── download.py
│ │ ├── exec.py
│ │ ├── run.py
│ │ ├── sh.py
│ │ ├── start.py
│ │ ├── status.py
│ │ ├── stop.py
│ │ ├── sync.py
│ │ └── writers/
│ │ ├── __init__.py
│ │ ├── abstract_output_writrer.py
│ │ ├── null_output_writrer.py
│ │ └── output_writrer.py
│ ├── config/
│ │ ├── __init__.py
│ │ ├── abstract_instance_config.py
│ │ ├── abstract_instance_volume.py
│ │ ├── config_utils.py
│ │ ├── container_config.py
│ │ ├── host_path_volume.py
│ │ ├── project_config.py
│ │ ├── tmp_dir_volume.py
│ │ └── validation.py
│ ├── configuration.py
│ ├── deployment/
│ │ ├── __init__.py
│ │ ├── abstract_cloud_instance/
│ │ │ ├── __init__.py
│ │ │ ├── abstract_bucket_manager.py
│ │ │ ├── abstract_cloud_instance_manager.py
│ │ │ ├── abstract_data_transfer.py
│ │ │ ├── abstract_instance_deployment.py
│ │ │ ├── errors/
│ │ │ │ ├── __init__.py
│ │ │ │ └── bucket_not_found.py
│ │ │ ├── file_structure.py
│ │ │ └── resources/
│ │ │ ├── __init__.py
│ │ │ ├── abstract_bucket.py
│ │ │ └── abstract_instance.py
│ │ ├── abstract_docker_instance_manager.py
│ │ ├── abstract_instance_manager.py
│ │ ├── abstract_ssh_instance_manager.py
│ │ ├── container/
│ │ │ ├── __init__.py
│ │ │ ├── abstract_container_commands.py
│ │ │ ├── abstract_container_script.py
│ │ │ └── docker/
│ │ │ ├── __init__.py
│ │ │ ├── docker_commands.py
│ │ │ └── scripts/
│ │ │ ├── __init__.py
│ │ │ ├── abstract_docker_script.py
│ │ │ ├── container_bash_script.py
│ │ │ ├── data/
│ │ │ │ ├── container_bash.sh.tpl
│ │ │ │ ├── start_container.sh.tpl
│ │ │ │ └── stop_container.sh.tpl
│ │ │ ├── start_container_script.py
│ │ │ └── stop_container_script.py
│ │ └── utils/
│ │ ├── __init__.py
│ │ ├── cli.py
│ │ ├── commands.py
│ │ ├── print_info.py
│ │ └── user_scripts.py
│ ├── errors/
│ │ ├── __init__.py
│ │ ├── instance_not_running.py
│ │ └── nothing_to_do.py
│ ├── providers/
│ │ ├── __init__.py
│ │ ├── aws/
│ │ │ ├── __init__.py
│ │ │ ├── cfn_templates/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── instance/
│ │ │ │ │ ├── __init__.py
│ │ │ │ │ ├── data/
│ │ │ │ │ │ ├── files/
│ │ │ │ │ │ │ └── tmux.conf
│ │ │ │ │ │ ├── startup_scripts/
│ │ │ │ │ │ │ ├── 01_prepare_instance.sh
│ │ │ │ │ │ │ ├── 02_mount_volumes.sh
│ │ │ │ │ │ │ ├── 03_set_docker_root.sh
│ │ │ │ │ │ │ ├── 04_sync_project.sh
│ │ │ │ │ │ │ ├── 05_run_instance_startup_commands.sh
│ │ │ │ │ │ │ └── user_data.sh
│ │ │ │ │ │ └── template.yaml
│ │ │ │ │ ├── start_container_script.py
│ │ │ │ │ └── template.py
│ │ │ │ └── instance_profile/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── data/
│ │ │ │ │ └── template.yaml
│ │ │ │ └── template.py
│ │ │ ├── commands/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── clean_logs.py
│ │ │ │ └── spot_prices.py
│ │ │ ├── config/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── ebs_volume.py
│ │ │ │ ├── instance_config.py
│ │ │ │ └── validation.py
│ │ │ ├── data_transfer.py
│ │ │ ├── deletion_policies.py
│ │ │ ├── errors/
│ │ │ │ ├── __init__.py
│ │ │ │ └── volume_not_found.py
│ │ │ ├── helpers/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── ami.py
│ │ │ │ ├── availability_zone.py
│ │ │ │ ├── instance_prices.py
│ │ │ │ ├── logs.py
│ │ │ │ ├── s3_sync.py
│ │ │ │ ├── subnet.py
│ │ │ │ └── vpc.py
│ │ │ ├── instance_deployment.py
│ │ │ ├── instance_manager.py
│ │ │ ├── resource_managers/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── bucket_manager.py
│ │ │ │ ├── instance_profile_stack_manager.py
│ │ │ │ ├── instance_stack_manager.py
│ │ │ │ └── key_pair_manager.py
│ │ │ └── resources/
│ │ │ ├── __init__.py
│ │ │ ├── bucket.py
│ │ │ ├── image.py
│ │ │ ├── instance.py
│ │ │ ├── snapshot.py
│ │ │ ├── stack.py
│ │ │ ├── subnet.py
│ │ │ ├── volume.py
│ │ │ └── vpc.py
│ │ ├── gcp/
│ │ │ ├── __init__.py
│ │ │ ├── config/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── disk_volume.py
│ │ │ │ ├── image_uri.py
│ │ │ │ ├── instance_config.py
│ │ │ │ └── validation.py
│ │ │ ├── data_transfer.py
│ │ │ ├── dm_templates/
│ │ │ │ ├── __init__.py
│ │ │ │ └── instance/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── data/
│ │ │ │ │ ├── startup_script.sh.tpl
│ │ │ │ │ ├── startup_scripts/
│ │ │ │ │ │ ├── 01_prepare_instance.sh
│ │ │ │ │ │ ├── 02_mount_volumes.sh
│ │ │ │ │ │ ├── 03_set_docker_root.sh
│ │ │ │ │ │ ├── 04_sync_project.sh
│ │ │ │ │ │ └── 05_run_instance_startup_commands.sh
│ │ │ │ │ └── template.yaml
│ │ │ │ └── instance_template.py
│ │ │ ├── errors/
│ │ │ │ ├── __init__.py
│ │ │ │ └── image_not_found.py
│ │ │ ├── helpers/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── ce_client.py
│ │ │ │ ├── deployment.py
│ │ │ │ ├── dm_client.py
│ │ │ │ ├── dm_resource.py
│ │ │ │ ├── gcp_credentials.py
│ │ │ │ ├── gs_client.py
│ │ │ │ ├── gsutil_rsync.py
│ │ │ │ ├── image.py
│ │ │ │ ├── rtc_client.py
│ │ │ │ └── volumes.py
│ │ │ ├── instance_deployment.py
│ │ │ ├── instance_manager.py
│ │ │ ├── resource_managers/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── bucket_manager.py
│ │ │ │ ├── instance_stack_manager.py
│ │ │ │ └── ssh_key_manager.py
│ │ │ └── resources/
│ │ │ ├── __init__.py
│ │ │ ├── bucket.py
│ │ │ ├── disk.py
│ │ │ ├── image.py
│ │ │ ├── instance.py
│ │ │ ├── snapshot.py
│ │ │ └── stack.py
│ │ ├── instance_manager_factory.py
│ │ ├── local/
│ │ │ ├── __init__.py
│ │ │ ├── config/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── instance_config.py
│ │ │ │ └── validation.py
│ │ │ └── instance_manager.py
│ │ └── remote/
│ │ ├── __init__.py
│ │ ├── config/
│ │ │ ├── __init__.py
│ │ │ ├── instance_config.py
│ │ │ └── validation.py
│ │ ├── helpers/
│ │ │ └── rsync.py
│ │ └── instance_manager.py
│ └── utils.py
└── tests/
├── __init__.py
├── container_config.py
├── helpers/
│ ├── __init__.py
│ ├── cli.py
│ └── spotty_cli.py
└── providers/
├── __init__.py
├── aws/
│ ├── __init__.py
│ ├── commands/
│ │ ├── data/
│ │ │ └── test-project/
│ │ │ ├── ignored-dir/
│ │ │ │ ├── ignored-file
│ │ │ │ └── included-file
│ │ │ ├── ignored-file
│ │ │ ├── local-file
│ │ │ └── spotty.yaml
│ │ ├── download.py
│ │ └── sync.py
│ ├── config/
│ │ ├── __init__.py
│ │ ├── container_deployment.py
│ │ ├── data/
│ │ │ ├── config-wo-mounts.yaml
│ │ │ └── config1.yaml
│ │ └── instance_config_validation.py
│ └── project_resources/
│ ├── __init__.py
│ ├── bucket.py
│ └── key_pair.py
├── gcp/
│ └── config/
│ ├── __init__.py
│ └── image_uri.py
└── local/
├── __init__.py
├── commands/
│ ├── __init__.py
│ ├── data/
│ │ └── test-project/
│ │ └── spotty.yaml
│ └── run.py
└── config/
├── __init__.py
├── container_deployment.py
└── data/
└── config1.yaml
SYMBOL INDEX (693 symbols across 130 files)
FILE: setup.py
function get_version (line 8) | def get_version():
function get_description (line 20) | def get_description():
FILE: spotty/cli.py
function get_parser (line 16) | def get_parser() -> argparse.ArgumentParser:
function add_subparsers (line 38) | def add_subparsers(parser: argparse.ArgumentParser, command_classes: Lis...
function _get_custom_commands (line 48) | def _get_custom_commands() -> List[Type[AbstractCommand]]:
FILE: spotty/commands/abstract_command.py
class AbstractCommand (line 6) | class AbstractCommand(ABC):
method name (line 11) | def name(self) -> str:
method description (line 16) | def description(self) -> str:
method configure (line 20) | def configure(self, parser: ArgumentParser):
method run (line 25) | def run(self, args: Namespace, output: AbstractOutputWriter):
FILE: spotty/commands/abstract_config_command.py
class AbstractConfigCommand (line 11) | class AbstractConfigCommand(AbstractCommand):
method _run (line 15) | def _run(self, instance_manager: AbstractInstanceManager, args: Namesp...
method configure (line 18) | def configure(self, parser: ArgumentParser):
method run (line 23) | def run(self, args: Namespace, output: AbstractOutputWriter):
method _get_instance_id (line 38) | def _get_instance_id(instances: List[dict], instance_name: str, output...
FILE: spotty/commands/abstract_provider_command.py
class AbstractProviderCommand (line 8) | class AbstractProviderCommand(AbstractCommand):
method commands (line 13) | def commands(self) -> list:
method configure (line 17) | def configure(self, parser: ArgumentParser):
method run (line 21) | def run(self, args: Namespace, output: AbstractOutputWriter):
FILE: spotty/commands/aws.py
class AwsCommand (line 6) | class AwsCommand(AbstractProviderCommand):
FILE: spotty/commands/download.py
class DownloadCommand (line 9) | class DownloadCommand(AbstractConfigCommand):
method configure (line 14) | def configure(self, parser: ArgumentParser):
method _run (line 22) | def _run(self, instance_manager: AbstractInstanceManager, args: Namesp...
FILE: spotty/commands/exec.py
class ExecCommand (line 11) | class ExecCommand(AbstractConfigCommand):
method configure (line 16) | def configure(self, parser: ArgumentParser):
method _run (line 30) | def _run(self, instance_manager: AbstractInstanceManager, args: Namesp...
FILE: spotty/commands/run.py
class RunCommand (line 11) | class RunCommand(AbstractConfigCommand):
method configure (line 16) | def configure(self, parser: ArgumentParser):
method _run (line 35) | def _run(self, instance_manager: AbstractInstanceManager, args: Namesp...
FILE: spotty/commands/sh.py
class ShCommand (line 9) | class ShCommand(AbstractConfigCommand):
method configure (line 14) | def configure(self, parser: ArgumentParser):
method _run (line 24) | def _run(self, instance_manager: AbstractInstanceManager, args: Namesp...
FILE: spotty/commands/start.py
class StartCommand (line 8) | class StartCommand(AbstractConfigCommand):
method configure (line 13) | def configure(self, parser: ArgumentParser):
method _run (line 21) | def _run(self, instance_manager: AbstractInstanceManager, args: Namesp...
FILE: spotty/commands/status.py
class StatusCommand (line 7) | class StatusCommand(AbstractConfigCommand):
method _run (line 12) | def _run(self, instance_manager: AbstractInstanceManager, args: Namesp...
FILE: spotty/commands/stop.py
class StopCommand (line 7) | class StopCommand(AbstractConfigCommand):
method _run (line 19) | def _run(self, instance_manager: AbstractInstanceManager, args: Namesp...
FILE: spotty/commands/sync.py
class SyncCommand (line 9) | class SyncCommand(AbstractConfigCommand):
method configure (line 14) | def configure(self, parser: ArgumentParser):
method _run (line 18) | def _run(self, instance_manager: AbstractInstanceManager, args: Namesp...
FILE: spotty/commands/writers/abstract_output_writrer.py
class AbstractOutputWriter (line 5) | class AbstractOutputWriter(ABC):
method __init__ (line 7) | def __init__(self):
method _write (line 12) | def _write(self, msg: str, newline: bool = True):
method write (line 15) | def write(self, msg: str = '', newline: bool = True):
method prefix (line 23) | def prefix(self, prefix):
FILE: spotty/commands/writers/null_output_writrer.py
class NullOutputWriter (line 4) | class NullOutputWriter(AbstractOutputWriter):
method _write (line 6) | def _write(self, msg: str, newline: bool = True):
FILE: spotty/commands/writers/output_writrer.py
class OutputWriter (line 4) | class OutputWriter(AbstractOutputWriter):
method _write (line 6) | def _write(self, msg: str, newline: bool = True):
FILE: spotty/config/abstract_instance_config.py
class AbstractInstanceConfig (line 17) | class AbstractInstanceConfig(ABC):
method __init__ (line 19) | def __init__(self, instance_config: dict, project_config: ProjectConfig):
method _validate_instance_params (line 44) | def _validate_instance_params(self, params: dict) -> dict:
method _get_instance_volumes (line 49) | def _get_instance_volumes(self) -> List[AbstractInstanceVolume]:
method project_config (line 54) | def project_config(self) -> ProjectConfig:
method container_config (line 58) | def container_config(self) -> ContainerConfig:
method user (line 63) | def user(self) -> str:
method name (line 67) | def name(self) -> str:
method provider_name (line 72) | def provider_name(self):
method container_name (line 77) | def container_name(self) -> str:
method full_container_name (line 81) | def full_container_name(self) -> str:
method docker_data_root (line 86) | def docker_data_root(self) -> str:
method local_ssh_port (line 91) | def local_ssh_port(self) -> int:
method commands (line 96) | def commands(self) -> str:
method host_project_dir (line 101) | def host_project_dir(self):
method volumes (line 106) | def volumes(self) -> List[AbstractInstanceVolume]:
method volume_mounts (line 110) | def volume_mounts(self) -> List[VolumeMount]:
method dockerfile_path (line 114) | def dockerfile_path(self):
method docker_context_path (line 123) | def docker_context_path(self):
method host_container_dir (line 132) | def host_container_dir(self):
method host_logs_dir (line 137) | def host_logs_dir(self):
method host_volumes_dir (line 142) | def host_volumes_dir(self):
method _get_volumes (line 149) | def _get_volumes(self) -> List[AbstractInstanceVolume]:
method _get_volume_mounts (line 165) | def _get_volume_mounts(self, volumes: List[AbstractInstanceVolume]) \
method _get_host_project_dir (line 184) | def _get_host_project_dir(self, volume_mounts: List[VolumeMount]) -> str:
FILE: spotty/config/abstract_instance_volume.py
class AbstractInstanceVolume (line 4) | class AbstractInstanceVolume(ABC):
method __init__ (line 6) | def __init__(self, volume_config: dict):
method _validate_volume_parameters (line 11) | def _validate_volume_parameters(self, params: dict) -> dict:
method name (line 15) | def name(self) -> str:
method host_path (line 21) | def host_path(self) -> str:
method title (line 27) | def title(self) -> str:
method deletion_policy_title (line 34) | def deletion_policy_title(self) -> str:
FILE: spotty/config/config_utils.py
function load_config (line 12) | def load_config(config_path: str = None) -> ProjectConfig:
function _read_yaml (line 44) | def _read_yaml(file_path: str):
function _merge_configs (line 52) | def _merge_configs(orig_config, override_config):
function _update_dict (line 113) | def _update_dict(d, u):
FILE: spotty/config/container_config.py
class ContainerConfig (line 8) | class ContainerConfig(object):
method __init__ (line 10) | def __init__(self, container_config: dict):
method name (line 15) | def name(self) -> str:
method project_dir (line 19) | def project_dir(self) -> str:
method image (line 23) | def image(self) -> str:
method file (line 27) | def file(self) -> str:
method run_as_host_user (line 31) | def run_as_host_user(self) -> str:
method volume_mounts (line 35) | def volume_mounts(self) -> list:
method commands (line 39) | def commands(self) -> str:
method working_dir (line 43) | def working_dir(self) -> str:
method env (line 52) | def env(self) -> dict:
method host_network (line 56) | def host_network(self) -> bool:
method ports (line 60) | def ports(self) -> List[dict]:
method runtime_parameters (line 64) | def runtime_parameters(self) -> list:
method _get_volume_mounts (line 67) | def _get_volume_mounts(self):
FILE: spotty/config/host_path_volume.py
class HostPathVolume (line 6) | class HostPathVolume(AbstractInstanceVolume):
method __init__ (line 10) | def __init__(self, volume_config: dict, base_dir: str = None):
method _validate_volume_parameters (line 15) | def _validate_volume_parameters(self, params: dict) -> dict:
method title (line 19) | def title(self):
method name (line 23) | def name(self):
method deletion_policy_title (line 27) | def deletion_policy_title(self) -> str:
method host_path (line 31) | def host_path(self) -> str:
FILE: spotty/config/project_config.py
class ProjectConfig (line 4) | class ProjectConfig(object):
method __init__ (line 6) | def __init__(self, config: dict, project_dir: str):
method project_dir (line 14) | def project_dir(self) -> str:
method project_name (line 18) | def project_name(self) -> str:
method sync_filters (line 22) | def sync_filters(self) -> list:
method containers (line 26) | def containers(self) -> list:
method instances (line 30) | def instances(self) -> list:
method scripts (line 34) | def scripts(self) -> dict:
FILE: spotty/config/tmp_dir_volume.py
class TmpDirVolume (line 4) | class TmpDirVolume(HostPathVolume):
method title (line 7) | def title(self):
method deletion_policy_title (line 11) | def deletion_policy_title(self) -> str:
FILE: spotty/config/validation.py
function validate_basic_config (line 9) | def validate_basic_config(data):
function validate_host_path_volume_parameters (line 112) | def validate_host_path_volume_parameters(params: dict):
function get_instance_parameters_schema (line 120) | def get_instance_parameters_schema(instance_parameters: dict, default_vo...
function is_unique_value (line 160) | def is_unique_value(x: List[dict], key):
function has_prefix (line 165) | def has_prefix(x: list):
function is_subdir (line 174) | def is_subdir(subdir_path, dir_path):
function validate_config (line 179) | def validate_config(schema: Schema, config):
class WrongKey (line 188) | class WrongKey(Hook):
method __init__ (line 189) | def __init__(self, *args, **kwargs):
method raise_error (line 192) | def raise_error(self, key, *args):
FILE: spotty/configuration.py
function get_spotty_config_dir (line 4) | def get_spotty_config_dir():
function get_spotty_keys_dir (line 13) | def get_spotty_keys_dir(provider_name: str):
FILE: spotty/deployment/abstract_cloud_instance/abstract_bucket_manager.py
class AbstractBucketManager (line 5) | class AbstractBucketManager(ABC):
method __init__ (line 7) | def __init__(self, project_name: str):
method project_name (line 11) | def project_name(self) -> str:
method get_bucket (line 14) | def get_bucket(self) -> AbstractBucket:
method create_bucket (line 17) | def create_bucket(self) -> AbstractBucket:
FILE: spotty/deployment/abstract_cloud_instance/abstract_cloud_instance_manager.py
class AbstractCloudInstanceManager (line 13) | class AbstractCloudInstanceManager(AbstractSshInstanceManager, ABC):
method __init__ (line 15) | def __init__(self, project_config: ProjectConfig, instance_config: dict):
method _get_bucket_manager (line 23) | def _get_bucket_manager(self) -> AbstractBucketManager:
method _get_data_transfer (line 28) | def _get_data_transfer(self) -> AbstractDataTransfer:
method _get_instance_deployment (line 33) | def _get_instance_deployment(self) -> AbstractInstanceDeployment:
method bucket_manager (line 38) | def bucket_manager(self) -> AbstractBucketManager:
method data_transfer (line 43) | def data_transfer(self) -> AbstractDataTransfer:
method instance_deployment (line 48) | def instance_deployment(self) -> AbstractInstanceDeployment:
method is_running (line 52) | def is_running(self) -> bool:
method start (line 57) | def start(self, output: AbstractOutputWriter, dry_run=False):
method stop (line 98) | def stop(self, only_shutdown: bool, output: AbstractOutputWriter):
method clean (line 107) | def clean(self, output: AbstractOutputWriter):
method sync (line 110) | def sync(self, output: AbstractOutputWriter, dry_run=False):
method download (line 132) | def download(self, download_filters: list, output: AbstractOutputWrite...
method ssh_host (line 157) | def ssh_host(self):
method ssh_port (line 174) | def ssh_port(self) -> int:
method use_tmux (line 181) | def use_tmux(self) -> bool:
FILE: spotty/deployment/abstract_cloud_instance/abstract_data_transfer.py
class AbstractDataTransfer (line 4) | class AbstractDataTransfer(ABC):
method __init__ (line 6) | def __init__(self, local_project_dir: str, host_project_dir: str, sync...
method instance_name (line 13) | def instance_name(self):
method scheme_name (line 18) | def scheme_name(self) -> str:
method _get_bucket_project_path (line 21) | def _get_bucket_project_path(self, bucket_name: str) -> str:
method _get_bucket_downloads_path (line 25) | def _get_bucket_downloads_path(self, bucket_name: str) -> str:
method upload_local_to_bucket (line 30) | def upload_local_to_bucket(self, bucket_name: str, dry_run: bool = Fal...
method download_bucket_to_local (line 35) | def download_bucket_to_local(self, bucket_name: str, download_filters:...
method get_download_bucket_to_instance_command (line 40) | def get_download_bucket_to_instance_command(self, bucket_name: str, us...
method get_upload_instance_to_bucket_command (line 45) | def get_upload_instance_to_bucket_command(self, bucket_name: str, down...
FILE: spotty/deployment/abstract_cloud_instance/abstract_instance_deployment.py
class AbstractInstanceDeployment (line 9) | class AbstractInstanceDeployment(ABC):
method __init__ (line 11) | def __init__(self, instance_config: AbstractInstanceConfig):
method instance_config (line 15) | def instance_config(self) -> AbstractInstanceConfig:
method get_instance (line 19) | def get_instance(self) -> AbstractInstance:
method deploy (line 24) | def deploy(self, container_commands: AbstractContainerCommands, bucket...
method delete (line 30) | def delete(self, output: AbstractOutputWriter):
FILE: spotty/deployment/abstract_cloud_instance/errors/bucket_not_found.py
class BucketNotFoundError (line 1) | class BucketNotFoundError(Exception):
method __init__ (line 2) | def __init__(self):
FILE: spotty/deployment/abstract_cloud_instance/resources/abstract_bucket.py
class AbstractBucket (line 4) | class AbstractBucket(ABC):
method name (line 7) | def name(self):
FILE: spotty/deployment/abstract_cloud_instance/resources/abstract_instance.py
class AbstractInstance (line 4) | class AbstractInstance(ABC):
method public_ip_address (line 7) | def public_ip_address(self):
method private_ip_address (line 11) | def private_ip_address(self):
method is_running (line 15) | def is_running(self):
method is_stopped (line 20) | def is_stopped(self):
method terminate (line 24) | def terminate(self, wait: bool = True):
method stop (line 27) | def stop(self, wait: bool = True):
FILE: spotty/deployment/abstract_docker_instance_manager.py
class AbstractDockerInstanceManager (line 13) | class AbstractDockerInstanceManager(AbstractInstanceManager, ABC):
method container_commands (line 16) | def container_commands(self) -> DockerCommands:
method is_container_running (line 20) | def is_container_running(self) -> bool:
method start_container (line 27) | def start_container(self, output: AbstractOutputWriter, dry_run=False):
method start (line 47) | def start(self, output: AbstractOutputWriter, dry_run=False):
method stop (line 51) | def stop(self, only_shutdown: bool, output: AbstractOutputWriter):
method get_status_text (line 60) | def get_status_text(self):
method _check_dockerfile_exists (line 68) | def _check_dockerfile_exists(self):
FILE: spotty/deployment/abstract_instance_manager.py
class AbstractInstanceManager (line 9) | class AbstractInstanceManager(ABC):
method __init__ (line 11) | def __init__(self, project_config: ProjectConfig, instance_config: dict):
method project_config (line 16) | def project_config(self) -> ProjectConfig:
method instance_config (line 20) | def instance_config(self) -> AbstractInstanceConfig:
method _get_instance_config (line 24) | def _get_instance_config(self, instance_config: dict) -> AbstractInsta...
method container_commands (line 30) | def container_commands(self) -> AbstractContainerCommands:
method is_running (line 35) | def is_running(self) -> bool:
method start (line 40) | def start(self, output: AbstractOutputWriter, dry_run=False):
method start_container (line 45) | def start_container(self, output: AbstractOutputWriter, dry_run=False):
method stop (line 50) | def stop(self, only_shutdown: bool, output: AbstractOutputWriter):
method exec (line 54) | def exec(self, command: str, tty: bool = True) -> int:
method clean (line 59) | def clean(self, output: AbstractOutputWriter):
method sync (line 64) | def sync(self, output: AbstractOutputWriter, dry_run=False):
method download (line 69) | def download(self, download_filters: list, output: AbstractOutputWrite...
method get_status_text (line 74) | def get_status_text(self) -> str:
method use_tmux (line 82) | def use_tmux(self) -> bool:
FILE: spotty/deployment/abstract_ssh_instance_manager.py
class AbstractSshInstanceManager (line 8) | class AbstractSshInstanceManager(AbstractDockerInstanceManager):
method exec (line 10) | def exec(self, command: str, tty: bool = True) -> int:
method ssh_host (line 23) | def ssh_host(self):
method ssh_port (line 28) | def ssh_port(self) -> int:
method ssh_key_path (line 33) | def ssh_key_path(self) -> str:
method ssh_user (line 37) | def ssh_user(self) -> str:
method ssh_env_vars (line 41) | def ssh_env_vars(self) -> dict:
method use_tmux (line 49) | def use_tmux(self) -> bool:
FILE: spotty/deployment/container/abstract_container_commands.py
class AbstractContainerCommands (line 5) | class AbstractContainerCommands(ABC):
method __init__ (line 7) | def __init__(self, instance_config: AbstractInstanceConfig):
method instance_config (line 11) | def instance_config(self) -> AbstractInstanceConfig:
method exec (line 15) | def exec(self, command: str, interactive: bool = False, tty: bool = Fa...
FILE: spotty/deployment/container/abstract_container_script.py
class AbstractContainerScript (line 5) | class AbstractContainerScript(ABC):
method __init__ (line 7) | def __init__(self, container_commands: AbstractContainerCommands):
method commands (line 11) | def commands(self) -> AbstractContainerCommands:
method render (line 15) | def render(self) -> str:
FILE: spotty/deployment/container/docker/docker_commands.py
class DockerCommands (line 6) | class DockerCommands(AbstractContainerCommands):
method build (line 8) | def build(self, image_name: str) -> str:
method pull (line 24) | def pull(self) -> str:
method run (line 27) | def run(self, image_name: str = None) -> str:
method is_created (line 59) | def is_created(self, container_name: str = None, is_running: bool = Fa...
method remove (line 67) | def remove(self):
method exec (line 70) | def exec(self, command: str, interactive: bool = False, tty: bool = Fa...
FILE: spotty/deployment/container/docker/scripts/abstract_docker_script.py
class AbstractDockerScript (line 6) | class AbstractDockerScript(AbstractContainerScript, ABC):
method commands (line 9) | def commands(self) -> DockerCommands:
FILE: spotty/deployment/container/docker/scripts/container_bash_script.py
class ContainerBashScript (line 7) | class ContainerBashScript(AbstractDockerScript):
method render (line 9) | def render(self) -> str:
FILE: spotty/deployment/container/docker/scripts/start_container_script.py
class StartContainerScript (line 8) | class StartContainerScript(AbstractDockerScript):
method _partials (line 10) | def _partials(self) -> dict:
method render (line 17) | def render(self, print_trace: bool = False) -> str:
FILE: spotty/deployment/container/docker/scripts/stop_container_script.py
class StopContainerScript (line 6) | class StopContainerScript(AbstractDockerScript):
method render (line 8) | def render(self) -> str:
FILE: spotty/deployment/utils/cli.py
function shlex_join (line 4) | def shlex_join(split_command: list):
FILE: spotty/deployment/utils/commands.py
function get_bash_command (line 8) | def get_bash_command() -> str:
function get_script_command (line 12) | def get_script_command(script_name: str, script_content: str, script_arg...
function get_log_command (line 42) | def get_log_command(command: str, log_file_path: str) -> str:
function get_tmux_session_command (line 56) | def get_tmux_session_command(command: str, session_name: str, window_nam...
function get_ssh_command (line 82) | def get_ssh_command(host: str, port: int, user: str, key_path: str, comm...
FILE: spotty/deployment/utils/print_info.py
function render_volumes_info_table (line 8) | def render_volumes_info_table(volume_mounts: List[VolumeMount], volumes:...
FILE: spotty/deployment/utils/user_scripts.py
function parse_script_parameters (line 6) | def parse_script_parameters(script_params: str):
function render_script (line 24) | def render_script(template: str, params: dict):
FILE: spotty/errors/instance_not_running.py
class InstanceNotRunningError (line 1) | class InstanceNotRunningError(Exception):
method __init__ (line 2) | def __init__(self, instance_name: str):
FILE: spotty/errors/nothing_to_do.py
class NothingToDoError (line 1) | class NothingToDoError(Exception):
FILE: spotty/providers/aws/cfn_templates/instance/start_container_script.py
class StartContainerScriptWithCfnSignals (line 4) | class StartContainerScriptWithCfnSignals(StartContainerScript):
method _get_signal_command (line 7) | def _get_signal_command(resource_name: str):
method _partials (line 10) | def _partials(self) -> dict:
method render (line 17) | def render(self, print_trace: bool = False) -> str:
FILE: spotty/providers/aws/cfn_templates/instance/template.py
function prepare_instance_template (line 25) | def prepare_instance_template(ec2, instance_config: InstanceConfig, dock...
function _read_template_file (line 240) | def _read_template_file(filename: str, params: dict = None):
function _get_volume_attachment_resource (line 250) | def _get_volume_attachment_resource(volume_id, device_name):
function _get_volume_resource (line 267) | def _get_volume_resource(ec2, volume: EbsVolume, output: AbstractOutputW...
function _get_volume_resources (line 314) | def _get_volume_resources(ec2, volumes: List[AbstractInstanceVolume], ou...
function get_template_parameters (line 357) | def get_template_parameters(ec2, instance_config: InstanceConfig, instan...
FILE: spotty/providers/aws/cfn_templates/instance_profile/template.py
function prepare_instance_profile_template (line 5) | def prepare_instance_profile_template(managed_policy_arns: list):
FILE: spotty/providers/aws/commands/clean_logs.py
class CleanLogsCommand (line 8) | class CleanLogsCommand(AbstractCommand):
method configure (line 13) | def configure(self, parser: ArgumentParser):
method run (line 19) | def run(self, args: Namespace, output: AbstractOutputWriter):
method _delete_log_groups (line 38) | def _delete_log_groups(logs, log_groups: list, prefixes: list, only_em...
FILE: spotty/providers/aws/commands/spot_prices.py
class SpotPricesCommand (line 8) | class SpotPricesCommand(AbstractCommand):
method configure (line 13) | def configure(self, parser: ArgumentParser):
method run (line 18) | def run(self, args: Namespace, output: AbstractOutputWriter):
FILE: spotty/providers/aws/config/ebs_volume.py
class EbsVolume (line 5) | class EbsVolume(AbstractInstanceVolume):
method __init__ (line 14) | def __init__(self, volume_config: dict, project_name: str, instance_na...
method _validate_volume_parameters (line 20) | def _validate_volume_parameters(self, params: dict) -> dict:
method title (line 24) | def title(self):
method size (line 28) | def size(self) -> int:
method type (line 32) | def type(self) -> str:
method deletion_policy (line 36) | def deletion_policy(self) -> str:
method deletion_policy_title (line 40) | def deletion_policy_title(self) -> str:
method ec2_volume_name (line 49) | def ec2_volume_name(self) -> str:
method mount_dir (line 58) | def mount_dir(self) -> str:
method host_path (line 68) | def host_path(self) -> str:
FILE: spotty/providers/aws/config/instance_config.py
class InstanceConfig (line 11) | class InstanceConfig(AbstractInstanceConfig):
method _validate_instance_params (line 13) | def _validate_instance_params(self, params: dict) -> dict:
method _get_instance_volumes (line 16) | def _get_instance_volumes(self) -> List[AbstractInstanceVolume]:
method user (line 28) | def user(self):
method ec2_instance_name (line 32) | def ec2_instance_name(self) -> str:
method region (line 36) | def region(self) -> str:
method availability_zone (line 40) | def availability_zone(self) -> str:
method subnet_id (line 44) | def subnet_id(self) -> str:
method instance_type (line 48) | def instance_type(self) -> str:
method is_spot_instance (line 52) | def is_spot_instance(self) -> bool:
method ami_name (line 56) | def ami_name(self) -> str:
method ami_id (line 60) | def ami_id(self) -> str:
method root_volume_size (line 64) | def root_volume_size(self) -> int:
method ports (line 68) | def ports(self) -> List[int]:
method max_price (line 72) | def max_price(self) -> float:
method managed_policy_arns (line 76) | def managed_policy_arns(self) -> list:
method instance_profile_arn (line 80) | def instance_profile_arn(self) -> str:
FILE: spotty/providers/aws/config/validation.py
function validate_instance_parameters (line 6) | def validate_instance_parameters(params: dict):
function validate_ebs_volume_parameters (line 54) | def validate_ebs_volume_parameters(params: dict):
FILE: spotty/providers/aws/data_transfer.py
class DataTransfer (line 7) | class DataTransfer(AbstractDataTransfer):
method __init__ (line 9) | def __init__(self, local_project_dir: str, host_project_dir: str, sync...
method scheme_name (line 16) | def scheme_name(self) -> str:
method upload_local_to_bucket (line 19) | def upload_local_to_bucket(self, bucket_name: str, dry_run: bool = Fal...
method download_bucket_to_local (line 34) | def download_bucket_to_local(self, bucket_name: str, download_filters:...
method get_download_bucket_to_instance_command (line 49) | def get_download_bucket_to_instance_command(self, bucket_name: str, us...
method get_upload_instance_to_bucket_command (line 58) | def get_upload_instance_to_bucket_command(self, bucket_name: str, down...
FILE: spotty/providers/aws/deletion_policies.py
function apply_deletion_policies (line 9) | def apply_deletion_policies(ec2, volumes: List[AbstractInstanceVolume], ...
function _delete_ec2_volume (line 92) | def _delete_ec2_volume(ec2_volume: Volume, output: AbstractOutputWriter):
function _delete_snapshot (line 100) | def _delete_snapshot(snapshot: Snapshot, output: AbstractOutputWriter):
FILE: spotty/providers/aws/errors/volume_not_found.py
class VolumeNotFoundError (line 1) | class VolumeNotFoundError(Exception):
method __init__ (line 2) | def __init__(self, volume_name):
FILE: spotty/providers/aws/helpers/ami.py
function get_ami (line 5) | def get_ami(ec2, ami_id: str = None, ami_name: str = None) -> Image:
FILE: spotty/providers/aws/helpers/availability_zone.py
function update_availability_zone (line 7) | def update_availability_zone(ec2, availability_zone: str, volumes: List[...
FILE: spotty/providers/aws/helpers/instance_prices.py
function get_spot_prices (line 8) | def get_spot_prices(ec2, instance_type: str):
function get_current_spot_price (line 24) | def get_current_spot_price(ec2, instance_type, availability_zone=''):
function get_on_demand_price (line 40) | def get_on_demand_price(instance_type: str, region: str):
function _get_region_name (line 64) | def _get_region_name(region: str):
function check_max_spot_price (line 77) | def check_max_spot_price(ec2, instance_type: str, is_spot_instance: bool...
FILE: spotty/providers/aws/helpers/logs.py
function get_logs_s3_path (line 8) | def get_logs_s3_path(bucket_name: str, instance_name: str) -> str:
function download_logs (line 12) | def download_logs(bucket_name: str, instance_name: str, stack_uuid: str,...
FILE: spotty/providers/aws/helpers/s3_sync.py
function check_aws_installed (line 5) | def check_aws_installed():
function get_s3_sync_command (line 11) | def get_s3_sync_command(from_path: str, to_path: str, profile: str = Non...
FILE: spotty/providers/aws/helpers/subnet.py
function check_az_and_subnet (line 4) | def check_az_and_subnet(ec2, region: str, availability_zone: str, subnet...
FILE: spotty/providers/aws/helpers/vpc.py
function get_vpc_id (line 5) | def get_vpc_id(ec2, subnet_id: str = None) -> str:
FILE: spotty/providers/aws/instance_deployment.py
class InstanceDeployment (line 20) | class InstanceDeployment(AbstractInstanceDeployment):
method __init__ (line 24) | def __init__(self, instance_config: InstanceConfig):
method stack_manager (line 31) | def stack_manager(self) -> InstanceStackManager:
method key_pair_manager (line 35) | def key_pair_manager(self) -> KeyPairManager:
method get_instance (line 38) | def get_instance(self) -> Instance:
method deploy (line 41) | def deploy(self, container_commands: DockerCommands, bucket_name: str,
method delete (line 122) | def delete(self, output: AbstractOutputWriter):
FILE: spotty/providers/aws/instance_manager.py
class InstanceManager (line 10) | class InstanceManager(AbstractCloudInstanceManager):
method _get_instance_config (line 17) | def _get_instance_config(self, instance_config: dict) -> InstanceConfig:
method _get_bucket_manager (line 21) | def _get_bucket_manager(self) -> BucketManager:
method _get_data_transfer (line 25) | def _get_data_transfer(self) -> DataTransfer:
method _get_instance_deployment (line 35) | def _get_instance_deployment(self) -> InstanceDeployment:
method get_status_text (line 39) | def get_status_text(self):
method ssh_key_path (line 67) | def ssh_key_path(self):
FILE: spotty/providers/aws/resource_managers/bucket_manager.py
class BucketManager (line 9) | class BucketManager(AbstractBucketManager):
method __init__ (line 11) | def __init__(self, project_name: str, region: str):
method get_bucket (line 18) | def get_bucket(self) -> Bucket:
method create_bucket (line 34) | def create_bucket(self) -> Bucket:
method delete_bucket (line 46) | def delete_bucket(self):
FILE: spotty/providers/aws/resource_managers/instance_profile_stack_manager.py
class InstanceProfileStackManager (line 8) | class InstanceProfileStackManager(object):
method __init__ (line 10) | def __init__(self, project_name: str, instance_name: str, region: str):
method create_or_update_stack (line 15) | def create_or_update_stack(self, managed_policy_arns: list, output: Ab...
method _create_stack (line 48) | def _create_stack(self, template: str, output: AbstractOutputWriter):
method _update_stack (line 63) | def _update_stack(self, template: str, output: AbstractOutputWriter):
FILE: spotty/providers/aws/resource_managers/instance_stack_manager.py
class InstanceStackManager (line 7) | class InstanceStackManager(object):
method __init__ (line 9) | def __init__(self, project_name: str, instance_name: str, region: str):
method name (line 16) | def name(self):
method create_or_update_stack (line 19) | def create_or_update_stack(self, template: str, parameters: dict, inst...
method delete_stack (line 105) | def delete_stack(self, output: AbstractOutputWriter, no_wait=False):
FILE: spotty/providers/aws/resource_managers/key_pair_manager.py
class KeyPairManager (line 6) | class KeyPairManager(object):
method __init__ (line 8) | def __init__(self, ec2, project_name: str, region: str):
method key_name (line 14) | def key_name(self):
method key_path (line 18) | def key_path(self):
method maybe_create_key (line 21) | def maybe_create_key(self):
method delete_key (line 49) | def delete_key(self):
method _ec2_key_exists (line 58) | def _ec2_key_exists(self):
FILE: spotty/providers/aws/resources/bucket.py
class Bucket (line 4) | class Bucket(AbstractBucket):
method __init__ (line 6) | def __init__(self, data: dict):
method name (line 10) | def name(self) -> str:
FILE: spotty/providers/aws/resources/image.py
class Image (line 1) | class Image(object):
method __init__ (line 3) | def __init__(self, ec2, ami_info):
method get_by_name (line 8) | def get_by_name(ec2, ami_name: str):
method get_by_id (line 23) | def get_by_id(ec2, ami_id: str):
method image_id (line 33) | def image_id(self) -> str:
method name (line 37) | def name(self) -> str:
method size (line 41) | def size(self) -> int:
method get_tag_value (line 44) | def get_tag_value(self, tag_name):
FILE: spotty/providers/aws/resources/instance.py
class Instance (line 6) | class Instance(AbstractInstance):
method __init__ (line 8) | def __init__(self, ec2, data: dict):
method get_by_stack_name (line 13) | def get_by_stack_name(ec2, stack_name):
method instance_id (line 31) | def instance_id(self):
method public_ip_address (line 35) | def public_ip_address(self) -> str:
method private_ip_address (line 39) | def private_ip_address(self) -> str:
method state (line 43) | def state(self) -> str:
method instance_type (line 47) | def instance_type(self) -> str:
method availability_zone (line 51) | def availability_zone(self) -> str:
method launch_time (line 55) | def launch_time(self) -> datetime:
method lifecycle (line 59) | def lifecycle(self) -> str:
method is_running (line 63) | def is_running(self):
method is_stopped (line 67) | def is_stopped(self):
method get_spot_price (line 70) | def get_spot_price(self):
method get_on_demand_price (line 74) | def get_on_demand_price(self):
method terminate (line 78) | def terminate(self, wait: bool = True):
method stop (line 84) | def stop(self, wait: bool = True):
FILE: spotty/providers/aws/resources/snapshot.py
class Snapshot (line 4) | class Snapshot(object):
method __init__ (line 6) | def __init__(self, ec2, snapshot_info):
method get_by_name (line 11) | def get_by_name(ec2, snapshot_name: str):
method name (line 26) | def name(self) -> str:
method snapshot_id (line 34) | def snapshot_id(self):
method size (line 38) | def size(self) -> int:
method creation_time (line 42) | def creation_time(self) -> int:
method rename (line 45) | def rename(self, new_name):
method delete (line 49) | def delete(self):
method wait_snapshot_completed (line 52) | def wait_snapshot_completed(self):
FILE: spotty/providers/aws/resources/stack.py
class Stack (line 12) | class Stack(object):
method __init__ (line 14) | def __init__(self, cf, stack_info):
method get_by_name (line 19) | def get_by_name(cf, stack_name: str):
method create_stack (line 37) | def create_stack(cf, *args, **kwargs):
method update_stack (line 42) | def update_stack(cf, *args, **kwargs):
method stack_id (line 47) | def stack_id(self) -> str:
method stack_uuid (line 51) | def stack_uuid(self) -> str:
method name (line 55) | def name(self) -> str:
method status (line 59) | def status(self) -> str:
method outputs (line 63) | def outputs(self) -> str:
method delete (line 66) | def delete(self):
method wait_stack_created (line 69) | def wait_stack_created(self, delay_secs: int = 30):
method wait_stack_updated (line 73) | def wait_stack_updated(self, delay_secs: int = 30):
method wait_stack_deleted (line 77) | def wait_stack_deleted(self, delay_secs: int = 30):
method wait_status_changed (line 81) | def wait_status_changed(self, stack_waiting_status: str, output: Abstr...
method wait_tasks (line 98) | def wait_tasks(self, tasks: List[Task], resource_success_status: str, ...
method _get_resource_statuses (line 128) | def _get_resource_statuses(self) -> Dict[str, str]:
FILE: spotty/providers/aws/resources/subnet.py
class Subnet (line 1) | class Subnet(object):
method __init__ (line 3) | def __init__(self, ec2, subnet_info):
method get_by_id (line 8) | def get_by_id(ec2, subnet_id: str):
method get_default_subnets (line 20) | def get_default_subnets(ec2):
method availability_zone (line 30) | def availability_zone(self) -> str:
method vpc_id (line 34) | def vpc_id(self) -> str:
FILE: spotty/providers/aws/resources/volume.py
class Volume (line 4) | class Volume(object):
method __init__ (line 6) | def __init__(self, ec2, volume_info):
method get_by_name (line 11) | def get_by_name(ec2, volume_name: str):
method name (line 26) | def name(self) -> str:
method volume_id (line 34) | def volume_id(self) -> str:
method size (line 38) | def size(self) -> int:
method availability_zone (line 42) | def availability_zone(self) -> str:
method state (line 46) | def state(self) -> str:
method is_available (line 49) | def is_available(self):
method create_snapshot (line 52) | def create_snapshot(self) -> Snapshot:
method delete (line 66) | def delete(self):
FILE: spotty/providers/aws/resources/vpc.py
class Vpc (line 1) | class Vpc(object):
method __init__ (line 3) | def __init__(self, ec2, vpc_info):
method get_default_vpc (line 8) | def get_default_vpc(ec2):
method vpc_id (line 17) | def vpc_id(self) -> str:
FILE: spotty/providers/gcp/config/disk_volume.py
class DiskVolume (line 5) | class DiskVolume(AbstractInstanceVolume):
method __init__ (line 14) | def __init__(self, volume_config: dict, project_name: str, instance_na...
method _validate_volume_parameters (line 20) | def _validate_volume_parameters(self, params: dict) -> dict:
method title (line 24) | def title(self):
method size (line 28) | def size(self) -> int:
method deletion_policy (line 32) | def deletion_policy(self) -> str:
method deletion_policy_title (line 36) | def deletion_policy_title(self) -> str:
method disk_name (line 45) | def disk_name(self) -> str:
method mount_dir (line 54) | def mount_dir(self) -> str:
method host_path (line 64) | def host_path(self) -> str:
FILE: spotty/providers/gcp/config/image_uri.py
class ImageUri (line 8) | class ImageUri(object):
method __init__ (line 10) | def __init__(self, image_uri: str):
method project_id (line 18) | def project_id(self) -> str:
method is_family (line 22) | def is_family(self):
method name (line 26) | def name(self):
FILE: spotty/providers/gcp/config/instance_config.py
class InstanceConfig (line 12) | class InstanceConfig(AbstractInstanceConfig):
method _validate_instance_params (line 14) | def _validate_instance_params(self, params: dict) -> dict:
method _get_instance_volumes (line 17) | def _get_instance_volumes(self) -> List[AbstractInstanceVolume]:
method user (line 29) | def user(self):
method machine_name (line 33) | def machine_name(self) -> str:
method project_id (line 38) | def project_id(self) -> str:
method zone (line 42) | def zone(self) -> str:
method machine_type (line 46) | def machine_type(self) -> str:
method gpu (line 50) | def gpu(self) -> dict:
method is_preemptible_instance (line 54) | def is_preemptible_instance(self) -> bool:
method boot_disk_size (line 58) | def boot_disk_size(self) -> int:
method ports (line 62) | def ports(self) -> List[int]:
method image_name (line 66) | def image_name(self) -> str:
method has_image_name (line 70) | def has_image_name(self) -> bool:
method image_uri (line 74) | def image_uri(self) -> str:
FILE: spotty/providers/gcp/config/validation.py
function validate_instance_parameters (line 7) | def validate_instance_parameters(params: dict):
function validate_disk_volume_parameters (line 46) | def validate_disk_volume_parameters(params: dict):
FILE: spotty/providers/gcp/data_transfer.py
class DataTransfer (line 7) | class DataTransfer(AbstractDataTransfer):
method scheme_name (line 10) | def scheme_name(self) -> str:
method upload_local_to_bucket (line 13) | def upload_local_to_bucket(self, bucket_name: str, dry_run: bool = Fal...
method download_bucket_to_local (line 28) | def download_bucket_to_local(self, bucket_name: str, download_filters:...
method get_download_bucket_to_instance_command (line 32) | def get_download_bucket_to_instance_command(self, bucket_name: str, us...
method get_upload_instance_to_bucket_command (line 41) | def get_upload_instance_to_bucket_command(self, bucket_name: str, down...
FILE: spotty/providers/gcp/dm_templates/instance/instance_template.py
function prepare_instance_template (line 17) | def prepare_instance_template(instance_config: InstanceConfig, docker_co...
function _get_disk_attachments (line 137) | def _get_disk_attachments(volumes: List[AbstractInstanceVolume], zone: s...
FILE: spotty/providers/gcp/errors/image_not_found.py
class ImageNotFoundError (line 1) | class ImageNotFoundError(Exception):
method __init__ (line 2) | def __init__(self, image_name):
FILE: spotty/providers/gcp/helpers/ce_client.py
class CEClient (line 6) | class CEClient(object):
method __init__ (line 9) | def __init__(self, project_id: str, zone: str):
method zone (line 15) | def zone(self):
method list_images (line 18) | def list_images(self, image_name: str = None, project_id: str = None):
method get_image_from_family (line 33) | def get_image_from_family(self, family_name: str, project_id: str = No...
method list_instances (line 41) | def list_instances(self, machine_name=None):
method list_disks (line 50) | def list_disks(self, disk_name=None):
method list_snapshots (line 59) | def list_snapshots(self, snapshot_name=None):
method get_accelerator_types (line 68) | def get_accelerator_types(self) -> OrderedDict:
method create_disk (line 75) | def create_disk(self, name: str, size: int = None, snapshot_link: str ...
method get_machine_types (line 92) | def get_machine_types(self, machine_type: str = None):
method stop_instance (line 104) | def stop_instance(self, machine_name: str, wait: bool = True) -> str:
method delete_instance (line 113) | def delete_instance(self, machine_name: str, wait: bool = True) -> str:
method _wait_operation (line 122) | def _wait_operation(self, operation: dict):
FILE: spotty/providers/gcp/helpers/deployment.py
function wait_resources (line 13) | def wait_resources(dm: DMClient, ce: CEClient, deployment_name: str, res...
function check_gpu_configuration (line 64) | def check_gpu_configuration(ce: CEClient, gpu_parameters: dict):
FILE: spotty/providers/gcp/helpers/dm_client.py
class DMClient (line 6) | class DMClient(object):
method __init__ (line 9) | def __init__(self, project_id: str, zone: str):
method get (line 14) | def get(self, deployment_name: str):
method deploy (line 25) | def deploy(self, deployment_name: str, template: str, dry_run: bool = ...
method stop (line 37) | def stop(self, deployment_name: str, fingerprint: str):
method delete (line 44) | def delete(self, deployment_name: str):
method get_resource (line 49) | def get_resource(self, deployment_name: str, resource_name: str) -> dict:
FILE: spotty/providers/gcp/helpers/dm_resource.py
class DMResource (line 4) | class DMResource(object):
method __init__ (line 6) | def __init__(self, dm: DMClient, data: dict):
method get_by_name (line 61) | def get_by_name(dm: DMClient, deployment_name: str, resource_name: str):
method is_created (line 70) | def is_created(self) -> bool:
method error_message (line 74) | def error_message(self) -> str:
method state (line 81) | def state(self) -> str:
method is_in_progress (line 85) | def is_in_progress(self) -> bool:
method is_failed (line 89) | def is_failed(self) -> bool:
FILE: spotty/providers/gcp/helpers/gcp_credentials.py
class GcpCredentials (line 4) | class GcpCredentials(object):
method __init__ (line 5) | def __init__(self):
method project_id (line 12) | def project_id(self):
method service_account_email (line 16) | def service_account_email(self):
FILE: spotty/providers/gcp/helpers/gs_client.py
class GSClient (line 6) | class GSClient(object):
method __init__ (line 9) | def __init__(self):
method list_buckets (line 12) | def list_buckets(self) -> List[Bucket]:
method create_bucket (line 16) | def create_bucket(self, bucket_name: str, region: str) -> Bucket:
method create_dir (line 22) | def create_dir(self, bucket_name: str, path: str):
FILE: spotty/providers/gcp/helpers/gsutil_rsync.py
function check_gsutil_installed (line 9) | def check_gsutil_installed():
function get_rsync_command (line 15) | def get_rsync_command(from_path: str, to_path: str, filters: List[dict] ...
FILE: spotty/providers/gcp/helpers/image.py
function get_image (line 6) | def get_image(ce: CEClient, image_uri: str = None, image_name: str = Non...
FILE: spotty/providers/gcp/helpers/rtc_client.py
class RtcClient (line 4) | class RtcClient(object):
method __init__ (line 6) | def __init__(self, project_id: str, zone: str):
method get_value (line 11) | def get_value(self, config_name, template):
method set_value (line 18) | def set_value(self, config_name: str, variable_name: str, value: str):
FILE: spotty/providers/gcp/helpers/volumes.py
function create_disks (line 10) | def create_disks(ce: CEClient, volumes: List[AbstractInstanceVolume], ou...
FILE: spotty/providers/gcp/instance_deployment.py
class InstanceDeployment (line 18) | class InstanceDeployment(AbstractInstanceDeployment):
method __init__ (line 22) | def __init__(self, instance_config: InstanceConfig):
method stack_manager (line 30) | def stack_manager(self) -> InstanceStackManager:
method ssh_key_manager (line 34) | def ssh_key_manager(self) -> SshKeyManager:
method get_instance (line 37) | def get_instance(self) -> Instance:
method deploy (line 40) | def deploy(self, container_commands: DockerCommands, bucket_name: str,
method delete (line 99) | def delete(self, output: AbstractOutputWriter):
FILE: spotty/providers/gcp/instance_manager.py
class InstanceManager (line 11) | class InstanceManager(AbstractCloudInstanceManager):
method _get_instance_config (line 18) | def _get_instance_config(self, instance_config: dict) -> InstanceConfig:
method _get_bucket_manager (line 22) | def _get_bucket_manager(self) -> BucketManager:
method _get_data_transfer (line 26) | def _get_data_transfer(self) -> DataTransfer:
method _get_instance_deployment (line 35) | def _get_instance_deployment(self) -> InstanceDeployment:
method download (line 39) | def download(self, download_filters: list, output: AbstractOutputWrite...
method get_status_text (line 42) | def get_status_text(self) -> str:
method ssh_key_path (line 61) | def ssh_key_path(self):
FILE: spotty/providers/gcp/resource_managers/bucket_manager.py
class BucketManager (line 9) | class BucketManager(AbstractBucketManager):
method __init__ (line 11) | def __init__(self, project_name: str, region: str):
method get_bucket (line 18) | def get_bucket(self) -> Bucket:
method create_bucket (line 35) | def create_bucket(self) -> Bucket:
FILE: spotty/providers/gcp/resource_managers/instance_stack_manager.py
class InstanceStackManager (line 11) | class InstanceStackManager(object):
method __init__ (line 13) | def __init__(self, machine_name: str, project_id: str, zone: str):
method name (line 26) | def name(self):
method create_stack (line 29) | def create_stack(self, template: str, output: AbstractOutputWriter):
method delete_stack (line 50) | def delete_stack(self, output: AbstractOutputWriter):
FILE: spotty/providers/gcp/resource_managers/ssh_key_manager.py
class SshKeyManager (line 8) | class SshKeyManager(object):
method __init__ (line 10) | def __init__(self, project_name: str, zone: str):
method private_key_file (line 15) | def private_key_file(self):
method public_key_file (line 19) | def public_key_file(self):
method get_public_key_value (line 22) | def get_public_key_value(self):
method _generate_ssh_key (line 33) | def _generate_ssh_key(self):
FILE: spotty/providers/gcp/resources/bucket.py
class Bucket (line 5) | class Bucket(AbstractBucket):
method __init__ (line 7) | def __init__(self, bucket: GSBucket):
method name (line 11) | def name(self) -> str:
FILE: spotty/providers/gcp/resources/disk.py
class Disk (line 4) | class Disk(object):
method __init__ (line 6) | def __init__(self, ce: CEClient, data: dict):
method get_by_name (line 33) | def get_by_name(ce: CEClient, disk_name: str):
method name (line 42) | def name(self) -> str:
method status (line 46) | def status(self) -> str:
method size (line 50) | def size(self) -> int:
method users (line 54) | def users(self) -> list:
method is_available (line 57) | def is_available(self):
FILE: spotty/providers/gcp/resources/image.py
class Image (line 5) | class Image(object):
method __init__ (line 7) | def __init__(self, data: dict):
method get_by_name (line 31) | def get_by_name(ce: CEClient, image_name: str):
method get_by_uri (line 40) | def get_by_uri(ce: CEClient, image_uri: str):
method image_id (line 54) | def image_id(self) -> str:
method name (line 58) | def name(self) -> str:
method size (line 62) | def size(self) -> int:
method self_link (line 66) | def self_link(self) -> str:
method source_disk (line 70) | def source_disk(self):
FILE: spotty/providers/gcp/resources/instance.py
class Instance (line 6) | class Instance(AbstractInstance):
method __init__ (line 8) | def __init__(self, ce: CEClient, data: dict):
method get_by_name (line 66) | def get_by_name(ce: CEClient, machine_name: str):
method name (line 75) | def name(self) -> str:
method is_running (line 79) | def is_running(self) -> bool:
method is_stopped (line 83) | def is_stopped(self) -> bool:
method public_ip_address (line 88) | def public_ip_address(self) -> str:
method status (line 92) | def status(self) -> str:
method machine_type (line 96) | def machine_type(self) -> str:
method zone (line 100) | def zone(self) -> str:
method creation_timestamp (line 104) | def creation_timestamp(self) -> datetime:
method is_preemtible (line 112) | def is_preemtible(self) -> bool:
method terminate (line 115) | def terminate(self, wait: bool = True):
method stop (line 118) | def stop(self, wait: bool = True):
FILE: spotty/providers/gcp/resources/snapshot.py
class Snapshot (line 4) | class Snapshot(object):
method __init__ (line 6) | def __init__(self, data: dict):
method get_by_name (line 27) | def get_by_name(ce: CEClient, snapshot_name: str):
method name (line 36) | def name(self) -> str:
method size (line 40) | def size(self) -> int:
method self_link (line 44) | def self_link(self) -> str:
FILE: spotty/providers/gcp/resources/stack.py
class Stack (line 7) | class Stack(object):
method __init__ (line 9) | def __init__(self, dm: DMClient, data: dict):
method get_by_name (line 53) | def get_by_name(dm: DMClient, deployment_name: str):
method create (line 62) | def create(dm: DMClient, deployment_name: str, template: str):
method name (line 66) | def name(self) -> str:
method status (line 70) | def status(self) -> str:
method is_running (line 74) | def is_running(self):
method is_done (line 78) | def is_done(self):
method error (line 83) | def error(self) -> str:
method fingerprint (line 88) | def fingerprint(self) -> str:
method stop (line 91) | def stop(self):
method delete (line 94) | def delete(self):
method wait_stack_deleted (line 97) | def wait_stack_deleted(self, delay=15):
method wait_stack_done (line 108) | def wait_stack_done(self, delay=5):
FILE: spotty/providers/instance_manager_factory.py
class InstanceManagerFactory (line 12) | class InstanceManagerFactory(object):
method get_instance (line 22) | def get_instance(cls, project_config: ProjectConfig, instance_config: ...
FILE: spotty/providers/local/config/instance_config.py
class InstanceConfig (line 11) | class InstanceConfig(AbstractInstanceConfig):
method __init__ (line 13) | def __init__(self, instance_config: dict, project_config: ProjectConfig):
method _validate_instance_params (line 16) | def _validate_instance_params(self, params: dict):
method _get_instance_volumes (line 20) | def _get_instance_volumes(self) -> List[AbstractInstanceVolume]:
method _get_volume_mounts (line 31) | def _get_volume_mounts(self, volumes: List[AbstractInstanceVolume]) ->...
method user (line 50) | def user(self) -> str:
FILE: spotty/providers/local/config/validation.py
function validate_instance_parameters (line 4) | def validate_instance_parameters(params: dict):
FILE: spotty/providers/local/instance_manager.py
class InstanceManager (line 7) | class InstanceManager(AbstractDockerInstanceManager):
method _get_instance_config (line 11) | def _get_instance_config(self, instance_config: dict) -> InstanceConfig:
method is_running (line 15) | def is_running(self):
method clean (line 18) | def clean(self, output: AbstractOutputWriter):
method sync (line 21) | def sync(self, output: AbstractOutputWriter, dry_run=False):
method download (line 24) | def download(self, download_filters: list, output: AbstractOutputWrite...
FILE: spotty/providers/remote/config/instance_config.py
class InstanceConfig (line 10) | class InstanceConfig(AbstractInstanceConfig):
method __init__ (line 12) | def __init__(self, instance_config: dict, project_config: ProjectConfig):
method _validate_instance_params (line 15) | def _validate_instance_params(self, params: dict):
method user (line 20) | def user(self) -> str:
method host (line 24) | def host(self) -> str:
method port (line 28) | def port(self) -> int:
method key_path (line 32) | def key_path(self) -> str:
method _get_instance_volumes (line 41) | def _get_instance_volumes(self) -> List[AbstractInstanceVolume]:
FILE: spotty/providers/remote/config/validation.py
function validate_instance_parameters (line 5) | def validate_instance_parameters(params: dict):
FILE: spotty/providers/remote/helpers/rsync.py
function check_rsync_installed (line 6) | def check_rsync_installed():
function get_upload_command (line 12) | def get_upload_command(local_dir: str, remote_dir: str, ssh_user: str, s...
function get_download_command (line 24) | def get_download_command(remote_dir: str, local_dir: str, ssh_user: str,...
function _get_rsync_command (line 33) | def _get_rsync_command(src_path: str, dst_path: str, ssh_port: int, ssh_...
function _fix_filter_path (line 70) | def _fix_filter_path(path: str) -> str:
FILE: spotty/providers/remote/instance_manager.py
class InstanceManager (line 9) | class InstanceManager(AbstractSshInstanceManager):
method _get_instance_config (line 13) | def _get_instance_config(self, instance_config: dict) -> InstanceConfig:
method is_running (line 17) | def is_running(self):
method clean (line 21) | def clean(self, output: AbstractOutputWriter):
method sync (line 24) | def sync(self, output: AbstractOutputWriter, dry_run=False):
method download (line 50) | def download(self, download_filters: list, output: AbstractOutputWrite...
method ssh_host (line 77) | def ssh_host(self) -> str:
method ssh_key_path (line 81) | def ssh_key_path(self) -> str:
method ssh_port (line 85) | def ssh_port(self) -> int:
FILE: spotty/utils.py
function package_dir (line 7) | def package_dir(path: str = ''):
function check_path (line 21) | def check_path(path):
function random_string (line 31) | def random_string(length: int, chars: str = string.ascii_lowercase + str...
function filter_list (line 35) | def filter_list(list_of_dicts, key_name, value):
function render_table (line 39) | def render_table(table: list, separate_title=False):
FILE: tests/container_config.py
class TestContainerConfig (line 5) | class TestContainerConfig(unittest.TestCase):
method test_working_dir (line 7) | def test_working_dir(self):
FILE: tests/helpers/cli.py
function run (line 5) | def run(command: str, capture_output: bool = False, assert_zero_code: bo...
function touch_file (line 20) | def touch_file(file_path: str):
FILE: tests/helpers/spotty_cli.py
class SpottyCli (line 7) | class SpottyCli:
method __init__ (line 9) | def __init__(self, instance_name: str):
method is_instance_running (line 12) | def is_instance_running(self) -> bool:
method start_instance (line 17) | def start_instance(self):
method list_remote_files (line 23) | def list_remote_files(self) -> List[str]:
method touch_file (line 30) | def touch_file(self, file_path: str):
method sync (line 34) | def sync(self):
method download (line 48) | def download(self, filter_pattern: str):
method exec (line 67) | def exec(self, container_command: str):
FILE: tests/providers/aws/commands/download.py
class TestInstanceDownload (line 7) | class TestInstanceDownload(unittest.TestCase):
method setUpClass (line 12) | def setUpClass(cls):
method test_download_file (line 30) | def test_download_file(self):
method test_wildcard (line 64) | def test_wildcard(self):
FILE: tests/providers/aws/commands/sync.py
class TestInstanceSync (line 7) | class TestInstanceSync(unittest.TestCase):
method setUpClass (line 12) | def setUpClass(cls):
method test_update_local_file (line 30) | def test_update_local_file(self):
method test_update_remote_file (line 54) | def test_update_remote_file(self):
method test_new_remote_file (line 73) | def test_new_remote_file(self):
FILE: tests/providers/aws/config/container_deployment.py
class TestContainerDeployment (line 9) | class TestContainerDeployment(unittest.TestCase):
method test_instance_volume (line 11) | def test_instance_volume(self):
method test_tmp_project_volume (line 33) | def test_tmp_project_volume(self):
FILE: tests/providers/aws/config/instance_config_validation.py
class TestBucketResource (line 5) | class TestBucketResource(unittest.TestCase):
method test_default_configuration (line 7) | def test_default_configuration(self):
method test_failed_validation (line 35) | def test_failed_validation(self):
FILE: tests/providers/aws/project_resources/bucket.py
class TestBucketResource (line 7) | class TestBucketResource(unittest.TestCase):
method test_create_and_find_bucket (line 10) | def test_create_and_find_bucket(self):
FILE: tests/providers/aws/project_resources/key_pair.py
class TestKeyPairResource (line 9) | class TestKeyPairResource(unittest.TestCase):
method test_key_path (line 11) | def test_key_path(self):
method test_create_and_delete_key (line 22) | def test_create_and_delete_key(self):
FILE: tests/providers/gcp/config/image_uri.py
class TestImageUrl (line 5) | class TestImageUrl(unittest.TestCase):
method test_image_url_parsing (line 7) | def test_image_url_parsing(self):
FILE: tests/providers/local/commands/run.py
class TestInstanceRun (line 8) | class TestInstanceRun(unittest.TestCase):
method setUpClass (line 13) | def setUpClass(cls):
method test_script_arguments (line 21) | def test_script_arguments(self):
method test_script_params (line 35) | def test_script_params(self):
method test_script_logging (line 48) | def test_script_logging(self):
FILE: tests/providers/local/config/container_deployment.py
class TestContainerDeployment (line 9) | class TestContainerDeployment(unittest.TestCase):
method test_instance_volume (line 11) | def test_instance_volume(self):
Condensed preview — 250 files, each showing path, character count, and a content snippet. Download the .json file or copy for the full structured content (397K chars).
[
{
"path": ".github/FUNDING.yml",
"chars": 18,
"preview": "github: [apls777]\n"
},
{
"path": ".github/workflows/generate-docs.yml",
"chars": 1118,
"preview": "# This workflows will upload a Python Package using Twine when a release is created\n# For more information see: https://"
},
{
"path": ".github/workflows/python-publish.yml",
"chars": 865,
"preview": "# This workflows will upload a Python Package using Twine when a release is created\n# For more information see: https://"
},
{
"path": ".gitignore",
"chars": 50,
"preview": ".idea/\nbuild/\ndist/\n*.egg-info/\n__pycache__/\ntodo\n"
},
{
"path": "CONTRIBUTING.md",
"chars": 2330,
"preview": "# Contributing to Spotty\n\n**Thank you for your interest in Spotty. Your contributions are highly welcome.**\n\nThere are m"
},
{
"path": "LICENSE",
"chars": 1079,
"preview": "The MIT License (MIT)\n\nCopyright (c) 2017 Oleg Polosin\n\nPermission is hereby granted, free of charge, to any person obta"
},
{
"path": "README.md",
"chars": 3211,
"preview": "<img src=\"https://spotty.cloud/_static/images/logo_740x240.png\" width=\"370\" height=\"120\" />\n\n[![Documentation](https://i"
},
{
"path": "bin/spotty",
"chars": 1008,
"preview": "#!/usr/bin/env python\n\nimport sys\nimport logging\nimport spotty\nfrom spotty.cli import get_parser\nfrom spotty.commands.wr"
},
{
"path": "docs/Makefile",
"chars": 610,
"preview": "# Minimal makefile for Sphinx documentation\n#\n\n# You can set these variables from the command line.\nSPHINXOPTS =\nSPHI"
},
{
"path": "docs/make.bat",
"chars": 772,
"preview": "@ECHO OFF\n\npushd %~dp0\n\nREM Command file for Sphinx documentation\n\nif \"%SPHINXBUILD%\" == \"\" (\n\tset SPHINXBUILD=python -m"
},
{
"path": "docs/requirements.txt",
"chars": 109,
"preview": "sphinx==3.1.2\nrecommonmark==0.6.0\nsphinx-argparse==0.2.5\nsphinx-rtd-theme==0.5.0\nPyYAML\nschema\nchevron\nboto3\n"
},
{
"path": "docs/source/_static/favicon/browserconfig.xml",
"chars": 246,
"preview": "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n<browserconfig>\n <msapplication>\n <tile>\n <square150x150logo"
},
{
"path": "docs/source/_static/favicon/site.webmanifest",
"chars": 426,
"preview": "{\n \"name\": \"\",\n \"short_name\": \"\",\n \"icons\": [\n {\n \"src\": \"/android-chrome-192x192.png\",\n "
},
{
"path": "docs/source/_static/scripts.js",
"chars": 174,
"preview": "window.onload = function() {\n var links = document.querySelectorAll('a.external');\n\n for(var i = 0; i < links.leng"
},
{
"path": "docs/source/_static/styles.css",
"chars": 392,
"preview": ".wy-nav-content {\n max-width: 1280px;\n}\n\n.wy-side-nav-search {\n background: none;\n}\n\n.wy-menu-vertical header, .wy"
},
{
"path": "docs/source/conf.py",
"chars": 6007,
"preview": "#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n#\n# spotty documentation build configuration file, created by\n# sphinx-qu"
},
{
"path": "docs/source/docs/cli/spotty-aws.rst",
"chars": 119,
"preview": "spotty aws\n==========\n\n.. argparse::\n :nodefaultconst:\n :ref: spotty.cli.get_parser\n :prog: spotty\n :path: aws\n"
},
{
"path": "docs/source/docs/cli/spotty-download.rst",
"chars": 134,
"preview": "spotty download\n===============\n\n.. argparse::\n :nodefaultconst:\n :ref: spotty.cli.get_parser\n :prog: spotty\n :p"
},
{
"path": "docs/source/docs/cli/spotty-exec.rst",
"chars": 122,
"preview": "spotty exec\n===========\n\n.. argparse::\n :nodefaultconst:\n :ref: spotty.cli.get_parser\n :prog: spotty\n :path: exe"
},
{
"path": "docs/source/docs/cli/spotty-run.rst",
"chars": 119,
"preview": "spotty run\n==========\n\n.. argparse::\n :nodefaultconst:\n :ref: spotty.cli.get_parser\n :prog: spotty\n :path: run\n"
},
{
"path": "docs/source/docs/cli/spotty-sh.rst",
"chars": 116,
"preview": "spotty sh\n=========\n\n.. argparse::\n :nodefaultconst:\n :ref: spotty.cli.get_parser\n :prog: spotty\n :path: sh\n"
},
{
"path": "docs/source/docs/cli/spotty-start.rst",
"chars": 125,
"preview": "spotty start\n============\n\n.. argparse::\n :nodefaultconst:\n :ref: spotty.cli.get_parser\n :prog: spotty\n :path: s"
},
{
"path": "docs/source/docs/cli/spotty-stop.rst",
"chars": 122,
"preview": "spotty stop\n===========\n\n.. argparse::\n :nodefaultconst:\n :ref: spotty.cli.get_parser\n :prog: spotty\n :path: sto"
},
{
"path": "docs/source/docs/cli/spotty-sync.rst",
"chars": 122,
"preview": "spotty sync\n===========\n\n.. argparse::\n :nodefaultconst:\n :ref: spotty.cli.get_parser\n :prog: spotty\n :path: syn"
},
{
"path": "docs/source/docs/cli/spotty.rst",
"chars": 446,
"preview": "Spotty Command-line Interface\n=============================\n\n.. argparse::\n :nosubcommands:\n :nodefaultconst:\n :no"
},
{
"path": "docs/source/docs/providers/aws/caching-docker-image-on-an-ebs-volume.md",
"chars": 908,
"preview": "# Caching Docker Image on an EBS Volume\n\nYou can cache images that you've built or downloaded from the internet on an EB"
},
{
"path": "docs/source/docs/providers/aws/ebs-volumes-and-deletion-policies.md",
"chars": 1466,
"preview": "# EBS Volumes and Deletion Policies\n\nBy default, EBS volumes have names in the following format: `<PROJECT_NAME>-<INSTAN"
},
{
"path": "docs/source/docs/providers/aws/faq.md",
"chars": 2537,
"preview": "# FAQ\n\n## How does Spotty choose the AWS Availability Zone where to run the instance?\n\n1. If the AZ is specified in the "
},
{
"path": "docs/source/docs/providers/aws/instance-parameters.md",
"chars": 5317,
"preview": "# Instance Parameters\n\nInstance parameters are part of the [configuration file], but for each provider they are differen"
},
{
"path": "docs/source/docs/providers/aws/overview.rst",
"chars": 171,
"preview": "AWS Provider Overview\n=====================\n\n.. toctree::\n\n instance-parameters\n ebs-volumes-and-deletion-policies"
},
{
"path": "docs/source/docs/providers/gcp/account-preparation.md",
"chars": 1761,
"preview": "# GCP Account Preparation\n\n1. [Create a project](https://console.cloud.google.com/projectcreate) \nif you don't have one "
},
{
"path": "docs/source/docs/providers/gcp/caching-docker-image-on-a-disk.md",
"chars": 725,
"preview": "# Caching Docker Image on a Disk\n\nYou can cache images that you've built or downloaded from the internet on a disk that "
},
{
"path": "docs/source/docs/providers/gcp/disks-and-deletion-policies.md",
"chars": 766,
"preview": "# Disks and Deletion Policies\n\nBy default, disks have names in the following format: `<PROJECT_NAME>-<INSTANCE_NAME>-<VO"
},
{
"path": "docs/source/docs/providers/gcp/instance-parameters.md",
"chars": 4785,
"preview": "# Instance Parameters\n\nInstance parameters are part of the [configuration file], but for each provider they are differen"
},
{
"path": "docs/source/docs/providers/gcp/overview.rst",
"chars": 174,
"preview": "GCP Provider Overview\n=====================\n\n.. toctree::\n\n account-preparation\n instance-parameters\n disks-and"
},
{
"path": "docs/source/docs/providers/local/instance-parameters.md",
"chars": 770,
"preview": "# Instance Parameters\n\nInstance parameters are part of the [configuration file], but for each provider they are differen"
},
{
"path": "docs/source/docs/providers/local/overview.rst",
"chars": 87,
"preview": "Local Provider Overview\n=======================\n\n.. toctree::\n\n instance-parameters\n"
},
{
"path": "docs/source/docs/providers/remote/instance-parameters.md",
"chars": 772,
"preview": "# Instance Parameters\n\nInstance parameters are part of the [configuration file], but for each provider they are differen"
},
{
"path": "docs/source/docs/providers/remote/overview.rst",
"chars": 89,
"preview": "Remote Provider Overview\n========================\n\n.. toctree::\n\n instance-parameters\n"
},
{
"path": "docs/source/docs/user-guide/configuration-file.md",
"chars": 9266,
"preview": "# Spotty Configuration File\n\nBy default, Spotty is looking for a `spotty.yaml` file in the root directory of the project"
},
{
"path": "docs/source/docs/user-guide/getting-started.md",
"chars": 2015,
"preview": "# Getting Started\n\n## Installation\n\nUse [pip](http://www.pip-installer.org/en/latest/) to install or upgrade Spotty:\n\n``"
},
{
"path": "docs/source/docs/user-guide/installation.md",
"chars": 602,
"preview": "# Installation\n\nUse [pip](http://www.pip-installer.org/en/latest/) to install or upgrade Spotty:\n\n```bash\npip install -U"
},
{
"path": "docs/source/index.rst",
"chars": 550,
"preview": ".. raw:: html\n :file: main.html\n\nWelcome to Spotty Documentation\n===============================\n\n.. toctree::\n :hid"
},
{
"path": "docs/source/main.html",
"chars": 4602,
"preview": "<style>\n #main-page-title {\n text-align: center;\n font-family: Lato,proxima-nova,Helvetica Neue,Arial,s"
},
{
"path": "setup.cfg",
"chars": 40,
"preview": "[metadata]\ndescription-file = README.md\n"
},
{
"path": "setup.py",
"chars": 2326,
"preview": "#!/usr/bin/env python\n\nimport os\nimport re\nfrom setuptools import setup, find_packages\n\n\ndef get_version():\n root_dir"
},
{
"path": "spotty/__init__.py",
"chars": 22,
"preview": "__version__ = '1.3.4'\n"
},
{
"path": "spotty/cli.py",
"chars": 1777,
"preview": "import argparse\nfrom typing import List, Type\nimport pkg_resources\nfrom spotty.commands.abstract_command import Abstract"
},
{
"path": "spotty/commands/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "spotty/commands/abstract_command.py",
"chars": 1069,
"preview": "from abc import ABC, abstractmethod\nfrom argparse import Namespace, ArgumentParser\nfrom spotty.commands.writers.abstract"
},
{
"path": "spotty/commands/abstract_config_command.py",
"chars": 2918,
"preview": "from abc import abstractmethod\nfrom typing import List\nfrom argparse import Namespace, ArgumentParser\nfrom spotty.config"
},
{
"path": "spotty/commands/abstract_provider_command.py",
"chars": 847,
"preview": "from abc import abstractmethod\nfrom argparse import Namespace, ArgumentParser\nimport sys\nfrom spotty.commands.abstract_c"
},
{
"path": "spotty/commands/aws.py",
"chars": 392,
"preview": "from spotty.commands.abstract_provider_command import AbstractProviderCommand\nfrom spotty.providers.aws.commands.clean_l"
},
{
"path": "spotty/commands/download.py",
"chars": 1816,
"preview": "from argparse import Namespace, ArgumentParser\nfrom spotty.commands.abstract_config_command import AbstractConfigCommand"
},
{
"path": "spotty/commands/exec.py",
"chars": 2665,
"preview": "import sys\nfrom argparse import ArgumentParser, Namespace\nfrom spotty.commands.abstract_config_command import AbstractCo"
},
{
"path": "spotty/commands/run.py",
"chars": 4209,
"preview": "from argparse import ArgumentParser, Namespace\nfrom spotty.commands.abstract_config_command import AbstractConfigCommand"
},
{
"path": "spotty/commands/sh.py",
"chars": 3081,
"preview": "from argparse import ArgumentParser, Namespace\nfrom spotty.commands.abstract_config_command import AbstractConfigCommand"
},
{
"path": "spotty/commands/start.py",
"chars": 2583,
"preview": "from argparse import Namespace, ArgumentParser\nfrom spotty.commands.abstract_config_command import AbstractConfigCommand"
},
{
"path": "spotty/commands/status.py",
"chars": 558,
"preview": "from argparse import Namespace\nfrom spotty.commands.abstract_config_command import AbstractConfigCommand\nfrom spotty.com"
},
{
"path": "spotty/commands/stop.py",
"chars": 1046,
"preview": "from argparse import Namespace\nfrom spotty.commands.abstract_config_command import AbstractConfigCommand\nfrom spotty.com"
},
{
"path": "spotty/commands/sync.py",
"chars": 1301,
"preview": "from argparse import Namespace, ArgumentParser\nfrom spotty.commands.abstract_config_command import AbstractConfigCommand"
},
{
"path": "spotty/commands/writers/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "spotty/commands/writers/abstract_output_writrer.py",
"chars": 711,
"preview": "from abc import ABC, abstractmethod\nfrom contextlib import contextmanager\n\n\nclass AbstractOutputWriter(ABC):\n\n def __"
},
{
"path": "spotty/commands/writers/null_output_writrer.py",
"chars": 225,
"preview": "from spotty.commands.writers.abstract_output_writrer import AbstractOutputWriter\n\n\nclass NullOutputWriter(AbstractOutput"
},
{
"path": "spotty/commands/writers/output_writrer.py",
"chars": 283,
"preview": "from spotty.commands.writers.abstract_output_writrer import AbstractOutputWriter\n\n\nclass OutputWriter(AbstractOutputWrit"
},
{
"path": "spotty/config/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "spotty/config/abstract_instance_config.py",
"chars": 7684,
"preview": "import os\nfrom abc import ABC, abstractmethod\nfrom collections import OrderedDict, namedtuple\nfrom typing import List\nfr"
},
{
"path": "spotty/config/abstract_instance_volume.py",
"chars": 1156,
"preview": "from abc import ABC, abstractmethod\n\n\nclass AbstractInstanceVolume(ABC):\n\n def __init__(self, volume_config: dict):\n "
},
{
"path": "spotty/config/config_utils.py",
"chars": 4376,
"preview": "import os\nfrom collections import namedtuple\nimport yaml\nfrom spotty.config.project_config import ProjectConfig\nfrom spo"
},
{
"path": "spotty/config/container_config.py",
"chars": 2300,
"preview": "from typing import List\nfrom spotty.config.validation import is_subdir\n\n\nPROJECT_VOLUME_MOUNT_NAME = '.project'\n\n\nclass "
},
{
"path": "spotty/config/host_path_volume.py",
"chars": 1151,
"preview": "import os\nfrom spotty.config.abstract_instance_volume import AbstractInstanceVolume\nfrom spotty.config.validation import"
},
{
"path": "spotty/config/project_config.py",
"chars": 844,
"preview": "from spotty.config.validation import validate_basic_config\n\n\nclass ProjectConfig(object):\n\n def __init__(self, config"
},
{
"path": "spotty/config/tmp_dir_volume.py",
"chars": 246,
"preview": "from spotty.config.host_path_volume import HostPathVolume\n\n\nclass TmpDirVolume(HostPathVolume):\n\n @property\n def t"
},
{
"path": "spotty/config/validation.py",
"chars": 8977,
"preview": "import os\nfrom typing import List\nfrom schema import Schema, And, Use, Optional, Or, Regex, Hook, SchemaError, SchemaFor"
},
{
"path": "spotty/configuration.py",
"chars": 500,
"preview": "import os\n\n\ndef get_spotty_config_dir():\n \"\"\"Spotty configuration directory.\"\"\"\n path = os.path.join(os.path.expan"
},
{
"path": "spotty/deployment/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "spotty/deployment/abstract_cloud_instance/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "spotty/deployment/abstract_cloud_instance/abstract_bucket_manager.py",
"chars": 482,
"preview": "from abc import ABC\nfrom spotty.deployment.abstract_cloud_instance.resources.abstract_bucket import AbstractBucket\n\n\ncla"
},
{
"path": "spotty/deployment/abstract_cloud_instance/abstract_cloud_instance_manager.py",
"chars": 7498,
"preview": "import logging\nfrom abc import ABC, abstractmethod\nfrom spotty.commands.writers.abstract_output_writrer import AbstractO"
},
{
"path": "spotty/deployment/abstract_cloud_instance/abstract_data_transfer.py",
"chars": 1939,
"preview": "from abc import ABC, abstractmethod\n\n\nclass AbstractDataTransfer(ABC):\n\n def __init__(self, local_project_dir: str, h"
},
{
"path": "spotty/deployment/abstract_cloud_instance/abstract_instance_deployment.py",
"chars": 1416,
"preview": "from abc import abstractmethod, ABC\nfrom spotty.commands.writers.abstract_output_writrer import AbstractOutputWriter\nfro"
},
{
"path": "spotty/deployment/abstract_cloud_instance/errors/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "spotty/deployment/abstract_cloud_instance/errors/bucket_not_found.py",
"chars": 124,
"preview": "class BucketNotFoundError(Exception):\n def __init__(self):\n super().__init__('Bucket for the project not found"
},
{
"path": "spotty/deployment/abstract_cloud_instance/file_structure.py",
"chars": 651,
"preview": "\"\"\"\nINSTANCE FILE STRUCTURE\n\"\"\"\n\n# a base temporary directory on an instance\nINSTANCE_SPOTTY_TMP_DIR = '/tmp/spotty'\n\n# "
},
{
"path": "spotty/deployment/abstract_cloud_instance/resources/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "spotty/deployment/abstract_cloud_instance/resources/abstract_bucket.py",
"chars": 118,
"preview": "from abc import ABC\n\n\nclass AbstractBucket(ABC):\n\n @property\n def name(self):\n raise NotImplementedError\n"
},
{
"path": "spotty/deployment/abstract_cloud_instance/resources/abstract_instance.py",
"chars": 653,
"preview": "from abc import ABC\n\n\nclass AbstractInstance(ABC):\n\n @property\n def public_ip_address(self):\n raise NotImpl"
},
{
"path": "spotty/deployment/abstract_docker_instance_manager.py",
"chars": 3258,
"preview": "import os\nfrom abc import ABC\nfrom spotty.commands.writers.abstract_output_writrer import AbstractOutputWriter\nfrom spot"
},
{
"path": "spotty/deployment/abstract_instance_manager.py",
"chars": 2979,
"preview": "import subprocess\nfrom abc import ABC, abstractmethod\nfrom spotty.commands.writers.abstract_output_writrer import Abstra"
},
{
"path": "spotty/deployment/abstract_ssh_instance_manager.py",
"chars": 1662,
"preview": "import logging\nimport os\nfrom abc import abstractmethod\nfrom spotty.deployment.utils.commands import get_ssh_command\nfro"
},
{
"path": "spotty/deployment/container/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "spotty/deployment/container/abstract_container_commands.py",
"chars": 599,
"preview": "from abc import ABC, abstractmethod\nfrom spotty.config.abstract_instance_config import AbstractInstanceConfig\n\n\nclass Ab"
},
{
"path": "spotty/deployment/container/abstract_container_script.py",
"chars": 466,
"preview": "from abc import ABC, abstractmethod\nfrom spotty.deployment.container.abstract_container_commands import AbstractContaine"
},
{
"path": "spotty/deployment/container/docker/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "spotty/deployment/container/docker/docker_commands.py",
"chars": 4278,
"preview": "import shlex\nfrom spotty.deployment.container.abstract_container_commands import AbstractContainerCommands\nfrom spotty.d"
},
{
"path": "spotty/deployment/container/docker/scripts/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "spotty/deployment/container/docker/scripts/abstract_docker_script.py",
"chars": 335,
"preview": "from abc import ABC\nfrom spotty.deployment.container.docker.docker_commands import DockerCommands\nfrom spotty.deployment"
},
{
"path": "spotty/deployment/container/docker/scripts/container_bash_script.py",
"chars": 866,
"preview": "import os\nimport chevron\nfrom spotty.deployment.utils.commands import get_bash_command\nfrom spotty.deployment.container."
},
{
"path": "spotty/deployment/container/docker/scripts/data/container_bash.sh.tpl",
"chars": 231,
"preview": "#!/bin/bash -e\n\nif [ -z \"$SPOTTY_CONTAINER_NAME\" ]; then\n echo -e \"\\nSPOTTY_CONTAINER_NAME environmental variable is no"
},
{
"path": "spotty/deployment/container/docker/scripts/data/start_container.sh.tpl",
"chars": 903,
"preview": "#!/usr/bin/env bash\n\n{{bash_flags}}\n\nif {{{is_created_cmd}}}; then\n printf 'Removing existing container... '\n {{{remov"
},
{
"path": "spotty/deployment/container/docker/scripts/data/stop_container.sh.tpl",
"chars": 173,
"preview": "#!/usr/bin/env bash\n\nset -e\n\nif {{{is_created_cmd}}}; then\n printf 'Removing the container... '\n {{{remove_cmd}}}\n ec"
},
{
"path": "spotty/deployment/container/docker/scripts/start_container_script.py",
"chars": 2224,
"preview": "import os\nimport time\nimport chevron\nfrom spotty.deployment.utils.commands import get_script_command\nfrom spotty.deploym"
},
{
"path": "spotty/deployment/container/docker/scripts/stop_container_script.py",
"chars": 624,
"preview": "import os\nimport chevron\nfrom spotty.deployment.container.docker.scripts.abstract_docker_script import AbstractDockerScr"
},
{
"path": "spotty/deployment/utils/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "spotty/deployment/utils/cli.py",
"chars": 224,
"preview": "import shlex\n\n\ndef shlex_join(split_command: list):\n \"\"\"Return a shell-escaped string from *split_command*.\n Copy-"
},
{
"path": "spotty/deployment/utils/commands.py",
"chars": 3668,
"preview": "import base64\nimport os\nimport shlex\nimport time\nfrom spotty.deployment.utils.cli import shlex_join\n\n\ndef get_bash_comma"
},
{
"path": "spotty/deployment/utils/print_info.py",
"chars": 1496,
"preview": "from typing import List\nfrom spotty.config.abstract_instance_config import VolumeMount\nfrom spotty.config.abstract_insta"
},
{
"path": "spotty/deployment/utils/user_scripts.py",
"chars": 1654,
"preview": "import re\nimport chevron\nfrom spotty.deployment.utils.commands import get_bash_command\n\n\ndef parse_script_parameters(scr"
},
{
"path": "spotty/errors/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "spotty/errors/instance_not_running.py",
"chars": 288,
"preview": "class InstanceNotRunningError(Exception):\n def __init__(self, instance_name: str):\n super().__init__('Instance"
},
{
"path": "spotty/errors/nothing_to_do.py",
"chars": 44,
"preview": "class NothingToDoError(Exception):\n pass\n"
},
{
"path": "spotty/providers/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "spotty/providers/aws/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "spotty/providers/aws/cfn_templates/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "spotty/providers/aws/cfn_templates/instance/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "spotty/providers/aws/cfn_templates/instance/data/files/tmux.conf",
"chars": 21,
"preview": "bind-key x kill-pane\n"
},
{
"path": "spotty/providers/aws/cfn_templates/instance/data/startup_scripts/01_prepare_instance.sh",
"chars": 722,
"preview": "#!/bin/bash -xe\n\ncfn-signal -e 0 --stack ${AWS::StackName} --region ${AWS::Region} --resource PreparingInstanceSignal\n\n#"
},
{
"path": "spotty/providers/aws/cfn_templates/instance/data/startup_scripts/02_mount_volumes.sh",
"chars": 1098,
"preview": "#!/bin/bash -xe\n\ncfn-signal -e 0 --stack ${AWS::StackName} --region ${AWS::Region} --resource MountingVolumesSignal\n\n# m"
},
{
"path": "spotty/providers/aws/cfn_templates/instance/data/startup_scripts/03_set_docker_root.sh",
"chars": 395,
"preview": "#!/bin/bash -xe\n\ncfn-signal -e 0 --stack ${AWS::StackName} --region ${AWS::Region} --resource SettingDockerRootSignal\n\n#"
},
{
"path": "spotty/providers/aws/cfn_templates/instance/data/startup_scripts/04_sync_project.sh",
"chars": 453,
"preview": "#!/bin/bash -xe\n\ncfn-signal -e 0 --stack ${AWS::StackName} --region ${AWS::Region} --resource SyncingProjectSignal\n\n# cr"
},
{
"path": "spotty/providers/aws/cfn_templates/instance/data/startup_scripts/05_run_instance_startup_commands.sh",
"chars": 208,
"preview": "#!/bin/bash -xe\n\ncfn-signal -e 0 --stack ${AWS::StackName} --region ${AWS::Region} --resource RunningInstanceStartupComm"
},
{
"path": "spotty/providers/aws/cfn_templates/instance/data/startup_scripts/user_data.sh",
"chars": 1028,
"preview": "#!/bin/bash -x\n\ncd /root || exit 1\n\n# install CloudFormation tools if they are not installed yet\nif [ ! -e /usr/local/bi"
},
{
"path": "spotty/providers/aws/cfn_templates/instance/data/template.yaml",
"chars": 4277,
"preview": "Description: Spotty EC2 Instance\nParameters:\n VpcId:\n Description: VPC ID\n Type: AWS::EC2::VPC::Id\n InstanceProf"
},
{
"path": "spotty/providers/aws/cfn_templates/instance/start_container_script.py",
"chars": 924,
"preview": "from spotty.deployment.container.docker.scripts.start_container_script import StartContainerScript\n\n\nclass StartContaine"
},
{
"path": "spotty/providers/aws/cfn_templates/instance/template.py",
"chars": 16802,
"preview": "from typing import List\nimport os\nimport chevron\nimport yaml\nfrom cfn_tools import CfnYamlLoader, CfnYamlDumper\nfrom spo"
},
{
"path": "spotty/providers/aws/cfn_templates/instance_profile/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "spotty/providers/aws/cfn_templates/instance_profile/data/template.yaml",
"chars": 1179,
"preview": "Description: Spotty EC2 Instance Profile\nResources:\n InstanceProfile:\n Type: AWS::IAM::InstanceProfile\n Propertie"
},
{
"path": "spotty/providers/aws/cfn_templates/instance_profile/template.py",
"chars": 455,
"preview": "import os\nimport chevron\n\n\ndef prepare_instance_profile_template(managed_policy_arns: list):\n with open(os.path.join("
},
{
"path": "spotty/providers/aws/commands/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "spotty/providers/aws/commands/clean_logs.py",
"chars": 2305,
"preview": "from argparse import ArgumentParser, Namespace\nfrom time import time\nimport boto3\nfrom spotty.commands.abstract_command "
},
{
"path": "spotty/providers/aws/commands/spot_prices.py",
"chars": 1730,
"preview": "from argparse import ArgumentParser, Namespace\nimport boto3\nfrom spotty.commands.abstract_command import AbstractCommand"
},
{
"path": "spotty/providers/aws/config/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "spotty/providers/aws/config/ebs_volume.py",
"chars": 2108,
"preview": "from spotty.config.abstract_instance_volume import AbstractInstanceVolume\nfrom spotty.providers.aws.config.validation im"
},
{
"path": "spotty/providers/aws/config/instance_config.py",
"chars": 2335,
"preview": "from typing import List\nfrom spotty.config.abstract_instance_config import AbstractInstanceConfig, VolumeMount\nfrom spot"
},
{
"path": "spotty/providers/aws/config/validation.py",
"chars": 4350,
"preview": "import os\nfrom schema import Schema, Optional, And, Regex, Or, Use\nfrom spotty.config.validation import validate_config,"
},
{
"path": "spotty/providers/aws/data_transfer.py",
"chars": 3573,
"preview": "import logging\nimport subprocess\nfrom spotty.deployment.abstract_cloud_instance.abstract_data_transfer import AbstractDa"
},
{
"path": "spotty/providers/aws/deletion_policies.py",
"chars": 4433,
"preview": "from typing import List\nfrom spotty.commands.writers.abstract_output_writrer import AbstractOutputWriter\nfrom spotty.con"
},
{
"path": "spotty/providers/aws/errors/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "spotty/providers/aws/errors/volume_not_found.py",
"chars": 139,
"preview": "class VolumeNotFoundError(Exception):\n def __init__(self, volume_name):\n super().__init__('Volume \"%s\" not fou"
},
{
"path": "spotty/providers/aws/helpers/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "spotty/providers/aws/helpers/ami.py",
"chars": 1685,
"preview": "from spotty.providers.aws.config.instance_config import DEFAULT_AMI_NAME\nfrom spotty.providers.aws.resources.image impor"
},
{
"path": "spotty/providers/aws/helpers/availability_zone.py",
"chars": 1601,
"preview": "from typing import List\nfrom spotty.config.abstract_instance_volume import AbstractInstanceVolume\nfrom spotty.providers."
},
{
"path": "spotty/providers/aws/helpers/instance_prices.py",
"chars": 3952,
"preview": "import datetime\nimport json\nimport logging\nimport boto3\nfrom pkg_resources import resource_filename\n\n\ndef get_spot_price"
},
{
"path": "spotty/providers/aws/helpers/logs.py",
"chars": 883,
"preview": "import os\nimport subprocess\nimport tempfile\nfrom glob import glob\nfrom spotty.providers.aws.helpers.s3_sync import get_s"
},
{
"path": "spotty/providers/aws/helpers/s3_sync.py",
"chars": 1544,
"preview": "from shutil import which\nfrom spotty.deployment.utils.cli import shlex_join\n\n\ndef check_aws_installed():\n \"\"\"Checks t"
},
{
"path": "spotty/providers/aws/helpers/subnet.py",
"chars": 2167,
"preview": "from spotty.providers.aws.resources.subnet import Subnet\n\n\ndef check_az_and_subnet(ec2, region: str, availability_zone: "
},
{
"path": "spotty/providers/aws/helpers/vpc.py",
"chars": 492,
"preview": "from spotty.providers.aws.resources.subnet import Subnet\nfrom spotty.providers.aws.resources.vpc import Vpc\n\n\ndef get_vp"
},
{
"path": "spotty/providers/aws/instance_deployment.py",
"chars": 6710,
"preview": "import boto3\nfrom spotty.commands.writers.abstract_output_writrer import AbstractOutputWriter\nfrom spotty.deployment.abs"
},
{
"path": "spotty/providers/aws/instance_manager.py",
"chars": 2997,
"preview": "from spotty.errors.instance_not_running import InstanceNotRunningError\nfrom spotty.deployment.abstract_cloud_instance.ab"
},
{
"path": "spotty/providers/aws/resource_managers/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "spotty/providers/aws/resource_managers/bucket_manager.py",
"chars": 1780,
"preview": "import boto3\nimport re\nfrom spotty.deployment.abstract_cloud_instance.abstract_bucket_manager import AbstractBucketManag"
},
{
"path": "spotty/providers/aws/resource_managers/instance_profile_stack_manager.py",
"chars": 3482,
"preview": "import boto3\nfrom botocore.exceptions import ClientError, WaiterError\nfrom spotty.commands.writers.abstract_output_writr"
},
{
"path": "spotty/providers/aws/resource_managers/instance_stack_manager.py",
"chars": 4709,
"preview": "import boto3\nfrom spotty.commands.writers.abstract_output_writrer import AbstractOutputWriter\nfrom spotty.providers.aws."
},
{
"path": "spotty/providers/aws/resource_managers/key_pair_manager.py",
"chars": 2144,
"preview": "import os\nfrom spotty.configuration import get_spotty_keys_dir\nfrom spotty.providers.instance_manager_factory import PRO"
},
{
"path": "spotty/providers/aws/resources/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "spotty/providers/aws/resources/bucket.py",
"chars": 266,
"preview": "from spotty.deployment.abstract_cloud_instance.resources.abstract_bucket import AbstractBucket\n\n\nclass Bucket(AbstractBu"
},
{
"path": "spotty/providers/aws/resources/image.py",
"chars": 1355,
"preview": "class Image(object):\n\n def __init__(self, ec2, ami_info):\n self._ec2 = ec2\n self._ami_info = ami_info\n\n"
},
{
"path": "spotty/providers/aws/resources/instance.py",
"chars": 2855,
"preview": "from datetime import datetime\nfrom spotty.deployment.abstract_cloud_instance.resources.abstract_instance import Abstract"
},
{
"path": "spotty/providers/aws/resources/snapshot.py",
"chars": 1585,
"preview": "import time\n\n\nclass Snapshot(object):\n\n def __init__(self, ec2, snapshot_info):\n self._ec2 = ec2\n self."
},
{
"path": "spotty/providers/aws/resources/stack.py",
"chars": 4891,
"preview": "from collections import namedtuple\nfrom time import sleep\nfrom typing import List, Dict\nfrom botocore.exceptions import "
},
{
"path": "spotty/providers/aws/resources/subnet.py",
"chars": 917,
"preview": "class Subnet(object):\n\n def __init__(self, ec2, subnet_info):\n self._ec2 = ec2\n self._subnet_info = sub"
},
{
"path": "spotty/providers/aws/resources/volume.py",
"chars": 1825,
"preview": "from spotty.providers.aws.resources.snapshot import Snapshot\n\n\nclass Volume(object):\n\n def __init__(self, ec2, volume"
},
{
"path": "spotty/providers/aws/resources/vpc.py",
"chars": 469,
"preview": "class Vpc(object):\n\n def __init__(self, ec2, vpc_info):\n self._ec2 = ec2\n self._vpc_info = vpc_info\n\n "
},
{
"path": "spotty/providers/gcp/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "spotty/providers/gcp/config/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "spotty/providers/gcp/config/disk_volume.py",
"chars": 2008,
"preview": "from spotty.config.abstract_instance_volume import AbstractInstanceVolume\nfrom spotty.providers.gcp.config.validation im"
},
{
"path": "spotty/providers/gcp/config/image_uri.py",
"chars": 738,
"preview": "import re\n\n\nIMAGE_URI_REGEX = '^(?:(?:https://compute.googleapis.com/compute/v1/)?projects/([a-z](?:[-a-z0-9]*[a-z0-9])?"
},
{
"path": "spotty/providers/gcp/config/instance_config.py",
"chars": 2192,
"preview": "from typing import List\nfrom spotty.config.abstract_instance_config import AbstractInstanceConfig, VolumeMount\nfrom spot"
},
{
"path": "spotty/providers/gcp/config/validation.py",
"chars": 2972,
"preview": "import os\nfrom schema import Schema, Optional, And, Regex, Or, Use\nfrom spotty.config.validation import validate_config,"
},
{
"path": "spotty/providers/gcp/data_transfer.py",
"chars": 2237,
"preview": "import logging\nimport subprocess\nfrom spotty.deployment.abstract_cloud_instance.abstract_data_transfer import AbstractDa"
},
{
"path": "spotty/providers/gcp/dm_templates/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "spotty/providers/gcp/dm_templates/instance/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "spotty/providers/gcp/dm_templates/instance/data/startup_script.sh.tpl",
"chars": 763,
"preview": "#!/usr/bin/env bash\n\nset -x\n\nmkdir -p \"{{INSTANCE_STARTUP_SCRIPTS_DIR}}\"\n\n# create startup scripts\n{{#STARTUP_SCRIPTS}}\n"
},
{
"path": "spotty/providers/gcp/dm_templates/instance/data/startup_scripts/01_prepare_instance.sh",
"chars": 994,
"preview": "#!/bin/bash -xe\n\n# install jq\napt-get install -y jq\n\n# create tmux config\necho \"bind-key x kill-pane\" > /home/{{SSH_USER"
},
{
"path": "spotty/providers/gcp/dm_templates/instance/data/startup_scripts/02_mount_volumes.sh",
"chars": 494,
"preview": "#!/bin/bash -xe\n\nDEVICE_NAMES=({{{DISK_DEVICE_NAMES}}})\nMOUNT_DIRS=({{{DISK_MOUNT_DIRS}}})\n\nfor i in ${!DEVICE_NAMES[*]}"
},
{
"path": "spotty/providers/gcp/dm_templates/instance/data/startup_scripts/03_set_docker_root.sh",
"chars": 289,
"preview": "#!/bin/bash -xe\n\n# change docker data root directory\nif [ -n \"{{DOCKER_DATA_ROOT_DIR}}\" ]; then\n jq '. + { \"data-root\":"
},
{
"path": "spotty/providers/gcp/dm_templates/instance/data/startup_scripts/04_sync_project.sh",
"chars": 181,
"preview": "#!/bin/bash -xe\n\n# create a project directory\nif [ -n \"{{HOST_PROJECT_DIR}}\" ]; then\n mkdir -p \"{{HOST_PROJECT_DIR}}\"\n "
},
{
"path": "spotty/providers/gcp/dm_templates/instance/data/startup_scripts/05_run_instance_startup_commands.sh",
"chars": 256,
"preview": "#!/bin/bash -xe\n\nmkdir -p \"{{INSTANCE_STARTUP_SCRIPTS_DIR}}\"\ncat > \"{{INSTANCE_STARTUP_SCRIPTS_DIR}}/instance_startup_co"
},
{
"path": "spotty/providers/gcp/dm_templates/instance/data/template.yaml",
"chars": 2538,
"preview": "resources:\n - name: {{MACHINE_NAME}}\n type: compute.v1.instance\n properties:\n zone: {{ZONE}}\n machineTy"
},
{
"path": "spotty/providers/gcp/dm_templates/instance/instance_template.py",
"chars": 6822,
"preview": "import os\nfrom typing import List\nimport chevron\nfrom spotty.commands.writers.abstract_output_writrer import AbstractOut"
},
{
"path": "spotty/providers/gcp/errors/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "spotty/providers/gcp/errors/image_not_found.py",
"chars": 279,
"preview": "class ImageNotFoundError(Exception):\n def __init__(self, image_name):\n super().__init__('The image \"%s\" was no"
},
{
"path": "spotty/providers/gcp/helpers/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "spotty/providers/gcp/helpers/ce_client.py",
"chars": 4826,
"preview": "from collections import OrderedDict\nfrom time import sleep\nimport googleapiclient.discovery\n\n\nclass CEClient(object):\n "
},
{
"path": "spotty/providers/gcp/helpers/deployment.py",
"chars": 3794,
"preview": "import logging\nfrom collections import OrderedDict\nfrom time import sleep\nfrom httplib2 import ServerNotFoundError\nfrom "
},
{
"path": "spotty/providers/gcp/helpers/dm_client.py",
"chars": 2154,
"preview": "import json\nimport googleapiclient.discovery\nfrom googleapiclient.errors import HttpError\n\n\nclass DMClient(object):\n "
},
{
"path": "spotty/providers/gcp/helpers/dm_resource.py",
"chars": 4509,
"preview": "from spotty.providers.gcp.helpers.dm_client import DMClient\n\n\nclass DMResource(object):\n\n def __init__(self, dm: DMCl"
},
{
"path": "spotty/providers/gcp/helpers/gcp_credentials.py",
"chars": 411,
"preview": "from google.auth import default\n\n\nclass GcpCredentials(object):\n def __init__(self):\n credentials, effective_p"
},
{
"path": "spotty/providers/gcp/helpers/gs_client.py",
"chars": 721,
"preview": "from typing import List\nfrom google.cloud import storage\nfrom google.cloud.storage import Bucket\n\n\nclass GSClient(object"
},
{
"path": "spotty/providers/gcp/helpers/gsutil_rsync.py",
"chars": 1198,
"preview": "import fnmatch\nfrom shutil import which\nimport os\nfrom typing import List\n\nfrom spotty.deployment.utils.cli import shlex"
},
{
"path": "spotty/providers/gcp/helpers/image.py",
"chars": 1502,
"preview": "from spotty.providers.gcp.config.instance_config import DEFAULT_IMAGE_NAME\nfrom spotty.providers.gcp.helpers.ce_client i"
},
{
"path": "spotty/providers/gcp/helpers/rtc_client.py",
"chars": 933,
"preview": "import googleapiclient.discovery\n\n\nclass RtcClient(object):\n\n def __init__(self, project_id: str, zone: str):\n "
},
{
"path": "spotty/providers/gcp/helpers/volumes.py",
"chars": 2956,
"preview": "from typing import List\nfrom spotty.commands.writers.abstract_output_writrer import AbstractOutputWriter\nfrom spotty.con"
},
{
"path": "spotty/providers/gcp/instance_deployment.py",
"chars": 4802,
"preview": "from spotty.commands.writers.abstract_output_writrer import AbstractOutputWriter\nfrom spotty.deployment.abstract_cloud_i"
},
{
"path": "spotty/providers/gcp/instance_manager.py",
"chars": 2723,
"preview": "from spotty.commands.writers.abstract_output_writrer import AbstractOutputWriter\nfrom spotty.errors.instance_not_running"
},
{
"path": "spotty/providers/gcp/resource_managers/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "spotty/providers/gcp/resource_managers/bucket_manager.py",
"chars": 1436,
"preview": "import re\nfrom spotty.deployment.abstract_cloud_instance.abstract_bucket_manager import AbstractBucketManager\nfrom spott"
},
{
"path": "spotty/providers/gcp/resource_managers/instance_stack_manager.py",
"chars": 3276,
"preview": "from collections import OrderedDict\nfrom spotty.commands.writers.abstract_output_writrer import AbstractOutputWriter\nfro"
},
{
"path": "spotty/providers/gcp/resource_managers/ssh_key_manager.py",
"chars": 1886,
"preview": "import os\nimport subprocess\nfrom spotty.configuration import get_spotty_keys_dir\nfrom shutil import which\nfrom spotty.pr"
},
{
"path": "spotty/providers/gcp/resources/__init__.py",
"chars": 0,
"preview": ""
}
]
// ... and 50 more files (download for full content)
About this extraction
This page contains the full source code of the spotty-cloud/spotty GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 250 files (358.8 KB), approximately 88.7k tokens, and a symbol index with 693 extracted functions, classes, methods, constants, and types. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.