Showing preview only (664K chars total). Download the full file or copy to clipboard to get everything.
Repository: apache/incubator-liminal
Branch: master
Commit: aee9827f3b33
Files: 266
Total size: 596.4 KB
Directory structure:
gitextract_29sbg3vw/
├── .asf.yaml
├── .bumpversion.cfg
├── .github/
│ ├── PULL_REQUEST_TEMPLATE.md
│ └── workflows/
│ ├── pre_commits.yml
│ ├── unittests.yml
│ └── versioning.yml
├── .gitignore
├── .pre-commit-config.yaml
├── CONTRIBUTING.md
├── DISCLAIMER-WIP
├── LICENSE
├── MANIFEST.in
├── NOTICE
├── README.md
├── RETIRED.txt
├── dev/
│ ├── LICENSE.txt
│ ├── RELEASE.md
│ └── sign.sh
├── docs/
│ ├── Makefile
│ ├── README.md
│ ├── architecture.md
│ ├── conf.py
│ ├── getting-started/
│ │ ├── hello_world.md
│ │ ├── iris_classification.md
│ │ └── spark_app_demo.md
│ ├── index.rst
│ ├── liminal/
│ │ ├── README.md
│ │ ├── advanced.liminal.yml.md
│ │ ├── executors/
│ │ │ ├── README.md
│ │ │ ├── emr.md
│ │ │ ├── index.rst
│ │ │ └── kubernetes.md
│ │ ├── extensibility.md
│ │ ├── images/
│ │ │ ├── README.md
│ │ │ ├── index.rst
│ │ │ ├── python.md
│ │ │ └── python_server.md
│ │ ├── index.rst
│ │ ├── kubernetes/
│ │ │ └── secret_util.md
│ │ ├── liminal.yml.md
│ │ ├── metrics_backends/
│ │ │ ├── README.md
│ │ │ ├── aws_cloudwatch.md
│ │ │ └── index.rst
│ │ ├── monitoring.md
│ │ ├── pipelines.md
│ │ ├── services.md
│ │ └── tasks/
│ │ ├── README.md
│ │ ├── create_cloudformation_stack.md
│ │ ├── delete_cloudformation_stack.md
│ │ ├── index.rst
│ │ ├── python.md
│ │ └── spark.md
│ ├── make.bat
│ └── source/
│ ├── How_to_install_liminal_in_airflow_on_kubernetes.md
│ └── liminal_aws_deploy.sh
├── examples/
│ ├── aws-ml-app-demo/
│ │ ├── __init__.py
│ │ ├── liminal.yml
│ │ ├── manifests/
│ │ │ └── aws-ml-app-demo.yaml
│ │ ├── model_store.py
│ │ ├── requirements.txt
│ │ ├── serving.py
│ │ └── training.py
│ ├── extensibility/
│ │ ├── __init__.py
│ │ ├── executors/
│ │ │ ├── __init__.py
│ │ │ └── custom_executor.py
│ │ ├── images/
│ │ │ ├── __init__.py
│ │ │ ├── custom_image.py
│ │ │ └── custom_image_builder/
│ │ │ ├── Dockerfile
│ │ │ └── __init__.py
│ │ ├── liminal.yml
│ │ └── tasks/
│ │ ├── __init__.py
│ │ └── custom_task.py
│ ├── liminal-getting-started/
│ │ ├── __init__.py
│ │ ├── hello_world.json
│ │ ├── helloworld/
│ │ │ ├── __init__.py
│ │ │ └── hello_world.py
│ │ ├── liminal.yml
│ │ ├── myserver/
│ │ │ ├── __init__.py
│ │ │ └── my_server.py
│ │ ├── pip.conf
│ │ └── requirements.txt
│ ├── sagemaker_example/
│ │ ├── README.md
│ │ ├── archetype/
│ │ │ └── liminal.yaml
│ │ ├── data_preparation/
│ │ │ ├── __init__.py
│ │ │ ├── data_preparation.py
│ │ │ └── data_uploader.py
│ │ ├── data_train/
│ │ │ ├── __init__.py
│ │ │ └── train.py
│ │ ├── inference.py
│ │ ├── liminal.yaml
│ │ ├── liminal_sm.py
│ │ ├── requirements.txt
│ │ └── sm_ops.py
│ └── spark-app-demo/
│ └── k8s/
│ ├── __init__.py
│ ├── archetype/
│ │ └── liminal.yml
│ ├── data/
│ │ └── iris.csv
│ ├── data_cleanup.py
│ ├── liminal.yml
│ ├── manifests/
│ │ └── spark-app-demo.yaml
│ ├── model_store.py
│ ├── requirements.txt
│ ├── serving.py
│ └── training.py
├── install.sh
├── liminal/
│ ├── __init__.py
│ ├── build/
│ │ ├── __init__.py
│ │ ├── image/
│ │ │ ├── __init__.py
│ │ │ ├── python/
│ │ │ │ ├── Dockerfile
│ │ │ │ ├── __init__.py
│ │ │ │ └── python.py
│ │ │ ├── python_server/
│ │ │ │ ├── Dockerfile
│ │ │ │ ├── __init__.py
│ │ │ │ ├── liminal_python_server.py
│ │ │ │ ├── python_server.py
│ │ │ │ └── python_server_requirements.txt
│ │ │ └── spark/
│ │ │ ├── Dockerfile
│ │ │ ├── __init__.py
│ │ │ └── spark.py
│ │ ├── image_builder.py
│ │ ├── liminal_apps_builder.py
│ │ └── python.py
│ ├── core/
│ │ ├── __init__.py
│ │ ├── config/
│ │ │ ├── __init__.py
│ │ │ ├── config.py
│ │ │ └── defaults/
│ │ │ ├── __init__.py
│ │ │ ├── base/
│ │ │ │ ├── __init__.py
│ │ │ │ └── liminal.yml
│ │ │ └── default_configs.py
│ │ ├── environment.py
│ │ └── util/
│ │ ├── __init__.py
│ │ ├── class_util.py
│ │ ├── dict_util.py
│ │ ├── env_util.py
│ │ ├── extensible.py
│ │ └── files_util.py
│ ├── docker/
│ │ └── __init__.py
│ ├── kubernetes/
│ │ ├── __init__.py
│ │ ├── secret_util.py
│ │ └── volume_util.py
│ ├── logging/
│ │ ├── __init__.py
│ │ └── logging_setup.py
│ ├── monitoring/
│ │ └── __init__.py
│ ├── runners/
│ │ ├── __init__.py
│ │ └── airflow/
│ │ ├── __init__.py
│ │ ├── config/
│ │ │ ├── __init__.py
│ │ │ └── standalone_variable_backend.py
│ │ ├── dag/
│ │ │ ├── __init__.py
│ │ │ ├── liminal_dags.py
│ │ │ └── liminal_register_dags.py
│ │ ├── executors/
│ │ │ ├── __init__.py
│ │ │ ├── airflow.py
│ │ │ ├── emr.py
│ │ │ └── kubernetes.py
│ │ ├── model/
│ │ │ ├── __init__.py
│ │ │ ├── executor.py
│ │ │ └── task.py
│ │ ├── operators/
│ │ │ ├── __init__.py
│ │ │ ├── cloudformation.py
│ │ │ ├── job_status_operator.py
│ │ │ └── operator_with_variable_resolving.py
│ │ └── tasks/
│ │ ├── __init__.py
│ │ ├── airflow.py
│ │ ├── containerable.py
│ │ ├── create_cloudformation_stack.py
│ │ ├── delete_cloudformation_stack.py
│ │ ├── hadoop.py
│ │ ├── job_end.py
│ │ ├── job_start.py
│ │ ├── python.py
│ │ ├── spark.py
│ │ └── sql.py
│ ├── settings.py
│ └── sql/
│ └── __init__.py
├── liminal-arch.md
├── pyproject.toml
├── requirements.txt
├── run_tests.sh
├── scripts/
│ ├── Dockerfile-airflow
│ ├── __init__.py
│ ├── docker-compose.yml
│ ├── liminal
│ ├── requirements-airflow.txt
│ └── webserver_config.py
├── setup.py
├── tests/
│ ├── __init__.py
│ ├── liminal/
│ │ ├── __init__.py
│ │ ├── core/
│ │ │ ├── __init__.py
│ │ │ ├── config/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── defaults/
│ │ │ │ │ ├── __init__.py
│ │ │ │ │ ├── test_apply_variables_substitution.py
│ │ │ │ │ ├── test_defaults_pipeline_config.py
│ │ │ │ │ ├── test_defaults_service_config.py
│ │ │ │ │ └── test_defaults_tasks_config.py
│ │ │ │ └── test_config.py
│ │ │ └── util/
│ │ │ ├── __init__.py
│ │ │ └── test_dict_utils.py
│ │ └── kubernetes/
│ │ ├── __init__.py
│ │ └── test_volume_util.py
│ ├── runners/
│ │ ├── __init__.py
│ │ ├── airflow/
│ │ │ ├── __init__.py
│ │ │ ├── build/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── http/
│ │ │ │ │ ├── __init__.py
│ │ │ │ │ └── python/
│ │ │ │ │ ├── __init__.py
│ │ │ │ │ └── test_python_server_image_builder.py
│ │ │ │ ├── python/
│ │ │ │ │ ├── __init__.py
│ │ │ │ │ └── test_python_image_builder.py
│ │ │ │ ├── spark/
│ │ │ │ │ ├── __init__.py
│ │ │ │ │ └── test_spark_image_builder.py
│ │ │ │ └── test_liminal_apps_builder.py
│ │ │ ├── compiler/
│ │ │ │ └── __init__.py
│ │ │ ├── dag/
│ │ │ │ ├── __init__.py
│ │ │ │ └── test_liminal_dags.py
│ │ │ ├── executors/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── test_airflow_executor.py
│ │ │ │ └── test_emr.py
│ │ │ ├── liminal/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── liminal.yml
│ │ │ │ ├── myserver/
│ │ │ │ │ ├── __init__.py
│ │ │ │ │ └── my_server.py
│ │ │ │ ├── pip.conf
│ │ │ │ ├── requirements.txt
│ │ │ │ ├── write_inputs/
│ │ │ │ │ ├── __init__.py
│ │ │ │ │ └── write_inputs.py
│ │ │ │ └── write_outputs/
│ │ │ │ ├── __init__.py
│ │ │ │ └── write_outputs.py
│ │ │ ├── operators/
│ │ │ │ ├── __init__.py
│ │ │ │ └── test_operator_with_variable_resolving.py
│ │ │ └── tasks/
│ │ │ ├── __init__.py
│ │ │ ├── test_create_cloudformation_stack.py
│ │ │ ├── test_delete_cloudformation_stack.py
│ │ │ ├── test_job_end.py
│ │ │ ├── test_job_start.py
│ │ │ ├── test_python.py
│ │ │ └── test_spark_task.py
│ │ └── apps/
│ │ ├── __init__.py
│ │ ├── test/
│ │ │ └── liminal.yml
│ │ ├── test_app/
│ │ │ ├── __init__.py
│ │ │ ├── defaults/
│ │ │ │ ├── __init__.py
│ │ │ │ └── liminal.yml
│ │ │ ├── extra/
│ │ │ │ ├── __init__.py
│ │ │ │ └── liminal.yml
│ │ │ ├── helloworld/
│ │ │ │ ├── __init__.py
│ │ │ │ └── hellp_world.py
│ │ │ ├── liminal.yml
│ │ │ └── my_server/
│ │ │ ├── __init__.py
│ │ │ └── my_server.py
│ │ └── test_spark_app/
│ │ ├── __init__.py
│ │ ├── liminal.yml
│ │ └── wordcount/
│ │ ├── __init__.py
│ │ ├── requirements.txt
│ │ ├── wordcount.py
│ │ └── words.txt
│ ├── test_licenses.py
│ └── util/
│ ├── __init__.py
│ ├── dag_test_utils.py
│ ├── test_class_utils.py
│ └── test_pkg_1/
│ ├── __init__.py
│ ├── test_clazz_base.py
│ └── test_pkg_1_1/
│ ├── __init__.py
│ ├── test_clazz_child_1.py
│ ├── test_clazz_child_2.py
│ ├── test_pkg_1_1_1/
│ │ ├── __init__.py
│ │ └── test_clazz_leaf_1.py
│ └── test_pkg_1_1_2/
│ ├── __init__.py
│ └── test_clazz_leaf_2.py
└── yamllint-config.yml
================================================
FILE CONTENTS
================================================
================================================
FILE: .asf.yaml
================================================
---
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
github:
description: Apache Liminals goal is to operationalise the machine learning process, allowing data scientists to quickly transition from a successful
experiment to an automated pipeline of model training, validation, deployment and inference in production. Liminal provides a Domain Specific Language
to build ML workflows on top of Apache Airflow.
homepage: https://liminal.apache.org
labels:
- data-science
- big-data
- machine-learning
- airflow
- ai
- ml
- workflows
notifications:
commits: commits@liminal.apache.org
issues: issues@liminal.apache.org
pullrequests: dev@liminal.apache.org
jira_options: link label worklog
================================================
FILE: .bumpversion.cfg
================================================
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
[bumpversion]
current_version = 0.0.1rc6
parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\.?\w*)(\-(?P<release>[a-z]+)(?P<build>\d+))?
serialize =
{major}.{minor}.{patch}-dev{build}
{major}.{minor}.{patch}
[bumpversion:part:release]
optional_value = prod
first_value = dev
values =
dev
prod
================================================
FILE: .github/PULL_REQUEST_TEMPLATE.md
================================================
## Changes Title (replace this with a logical title for your changes)
*Issue #, if available:*
### *Description of changes:*
Replace this with the PR description (mention the changes you have made, why you have made them, provide some background and any references to the provider documentation if needed, etc.).
For more information on contributing, please see [Contributing](../CONTRIBUTING.md)
section of our documentation.
### Status
Replace this: describe the PR status. Examples:
- WIP - work in progress
- DONE - ready for review
### Checklist (tick everything that applies)
- [ ] [PreCommitChecks - Code linting](../CONTRIBUTING.md#InstallRunPreCommit) (required)
- [ ] [Tests](../CONTRIBUTING.md#RunningTests)
================================================
FILE: .github/workflows/pre_commits.yml
================================================
---
name: PreCommitChecks
on:
push:
branches:
- master
# - '!release*'
pull_request:
branches:
- master
# - '!release*'
jobs:
linting:
name: Run pre-commit hooks on py3.6
runs-on: ubuntu-20.04
steps:
#----------------------------------------------
# Checkout, SetUp Python, Load Cache and Run PreCommitChecks
#----------------------------------------------
- uses: actions/checkout@v2
with:
fetch-depth: 0
# Install Python
- uses: actions/setup-python@v2
with:
python-version: 3.6
- name: Check Python ${{ matrix.python-version }} version
run: python -V
- uses: actions/cache@v2
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip
restore-keys: ${{ runner.os }}-pip
- name: Install python requirements
run: |
python -m pip install --upgrade pip
pip install -r requirements.txt
pip install pre-commit
# load cached venv if cache exists
- name: Load cached venv
id: cached-poetry-dependencies
uses: actions/cache@v2
with:
path: .venv
key: venv-${{ runner.os }}-${{ hashFiles('**/requirements.txt') }}
- run: pre-commit install
- name: Run pre-commit hooks on all the files
run: pre-commit run --all-files --show-diff-on-failure --color always --verbose
# - uses: pre-commit/action@v2.0.3
# with:
# token: ${{ secrets.GITHUB_TOKEN }}
================================================
FILE: .github/workflows/unittests.yml
================================================
---
name: Running unittest
on:
# Trigger the workflow on push or pull request, but only for the master branch
push:
branches:
- master
pull_request:
branches:
- master
# Trigger the workflow on cron schedule
schedule:
- cron: '7 0 * * *'
jobs:
unittest:
runs-on: ubuntu-20.04
timeout-minutes: 20
steps:
- name: Checkout
uses: actions/checkout@v1
- name: Setup Minikube
uses: manusa/actions-setup-minikube@v2.7.1
with:
minikube version: 'v1.26.1'
kubernetes version: 'v1.25.0'
github token: ${{ secrets.GITHUB_TOKEN }}
- name: Set up Python
uses: actions/setup-python@v1
with:
python-version: '3.6'
- name: Install python requirements
run: |
python -m pip install --upgrade pip
pip install -r requirements.txt
- name: Run unittest - runners
run: ./run_tests.sh
approve:
needs: unittest
runs-on: ubuntu-20.04
steps:
- run: | # approve the pull request
curl --request POST \
--url https://api.github.com/repos/${{github.repository}}/pulls/${{github.event.number}}/reviews \
--header 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' \
--header 'content-type: application/json' \
-d '{"event":"APPROVE"}'
================================================
FILE: .github/workflows/versioning.yml
================================================
---
name: Versioning RC
on:
workflow_run:
workflows: [Running unittest]
branches: [master]
types:
- completed
jobs:
versioning-auto-commit:
if: ${{ github.event.workflow_run.conclusion == 'success' }}
name: Publish package
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v1
- name: Set up Python
uses: actions/setup-python@v1
with:
python-version: '3.6'
- name: Install dependencies for setup
run: |
python -m pip install --upgrade pip
pip install bump2version
- name: Project versioning
run: bumpversion build
- name: Commit report
run: |
git config --global user.name 'Liminal Bot'
git commit -am "Increment version"
git push
================================================
FILE: .gitignore
================================================
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
env
.idea
bin
include
lib
venv
/build
.Python
*.pyc
pip-selfcheck.json
.DS_Store
/build
apache_liminal.egg-info
scripts/*.tar.gz
scripts/*.whl
dist
================================================
FILE: .pre-commit-config.yaml
================================================
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
---
minimum_pre_commit_version: 2.16.0
exclude: >
(?x)^(
.+/.venv/.+|.+/dist/.+|.+/.autovenv|.+/docs/|.github
)$
fail_fast: true
default_language_version:
python: python3
default_stages:
- prepare-commit-msg
- commit
# - push
repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v4.0.1
hooks:
- id: check-case-conflict
- id: check-merge-conflict
stages:
- commit
- id: check-added-large-files
args: [--maxkb=1000]
stages:
- commit
- id: detect-aws-credentials
args:
- --allow-missing-credentials
- id: fix-encoding-pragma
args:
- --remove
- id: detect-private-key
- id: destroyed-symlinks
- id: mixed-line-ending
- id: trailing-whitespace
- id: check-toml
# To match test*.py instead
# - id: name-tests-test
# args: [--django]
# exclude: |
# - tests/test_licenses
- id: end-of-file-fixer
description: Ensures that a file is either empty, or ends with one newline.
exclude_types: [sql]
# types: [text]
- id: pretty-format-json
args:
- --autofix
- --no-sort-keys
- --indent
- '2'
# files:
pass_filenames: true
- repo: https://github.com/Lucas-C/pre-commit-hooks
rev: v1.1.10
hooks:
- id: insert-license
name: Add license for all md files
files: \.md$
exclude: ^\.github/.*$
args:
- --comment-style
- <!--|| -->
- --license-filepath
- dev/LICENSE.txt
- --fuzzy-match-generates-todo
- id: insert-license
name: Add license for all SQL files
files: \.sql$
exclude: ^\.github/.*$|^airflow/_vendor/
args:
- --comment-style
- /*||*/
- --license-filepath
- dev/LICENSE.txt
- --fuzzy-match-generates-todo
- id: insert-license
name: Add license for all other files
exclude: ^\.github/.*$|^airflow/_vendor/
args:
- --comment-style
- '|#|'
- --license-filepath
- dev/LICENSE.txt
- --fuzzy-match-generates-todo
files: >
\.properties$|\.cfg$|\.conf$|\.ini$|\.ldif$|\.readthedocs$|\.service$|\.tf$|Dockerfile.*$
- id: insert-license
name: Add license for all JINJA template files
files: ^airflow/www/templates/.*\.html$|^docs/templates/.*\.html$.*\.jinja2
exclude: ^\.github/.*$^airflow/_vendor/
args:
- --comment-style
- '{#||#}'
- --license-filepath
- dev/LICENSE.txt
- --fuzzy-match-generates-todo
- id: insert-license
name: Add license for all shell files
exclude: ^\.github/.*$|^airflow/_vendor/
files: ^breeze$|^breeze-complete$|\.sh$|\.bash$|\.bats$
args:
- --comment-style
- '|#|'
- --license-filepath
- dev/LICENSE.txt
- --fuzzy-match-generates-todo
- id: insert-license
name: Add license for all Python files
exclude: ^\.github/.*$|^airflow/_vendor/
types: [python]
args:
- --comment-style
- '|#|'
- --license-filepath
- dev/LICENSE.txt
- --fuzzy-match-generates-todo
- id: insert-license
name: Add license for all YAML files
exclude: ^\.github/.*$|^airflow/_vendor/
types: [yaml]
files: \.yml$|\.yaml$
args:
- --comment-style
- '|#|'
- --license-filepath
- dev/LICENSE.txt
- --fuzzy-match-generates-todo
# Python: Black formatter
- repo: https://github.com/psf/black
rev: 21.12b0
hooks:
- id: black
args: [--safe, --quiet, --config=./pyproject.toml]
files: \.pyi?$
exclude: .github/
# override until resolved: https://github.com/psf/black/issues/402
types: []
# - repo: https://github.com/jumanjihouse/pre-commit-hook-yamlfmt
# rev: 0.1.0
# hooks:
# - id: yamlfmt
# args: [--mapping, '2', --sequence, '4', --offset, '2']
# Yaml: lint
- repo: https://github.com/adrienverge/yamllint
rev: v1.26.3
hooks:
- id: yamllint
name: Check YAML files with yamllint
entry: yamllint -c yamllint-config.yml # --strict
types: [yaml]
# Python - isort to sort imports in Python files
- repo: https://github.com/timothycrosley/isort
rev: 5.10.1
hooks:
- id: isort
name: Run isort to sort imports in Python files
args: [--profile, black]
files: \.py$
# To keep consistent with the global isort skip config defined in setup.cfg
exclude: ^build/.*$|^.tox/.*$|^venv/.*$
# PyUpgrade - 3.6
- repo: https://github.com/asottile/pyupgrade
rev: v2.29.1
hooks:
- id: pyupgrade
args: [--py36-plus]
exclude: ^scripts/|^docs
- repo: https://github.com/asottile/blacken-docs
rev: v1.12.0
hooks:
- id: blacken-docs
alias: black
additional_dependencies: [black==21.9b0]
# - repo: https://github.com/PyCQA/bandit
# rev: 1.7.1
# hooks:
# - id: bandit
# args: [-ll, -r, liminal]
# files: .py$
# exclude: tests/test_licenses.py|^tests/runners
================================================
FILE: CONTRIBUTING.md
================================================
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
# Contributing to Liminal docs
Apache Liminal use Sphinx and readthedocs.org to create and publish documentation to readthedocs.org
## Basic setup
Here is what you need to do in order to work and create docs locally
## Installing sphinx and documentation
More details about sphinx and readthedocs can be found [here:](https://docs.readthedocs.io/en/stable/intro/getting-started-with-sphinx.html)
Here is what you need to do:
```
pip install sphinx
```
Inside your project go to docs lib, update the docs you like and build them:
```
cd docs
make html
```
## Automatically rebuild sphinx docs
```
pip install sphinx-autobuild
cd docs
sphinx-autobuild source/ build/html/
```
Now open a browser on
http://localhost:8000/
Your docs will automatically update
## Rebuilding the pythondoc liminal code documentation
```
cd docs
sphinx-apidoc -o source/ ../liminal
```
## RunningTests
When doing local development and running Liminal unit-tests, make sure to set LIMINAL_STAND_ALONE_MODE=True
1. Setup Minikube
2.Install python requirements: `pip install -r requirements.txt`
3. Run tests: `./run_tests.sh`
## InstallRunPreCommit
[pre-commit](https://pre-commit.com/) is used to install Python code linting and formatting tools:
### Getting started
**Requires `python >=3.6`, `pre-commit>=1.14` and a `git` repository**
1. Run `pip install pre-commit` or `pip install -r requirements.txt`
2. Install the hooks:
Simple install without post hooks: `pre-commit install`
OR
Install the hooks: `pre-commit install --install-hooks`
Optional:
Install the post commit hooks: `pre-commit install --hook-type post-commit`
1. To run pre commit hooks:
1. Either run `git commit` the new configuration files
1. Run `pre-commit run -a` to lint and format your entire project
Now on every commit, `pre-commit` will use a git hook to run the tools.
**Warning: the first commit will take some time because the tools are being installed by
`pre-commit`**
`pre-commit run -a` to lint and format your entire project
### Resolving failed commits
* If `black` fail, they have reformatted your code.
* You should check the changes made. Then simply "git add --update ." and re-commit or `git add` and `git commit` the changes.
Example:
### EnabledHooks
1. [black](https://black.readthedocs.io/en/stable/): a Python automatic code formatter
1. [yamllint](https://yamllint.readthedocs.io/): A linter for YAML files.
yamllint does not only check for syntax validity, but for weirdnesses like key repetition and cosmetic problems such as lines length, trailing spaces, indentation, etc.
1. [OutOfBoxHooks](https://github.com/pre-commit/pre-commit-hooks) - Out-of-the-box hooks for pre-commit like Check for files that contain merge conflict strings
1. [Bandit](https://bandit.readthedocs.io/en/latest/) is a tool designed to find common security issues in Python code.
1. [blacken-docs](https://github.com/asottile/blacken-docs) Run `black` on python code blocks in documentation files
1. [pyupgrade](https://github.com/asottile/pyupgrade) A tool (and pre-commit hook) to automatically upgrade syntax for newer versions of the language.
1. [isort](https://pycqa.github.io/isort/) A Python utility / library to sort imports.
================================================
FILE: DISCLAIMER-WIP
================================================
Apache Liminal is an effort undergoing incubation at the Apache Software
Foundation (ASF), sponsored by the Apache Incubator PMC.
Incubation is required of all newly accepted projects until a further review
indicates that the infrastructure, communications, and decision making process
have stabilized in a manner consistent with other successful ASF projects.
While incubation status is not necessarily a reflection of the completeness
or stability of the code, it does indicate that the project has yet to be
fully endorsed by the ASF.
Some of the incubating project’s releases may not be fully compliant with ASF policy.
For example, releases may have incomplete or un-reviewed licensing conditions.
What follows is a list of known issues the project is currently aware of
(note that this list, by definition, is likely to be incomplete):
1. The Liminal source code is distributed under the Apache License, Version 2.0. However
building Liminal requires using a transitive required library chardet V 4.0.0, which is under LGPL license, Version 2.1.
If you are planning to incorporate this work into your product/project,
please be aware that you will need to conduct a thorough licensing review
to determine the overall implications of including this work.
================================================
FILE: LICENSE
================================================
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
================================================
FILE: MANIFEST.in
================================================
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
include scripts/* requirements.txt requirements-airflow.txt
recursive-include liminal/build/ *
================================================
FILE: NOTICE
================================================
Apache Liminal
Copyright 2020-2023 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
Licensed under the Apache License 2.0.
================================================
FILE: README.md
================================================
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
# Apache Liminal
Apache Liminal is an end-to-end platform for data engineers & scientists, allowing them to build,
train and deploy machine learning models in a robust and agile way.
The platform provides the abstractions and declarative capabilities for
data extraction & feature engineering followed by model training and serving.
Liminal's goal is to operationalize the machine learning process, allowing data scientists to
quickly transition from a successful experiment to an automated pipeline of model training,
validation, deployment and inference in production, freeing them from engineering and
non-functional tasks, and allowing them to focus on machine learning code and artifacts.
# Basics
Using simple YAML configuration, create your own schedule data pipelines (a sequence of tasks to
perform), application servers, and more.
## Getting Started
A simple getting stated guide for Liminal can be found [here](docs/getting-started/hello_world.md)
## Apache Liminal Documentation
Full documentation of Apache Liminal can be found [here](docs/liminal)
## High Level Architecture
High level architecture documentation can be found [here](docs/architecture.md)
## Example YAML config file
```yaml
---
name: MyLiminalStack
owner: Bosco Albert Baracus
volumes:
- volume: myvol1
local:
path: /Users/me/myvol1
images:
- image: my_python_task_img
type: python
source: write_inputs
- image: my_parallelized_python_task_img
source: write_outputs
- image: my_server_image
type: python_server
source: myserver
endpoints:
- endpoint: /myendpoint1
module: my_server
function: myendpoint1func
pipelines:
- pipeline: my_pipeline
start_date: 1970-01-01
timeout_minutes: 45
schedule: 0 * 1 * *
metrics:
namespace: TestNamespace
backends: [ 'cloudwatch' ]
tasks:
- task: my_python_task
type: python
description: static input task
image: my_python_task_img
env_vars:
NUM_FILES: 10
NUM_SPLITS: 3
mounts:
- mount: mymount
volume: myvol1
path: /mnt/vol1
cmd: python -u write_inputs.py
- task: my_parallelized_python_task
type: python
description: parallelized python task
image: my_parallelized_python_task_img
env_vars:
FOO: BAR
executors: 3
mounts:
- mount: mymount
volume: myvol1
path: /mnt/vol1
cmd: python -u write_inputs.py
services:
- service: my_python_server
description: my python server
image: my_server_image
```
# Installation
1. Install this repository (HEAD)
```bash
pip install git+https://github.com/apache/incubator-liminal.git
```
2. Optional: set LIMINAL_HOME to path of your choice (if not set, will default to ~/liminal_home)
```bash
echo 'export LIMINAL_HOME=</path/to/some/folder>' >> ~/.bash_profile && source ~/.bash_profile
```
# Authoring pipelines
This involves at minimum creating a single file called liminal.yml as in the example above.
If your pipeline requires custom python code to implement tasks, they should be organized
[like this](https://github.com/apache/incubator-liminal/tree/master/tests/runners/airflow/liminal)
If your pipeline introduces imports of external packages which are not already a part
of the liminal framework (i.e. you had to pip install them yourself), you need to also provide
a requirements.txt in the root of your project.
# Testing the pipeline locally
When your pipeline code is ready, you can test it by running it locally on your machine.
1. Ensure you have The Docker engine running locally, and enable a local Kubernetes cluster:
And allocate it at least 3 CPUs (under "Resources" in the Docker preference UI).
If you want to execute your pipeline on a remote kubernetes cluster, make sure the cluster is configured using:
```bash
kubectl config set-context <your remote kubernetes cluster>
```
2. Build the docker images used by your pipeline.
In the example pipeline above, you can see that tasks and services have an "image" field - such as
"my_static_input_task_image". This means that the task is executed inside a docker container, and the docker container
is created from a docker image where various code and libraries are installed.
You can take a look at what the build process looks like, e.g.
[here](https://github.com/apache/incubator-liminal/tree/master/liminal/build/image/python)
In order for the images to be available for your pipeline, you'll need to build them locally:
```bash
cd </path/to/your/liminal/code>
liminal build
```
You'll see that a number of outputs indicating various docker images built.
3. Create a kubernetes local volume \
In case your Yaml includes working with [volumes](https://github.com/apache/incubator-liminal/blob/6253f8b2c9dc244af032979ec6d462dc3e07e170/docs/getting_started.md#mounted-volumes)
please first run the following command:
```bash
cd </path/to/your/liminal/code>
liminal create
```
4. Deploy the pipeline:
```bash
cd </path/to/your/liminal/code>
liminal deploy
```
Note: after upgrading liminal, it's recommended to issue the command
```bash
liminal deploy --clean
```
This will rebuild the airlfow docker containers from scratch with a fresh version of liminal, ensuring consistency.
5. Start the server
```bash
liminal start
```
6. Stop the server
```bash
liminal stop
```
7. Display the server logs
```bash
liminal logs --follow/--tail
Number of lines to show from the end of the log:
liminal logs --tail=10
Follow log output:
liminal logs --follow
```
8. Navigate to [http://localhost:8080/admin](http://localhost:8080/admin)
9. You should see your 
The pipeline is scheduled to run according to the ```json schedule: 0 * 1 * *``` field in the .yml file you provided.
10. To manually activate your pipeline:
- Click your pipeline and then click "trigger DAG"
- Click "Graph view"
You should see the steps in your pipeline getting executed in "real time" by clicking "Refresh" periodically.
# Contributing
More information on contributing can be found [here](CONTRIBUTING.md)
# Community
The Liminal community holds a public call every Monday
- [Liminal Community Calendar](https://calendar.google.com/calendar/u/0/r?cid=jom1i20emghura6s6ookhe2skk@group.calendar.google.com)
- [Dev-Mailing-List](https://lists.apache.org/list.html?dev@liminal.apache.org)
## Running Tests (for contributors)
When doing local development and running Liminal unit-tests, make sure to set LIMINAL_STAND_ALONE_MODE=True
================================================
FILE: RETIRED.txt
================================================
This podling has been retired. Please see http://incubator.apache.org/projects/index.html#liminal
================================================
FILE: dev/LICENSE.txt
================================================
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
================================================
FILE: dev/RELEASE.md
================================================
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
<!-- START doctoc generated TOC please keep comment here to allow auto update -->
<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
- [Apache Liminal source releases](#apache-liminal-source-releases)
- [Apache Liminal Package](#apache-liminal-package)
- [Prerequisites for the release manager preparing the release](#prerequisites-for-the-release-manager-preparing-the-release)
- [version IDs](#version-ids)
- [Testing the release locally](#testing-the-release-locally)
- [Upload Public keys to id.apache.org](#upload-public-keys-to-idapacheorg)
- [Configure PyPI uploads](#configure-pypi-uploads)
- [Hardware used to prepare and verify the packages](#hardware-used-to-prepare-and-verify-the-packages)
- [Apache Liminal packages](#apache-liminal-packages)
- [Prepare the Apache Liminal Package RC](#prepare-the-apache-liminal-package-rc)
- [Build RC artifacts (both source packages and convenience packages)](#build-rc-artifacts-both-source-packages-and-convenience-packages)
- [Prepare PyPI convenience "snapshot" packages](#prepare-pypi-convenience-snapshot-packages)
- [Vote and verify the Apache Liminal release candidate](#vote-and-verify-the-apache-liminal-release-candidate)
- [Prepare Vote email on the Apache Liminal release candidate](#prepare-vote-email-on-the-apache-liminal-release-candidate)
- [Verify the release candidate by PMCs (legal)](#verify-the-release-candidate-by-pmcs-legal)
- [PMC responsibilities](#pmc-responsibilities)
- [SVN check](#svn-check)
- [Verify the licences](#verify-the-licences)
- [Verify the signatures](#verify-the-signatures)
- [Verify the SHA512 sum](#verify-the-sha512-sum)
- [Verify if the release candidate "works" by Contributors](#verify-if-the-release-candidate-works-by-contributors)
- [Publish the final Apache Liminal release](#publish-the-final-apache-liminal-release)
- [Summarize the voting for the Apache Liminal release](#summarize-the-voting-for-the-apache-liminal-release)
- [Publish release to SVN](#publish-release-to-svn)
- [Prepare PyPI "release" packages](#prepare-pypi-release-packages)
- [Update CHANGELOG.md](#update-changelogmd)
- [Notify developers of release](#notify-developers-of-release)
<!-- END doctoc generated TOC please keep comment here to allow auto update -->
# Apache Liminal source releases
## Apache Liminal Package
This package contains sources that allow the user building fully-functional Apache Liminal package.
They contain sources for "apache-liminal" python package that installs "liminal" Python package and includes
all the assets required to release the webserver UI coming with Apache Liminal
The Source releases are the only "official" Apache Software Foundation releases, and they are distributed
via [Official Apache Download sources](https://downloads.apache.org/)
Following source releases Apache Liminal release manager also distributes convenience packages:
* PyPI packages released via https://pypi.org/project/apache-liminal/
* Docker Images released via https://hub.docker.com/repository/docker/apache/liminal (coming soon...)
Those convenience packages are not "official releases" of Apache Liminal, but the users who
cannot or do not want to build the packages themselves can use them as a convenient way of installing
Apache Liminal, however they are not considered as "official source releases". You can read more
details about it in the [ASF Release Policy](http://www.apache.org/legal/release-policy.html).
This document describes the process of releasing both - official source packages and convenience
packages for Apache Liminal packages.
# Prerequisites for the release manager preparing the release
The person acting as release manager has to fulfill certain pre-requisites. More details and FAQs are
available in the [ASF Release Policy](http://www.apache.org/legal/release-policy.html) but here some important
pre-requisites are listed below. Note that release manager does not have to be a PMC - it is enough
to be committer to assume the release manager role, but there are final steps in the process (uploading
final releases to SVN) that can only be done by PMC member. If needed, the release manager
can ask PMC to perform that final step of release.
## version IDs
Should follow https://www.python.org/dev/peps/pep-0440/
## Testing the release locally
- Build a local version of liminal:
```bash
# Set Version
export LIMINAL_BUILD_VERSION=0.0.1rc1-incubating
python setup.py sdist bdist_wheel
```
- Start a test project with a liminal .yml file in it
- Clear folder $LIMINAL_HOME
- Install liminal in the test project
```bash
pip install <path to lminal .whl created by setup.py>
liminal build
liminal deploy --clean
liminal start
```
Note:
When you run liminal deploy, a liminal installs itself inside airflow container.
You can control which version of liminal gets installed inside docker using LIMINAL_VERSION environment variable.
This can be any string which results in a legal call to:
```bash
pip install ${LIMINAL_VERSION}
```
This includes
- A standard pip version like apache-liminal==0.0.1dev1 avail from pypi
- A URL for git e.g. git+https://github.com/apache/incubator-liminal.git
- A string indicating where to get the package from like --index <url> apache-liminal==xyz
- A local path to a .whl which is available inside the docker (e.g. which you placed
in the scripts/ folder)
This is useful if you are making changes in liminal locally and want to test them.
If you don't specify this variable, liminal attempts to discover how to install itself
by running a pip freeze and looking at the result.
This covers pip repositories, files and installtion from URL.
The fallback in case no string is found, is simply 'apache-liminal' assuming your .pypirc contains an
index which has this package.
## Testing a version from testpypi:
Installing liminal locally:
pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple apache-liminal
Tell liminal where to take the version to put inside the airflow docker:
export LIMINAL_VERSION='--index-url https://test.pypi.org/simple/ apache-liminal==0.0.1rc1-incubating'
## Upload Public keys to id.apache.org
Make sure your public key is on id.apache.org and in KEYS. You will need to sign the release artifacts
with your pgp key. After you have created a key, make sure you:
- Add your GPG pub key to https://dist.apache.org/repos/dist/release/liminal/KEYS , follow the instructions at the top of that file. Upload your GPG public key to https://pgp.mit.edu
- Add your key fingerprint to https://id.apache.org/ (login with your apache credentials, paste your fingerprint into the pgp fingerprint field and hit save).
```shell script
# Create PGP Key
gpg --gen-key
# Checkout ASF dist repo
svn checkout https://dist.apache.org/repos/dist/release/incubator/liminal
cd liminal
# Add your GPG pub key to KEYS file. Replace "Aviem Zur" with your name
(gpg --list-sigs "Aviem Zur" && gpg --armor --export "Aviem Zur" ) >> KEYS
# Commit the changes
svn commit -m "Add PGP keys of Liminal developers"
```
See this for more detail on creating keys and what is required for signing releases.
http://www.apache.org/dev/release-signing.html#basic-facts
## Configure PyPI uploads
In order to not reveal your password in plain text, it's best if you create and configure API Upload tokens.
You can add and copy the tokens here:
* [Test PyPI](https://test.pypi.org/manage/account/token/)
* [Prod PyPI](https://pypi.org/manage/account/token/)
Create a `~/.pypirc` file:
```ini
[distutils]
index-servers =
pypi
pypitest
[pypi]
username=__token__
password=<API Upload Token>
[pypitest]
repository=https://test.pypi.org/legacy/
username=__token__
password=<API Upload Token>
```
Set proper permissions for the pypirc file:
```shell script
$ chmod 600 ~/.pypirc
```
- Install [twine](https://pypi.org/project/twine/) if you do not have it already (it can be done
in a separate virtual environment).
```shell script
pip install twine
```
(more details [here](https://peterdowns.com/posts/first-time-with-pypi.html).)
- Set proper permissions for the pypirc file:
`$ chmod 600 ~/.pypirc`
## Hardware used to prepare and verify the packages
The best way to prepare and verify the releases is to prepare them on a hardware owned and controlled
by the committer acting as release manager. While strictly speaking, releases must only be verified
on hardware owned and controlled by the committer, for practical reasons it's best if the packages are
prepared using such hardware. More information can be found in this
[FAQ](http://www.apache.org/legal/release-policy.html#owned-controlled-hardware)
# Apache Liminal packages
## Prepare the Apache Liminal Package RC
### Build RC artifacts (both source packages and convenience packages)
The Release Candidate artifacts we vote upon should be the exact ones we vote against, without any modification than renaming – i.e. the contents of the files must be the same between voted release canidate and final release. Because of this the version in the built artifacts that will become the official Apache releases must not include the rcN suffix.
- Set environment variables
```bash
# Set Version
export LIMINAL_BUILD_VERSION=0.0.1rc1-incubating
# Example after cloning
git clone https://github.com/apache/incubating-liminal.git
cd incubating-liminal
```
- Mark the next version
```bash
# Set Version
sed -i '/^current_version /s/=.*$/= '"$LIMINAL_BUILD_VERSION"'/' .bumpversion.cfg
```
- Tag your release
```bash
git tag -a ${LIMINAL_BUILD_VERSION} -m "some message"
```
- Clean the checkout: the sdist step below will
```bash
git clean -fxd
```
- Tarball the repo
```bash
git archive --format=tar.gz ${LIMINAL_BUILD_VERSION} --prefix=apache-liminal-${LIMINAL_BUILD_VERSION}/ -o apache-liminal-${LIMINAL_BUILD_VERSION}-source.tar.gz
```
- Generate sdist
NOTE: Make sure your checkout is clean at this stage - any untracked or changed files will otherwise be included
in the file produced.
```bash
python setup.py sdist bdist_wheel
```
- Generate SHA512/ASC (If you have not generated a key yet, generate it by following instructions on http://www.apache.org/dev/openpgp.html#key-gen-generate-key)
```bash
dev/sign.sh apache-liminal-${LIMINAL_BUILD_VERSION}-source.tar.gz
dev/sign.sh dist/apache-liminal-${LIMINAL_BUILD_VERSION}.tar.gz
dev/sign.sh dist/apache_liminal-${LIMINAL_BUILD_VERSION}-py3-none-any.whl
```
- Push Tags
```bashs
git push origin ${LIMINAL_BUILD_VERSION}
```
- Push the artifacts to ASF dev dist repo
```
# First clone the repo
svn checkout https://dist.apache.org/repos/dist/dev/liminal liminal-dev
# Create new folder for the release
cd liminal-dev
svn mkdir ${LIMINAL_BUILD_VERSION}
# Move the artifacts to svn folder & commit
mv apache{-,_}liminal-${LIMINAL_BUILD_VERSION}* ${LIMINAL_BUILD_VERSION}/
cd ${LIMINAL_BUILD_VERSION}
svn add *
svn commit -m "Add artifacts for Liminal ${LIMINAL_BUILD_VERSION}"
```
### Prepare PyPI convenience "snapshot" packages
At this point we have the artefact that we vote on, but as a convenience to developers we also want to
publish "snapshots" of the RC builds to pypi for installing via pip. To do this we need to
- Verify the artifacts that would be uploaded:
```bash
twine check dist/*
```
- Upload the package to PyPi's test environment:
```bash
twine upload -r pypitest dist/*
```
- Verify that the test package looks good by downloading it and installing it into a virtual environment. The package download link is available at:
https://test.pypi.org/project/apache-liminal/#files
- Upload the package to PyPi's production environment:
`twine upload -r pypi dist/*`
- Again, confirm that the package is available here:
https://pypi.python.org/pypi/apache-liminal
It is important to stress that this snapshot should not be named "release", and it
is not supposed to be used by and advertised to the end-users who do not read the devlist.
## Vote and verify the Apache Liminal release candidate
### Prepare Vote email on the Apache Liminal release candidate
- Use the dev/liminal-jira script to generate a list of Liminal JIRAs that were closed in the release.
- Send out a vote to the dev@liminal.apache.org mailing list:
Subject:
```
[VOTE] Liminal 1.10.2rc3
```
Body:
```
Hey all,
I have cut Liminal 1.10.2 RC3. This email is calling a vote on the release,
which will last for 72 hours. Consider this my (binding) +1.
Liminal 1.10.2 RC3 is available at:
https://dist.apache.org/repos/dist/dev/liminal/1.10.2rc3/
*apache-liminal-1.10.2rc3-source.tar.gz* is a source release that comes
with INSTALL instructions.
*apache-liminal-1.10.2rc3-bin.tar.gz* is the binary Python "sdist" release.
Public keys are available at:
https://dist.apache.org/repos/dist/release/liminal/KEYS
Only votes from PMC members are binding, but the release manager should encourage members of the community
to test the release and vote with "(non-binding)".
The test procedure for PMCs and Contributors who would like to test this RC are described in
https://github.com/apache/liminal/blob/master/dev/README.md#vote-and-verify-the-apache-liminal-release-candidate
Please note that the version number excludes the `rcX` string, so it's now
simply 1.10.2. This will allow us to rename the artifact without modifying
the artifact checksums when we actually release.
Changes since 1.10.2rc2:
*Bugs*:
[LIMINAL-3732] ...
...
*Improvements*:
[LIMINAL-3302] ...
...
*New features*:
[LIMINAL-2874] ...
...
*Doc-only Change*:
[LIMINAL-XXX] Fix Minor issues in Documentation
...
Cheers,
<your name>
```
### Verify the release candidate by PMCs (legal)
#### PMC responsibilities
The PMCs should verify the releases in order to make sure the release is following the
[Apache Legal Release Policy](http://www.apache.org/legal/release-policy.html).
At least 3 (+1) votes should be recorded in accordance to
[Votes on Package Releases](https://www.apache.org/foundation/voting.html#ReleaseVotes)
The legal checks include:
* checking if the packages are present in the right dist folder on svn
* verifying if all the sources have correct licences
* verifying if release manager signed the releases with the right key
* verifying if all the checksums are valid for the release
#### SVN check
The files should be present in the sub-folder of
[Liminal dist](https://dist.apache.org/repos/dist/dev/incubator/liminal/)
The following files should be present (9 files):
* .tar.gz + .asc + .sha512
* -.whl + .asc + .sha512
As a PMC you should be able to clone the SVN repository:
```shell script
svn co https://dist.apache.org/repos/dist/dev/incubator/liminal
```
Or update it if you already checked it out:
```shell script
svn update .
```
#### Verify the licences
This can be done with the Apache RAT tool.
* Download the latest jar from https://creadur.apache.org/rat/download_rat.cgi (unpack the sources,
the jar is inside)
* Unpack the .tar.gz to a folder
* Enter the folder and run the check (point to the place where you extracted the .jar)
```shell script
java -jar ../../apache-rat-0.13/apache-rat-0.13.jar -E .rat-excludes -d .
```
#### Verify the signatures
Make sure you have the key of person signed imported in your GPG. You can find the valid keys in
[KEYS](https://dist.apache.org/repos/dist/release/liminal/KEYS).
You can import the whole KEYS file:
```shell script
gpg --import KEYS
```
You can also import the keys individually from a keyserver. The below one uses Kaxil's key and
retrieves it from the default GPG keyserver
[OpenPGP.org](https://keys.openpgp.org):
```shell script
gpg --receive-keys 12717556040EEF2EEAF1B9C275FCCD0A25FA0E4B
```
You should choose to import the key when asked.
Note that by being default, the OpenPGP server tends to be overloaded often and might respond with
errors or timeouts. Many of the release managers also uploaded their keys to the
[GNUPG.net](https://keys.gnupg.net) keyserver, and you can retrieve it from there.
```shell script
gpg --keyserver keys.gnupg.net --receive-keys 12717556040EEF2EEAF1B9C275FCCD0A25FA0E4B
```
Once you have the keys, the signatures can be verified by running this:
```shell script
for i in *.asc
do
echo "Checking $i"; gpg --verify `basename $i .sha512 `
done
```
This should produce results similar to the below. The "Good signature from ..." is indication
that the signatures are correct. Do not worry about the "not certified with a trusted signature"
warning. Most of the certificates used by release managers are self signed, that's why you get this
warning. By importing the server in the previous step and importing it via ID from
[KEYS](https://dist.apache.org/repos/dist/release/liminal/KEYS) page, you know that
this is a valid Key already.
```
Checking apache-liminal-1.10.12rc4-bin.tar.gz.asc
gpg: assuming signed data in 'apache-liminal-1.10.12rc4-bin.tar.gz'
gpg: Signature made sob, 22 sie 2020, 20:28:28 CEST
gpg: using RSA key 12717556040EEF2EEAF1B9C275FCCD0A25FA0E4B
gpg: Good signature from "Kaxil Naik <kaxilnaik@gmail.com>" [unknown]
gpg: WARNING: This key is not certified with a trusted signature!
gpg: There is no indication that the signature belongs to the owner.
Primary key fingerprint: 1271 7556 040E EF2E EAF1 B9C2 75FC CD0A 25FA 0E4B
Checking apache_liminal-1.10.12rc4-py2.py3-none-any.whl.asc
gpg: assuming signed data in 'apache_liminal-1.10.12rc4-py2.py3-none-any.whl'
gpg: Signature made sob, 22 sie 2020, 20:28:31 CEST
gpg: using RSA key 12717556040EEF2EEAF1B9C275FCCD0A25FA0E4B
gpg: Good signature from "Kaxil Naik <kaxilnaik@gmail.com>" [unknown]
gpg: WARNING: This key is not certified with a trusted signature!
gpg: There is no indication that the signature belongs to the owner.
Primary key fingerprint: 1271 7556 040E EF2E EAF1 B9C2 75FC CD0A 25FA 0E4B
Checking apache-liminal-1.10.12rc4-source.tar.gz.asc
gpg: assuming signed data in 'apache-liminal-1.10.12rc4-source.tar.gz'
gpg: Signature made sob, 22 sie 2020, 20:28:25 CEST
gpg: using RSA key 12717556040EEF2EEAF1B9C275FCCD0A25FA0E4B
gpg: Good signature from "Kaxil Naik <kaxilnaik@gmail.com>" [unknown]
gpg: WARNING: This key is not certified with a trusted signature!
gpg: There is no indication that the signature belongs to the owner.
Primary key fingerprint: 1271 7556 040E EF2E EAF1 B9C2 75FC CD0A 25FA 0E4B
```
#### Verify the SHA512 sum
Run this:
```shell script
for i in *.sha512
do
echo "Checking $i"; gpg --print-md SHA512 `basename $i .sha512 ` | diff - $i
done
```
You should get output similar to:
```
Checking apache_liminal-1.10.12rc4-py3-none-any.whl.sha512
Checking apache-liminal-1.10.12rc4.tar.gz.sha512
```
### Verify if the release candidate "works" by Contributors
This can be done (and we encourage to) by any of the Contributors. In fact, it's best if the
actual users of Apache Liminal test it in their own staging/test installations. Each release candidate
is available on PyPI apart from SVN packages, so everyone should be able to install
the release candidate version of Liminal via simply (<VERSION> is 1.10.12 for example, and <X> is
release candidate number 1,2,3,....).
```shell script
pip install apache-liminal==<LIMINAL_BUID_VERSION>rc<X>
```
### Seek approval from Incubator PMC
[Post to the Incubator’s general list requesting approval from the Incubator PMC](https://lists.apache.org/thread.html/06655226ba08c16a8cb273f9b45e0b0a15ebaed0d06783fdd06a03f6@%3Cgeneral.incubator.apache.org%3E).
Should the Incubator PMC vote to approve a release, the Podling MAY make that release available to the public under these conditions:
* The release archive(s) MUST include the word "incubating" in the filename.
* The release archive(s) MUST contain a disclaimer found in DISCLAIMER file.
Releases for the Podling MUST be distributed through http://www.apache.org/dist/incubator/Podling
https://lists.apache.org/thread.html/06655226ba08c16a8cb273f9b45e0b0a15ebaed0d06783fdd06a03f6@%3Cgeneral.incubator.apache.org%3E
## Publish the final Apache Liminal release
### Summarize the voting for the Apache Liminal release
Once the vote has been passed, you will need to send a result vote to dev@liminal.apache.org:
Subject:
```
[RESULT][VOTE] Liminal 1.10.2rc3
```
Message:
```
Hello,
Apache Liminal 1.10.2 (based on RC3) has been accepted.
3 “+1” binding votes received:
....
3 "+1" non-binding votes received:
...
Vote thread:
...
I'll continue with the release process, and the release announcement will follow shortly.
Cheers,
<your name>
```
### Publish release to SVN
You need to migrate the RC artifacts that passed to this repository:
https://dist.apache.org/repos/dist/release/liminal/
(The migration should include renaming the files so that they no longer have the RC number in their filenames.)
The best way of doing this is to svn cp between the two repos (this avoids having to upload the binaries again, and gives a clearer history in the svn commit logs):
```shell script
# First clone the repo
export RC=1.10.4rc5
export VERSION=${RC/rc?/}
svn checkout https://dist.apache.org/repos/dist/release/liminal liminal-release
# Create new folder for the release
cd liminal-release
svn mkdir ${VERSION}
cd ${VERSION}
# Move the artifacts to svn folder & commit
for f in ../../liminal-dev/$RC/*; do svn cp $f ${$(basename $f)/rc?/}; done
svn commit -m "Release Liminal ${VERSION} from ${RC}"
# Remove old release
# http://www.apache.org/legal/release-policy.html#when-to-archive
cd ..
export PREVIOUS_VERSION=1.10.1
svn rm ${PREVIOUS_VERSION}
svn commit -m "Remove old release: ${PREVIOUS_VERSION}"
```
Verify that the packages appear in [liminal](https://dist.apache.org/repos/dist/release/liminal/)
### Prepare PyPI "release" packages
At this point we release an official package:
- Build the package:
```shell script
python setup.py sdist bdist_wheel`
```
- Verify the artifacts that would be uploaded:
```shell script
twine check dist/*`
```
- Upload the package to PyPi's test environment:
```shell script
twine upload -r pypitest dist/*
```
- Verify that the test package looks good by downloading it and installing it into a virtual environment.
The package download link is available at: https://test.pypi.org/project/apache-liminal/#files
- Upload the package to PyPi's production environment:
```shell script
twine upload -r pypi dist/*
```
- Again, confirm that the package is available here: https://pypi.python.org/pypi/apache-liminal
### Update CHANGELOG.md
- Get a diff between the last version and the current version:
```shell script
$ git log 1.8.0..1.9.0 --pretty=oneline
```
- Update CHANGELOG.md with the details, and commit it.
### Notify developers of release
- Notify users@liminal.apache.org (cc'ing dev@liminal.apache.org and announce@apache.org) that
the artifacts have been published:
Subject:
```shell script
cat <<EOF
Liminal ${VERSION} is released
EOF
```
Body:
```shell script
cat <<EOF
Dear Liminal community,
I'm happy to announce that Liminal ${VERSION} was just released.
The source release, as well as the binary "sdist" release, are available
here:
https://dist.apache.org/repos/dist/release/liminal/${VERSION}/
We also made this version available on PyPi for convenience (`pip install apache-liminal`):
https://pypi.python.org/pypi/apache-liminal
The documentation is available on:
https://liminal.apache.org/
https://liminal.apache.org/1.10.2/
https://liminal.readthedocs.io/en/1.10.2/
https://liminal.readthedocs.io/en/stable/
Find the CHANGELOG here for more details:
https://liminal.apache.org/changelog.html#liminal-1-10-2-2019-01-19
Cheers,
<your name>
EOF
```
================================================
FILE: dev/sign.sh
================================================
#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# Use this to sign the tar balls generated from
# python setup.py sdist --formats=gztar
# ie. sign.sh <my_tar_ball>
# you will still be required to type in your signing key password
# or it needs to be available in your keychain
NAME="${1}"
gpg --armor --output "${NAME}.asc" --detach-sig "${NAME}"
gpg --print-md SHA512 "${NAME}" > "${NAME}.sha512"
================================================
FILE: docs/Makefile
================================================
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# Minimal makefile for Sphinx documentation
# You can set these variables from the command line.
SPHINXOPTS =
SPHINXBUILD = sphinx-build
SOURCEDIR = .
BUILDDIR = build
# Put it first so that "make" without argument is like "make help".
help:
@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
.PHONY: help Makefile
# Catch-all target: route all unknown targets to Sphinx using the new
# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).
%: Makefile
@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
================================================
FILE: docs/README.md
================================================
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
# Apache Liminal
Apache Liminal is an end-to-end platform for data engineers & scientists, allowing them to build,
train and deploy machine learning models in a robust and agile way.
The platform provides the abstractions and declarative capabilities for
data extraction & feature engineering followed by model training and serving.
Liminal's goal is to operationalize the machine learning process, allowing data scientists to
quickly transition from a successful experiment to an automated pipeline of model training,
validation, deployment and inference in production, freeing them from engineering and
non-functional tasks, and allowing them to focus on machine learning code and artifacts.
## Basics
Using simple YAML configuration, create your own schedule data pipelines (a sequence of tasks to
perform), application servers, and more.
## Getting Started
A simple hello world guide for Liminal can be found [here](getting-started/hello_world.md) \
A more advanced example which demonstrates a simple data-science workflow can be found [here](getting-started/iris_classification.md)
## Apache Liminal Documentation
Full documentation of Apache Liminal can be found [here](liminal)
## High Level Architecture
High level architecture documentation can be found [here](architecture.md)
================================================
FILE: docs/architecture.md
================================================
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
# High Level Architecture
Liminal is an end-to-end platform for data engineers & scientists, allowing them to build, train and deploy machine learning models in a robust and agile way. The platform provides the abstractions and declarative capabilities for data extraction & feature engineering followed by model training and serving. Apache Liminal's goal is to operationalise the machine learning process, allowing data scientists to quickly transition from a successful experiment to an automated pipeline of model training, validation, deployment and inference in production, freeing them from engineering and non-functional tasks, and allowing them to focus on machine learning code and artifacts.
## Motivation
The challenges involved in operationalizing machine learning models are one of the main reasons why many machine learning projects never make it to production.
The process involves automating and orchestrating multiple steps which run on heterogeneous infrastructure - different compute environments, data processing platforms, ML frameworks, notebooks, containers and monitoring tools.
There are no mature standards for this workflow, and most organizations do not have the experience to build it in-house. In the best case, dev-ds-devops teams form in order to accomplish this task together; in many cases, it's the data scientists who try to deal with this themselves without the knowledge or the inclination to become infrastructure experts.
As a result, many projects never make it through the cycle. Those who do suffer from a very long lead time from a successful experiment to an operational, refreshable, deployed and monitored model in production.
The goal of Apache Liminal is to simplify the creation and management of machine learning pipelines by data engineers & scientists. The platform provides declarative building blocks which define the workflow, orchestrate the underlying infrastructure, take care of non functional concerns, enabling focus in business logic / algorithm code.
Some Commercial E2E solutions have started to emerge in the last few years, however, they are limited to specific parts of the workflow, such as Databricks MLFlow. Other solutions are tied to specific environments (e.g. SageMaker on AWS).
## High Level Architecture
The platform is aimed to provide data engineers & scientists with a solution for end to end flows from model training to real time inference in production. It’s architecture enables and promotes adoption of specific components in existing (non-Liminal) frameworks, as well as seamless integration with other open source projects. Liminal was created to enable scalability in ML efforts and after a thorough review of available solutions and frameworks, which did not meet our main KPIs:
- Provide an opinionated but customizable end-to-end workflow
- Abstract away the complexity of underlying infrastructure
- Support major open source tools and cloud-native infrastructure to carry out many of the steps
- Allow teams to leverage their existing investments or bring in their tools of choice into the workflow
We have found that other tech companies in the Hi-Tech ecosystem also have an interest in such a platform, hence decided to share our work with the community.
The following diagram depicts these main components and where Apache Liminal comes in:
A classical data scientist workflow includes some base phases:
_Train, Deploy and Consume._
**The Train phase includes the following tasks:**
1. Fetch - get the data needed to build a model - usually using SQL
1. Clean - make sure the data is useful for building the model
1. Prepare - split data and encode features from the data according to model needs
1. Train - Build the model and tune it
1. Evaluate - make sure the model is correct - run it on a test set, etc…
1. Validate - make sure the model is up to the standards you need
**The Deploy phase includes these tasks:**
1. Deploy - make it available for usage in production
1. Inference - Batch or Real-time - use the model to evaluate data by your offline or online by your applications
1. Consume - The actual use of the models created by applications and ETLs, usually through APIs to the batch or real-time inference that usually rely on Model and Feature stores.
Liminal provides its users a declarative composition capabilities to materialize these steps in a robust way, while exploiting existing frameworks and tools. e.g. Data science frameworks such as scikit-learn, Tensor flow, Keras and such, for running core data science algorithms; as numerous core mechanisms as data stores, processing engines, parallelism, schedulers, code deployment as well as batch and real-time inference.
Liminal allows the creation and wiring of these kinds of functional and non functional tasks while making the underlying infrastructure used by these tasks very easy to use and even abstracted away entirely. While handling the non-functional aspects as monitoring (in a standard fashion) deployment, scheduling, resource management and execution.
================================================
FILE: docs/conf.py
================================================
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
# -*- coding: utf-8 -*-
#
# Configuration file for the Sphinx documentation builder.
#
# This file does only contain a selection of the most common options. For a
# full list see the documentation:
# http://www.sphinx-doc.org/en/master/config
# -- Path setup --------------------------------------------------------------
# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#
import os
import sys
sys.path.insert(0, os.path.abspath('source'))
sys.path.insert(0, os.path.abspath('/'))
# -- Project information -----------------------------------------------------
project = u'Apchae Liminal'
copyright = u'Apache Liminal'
author = u'Apache Software Foundation'
# The short X.Y version
version = u''
# The full version, including alpha/beta/rc tags
release = u'0.0.1'
# -- General configuration ---------------------------------------------------
# If your documentation needs a minimal Sphinx version, state it here.
#
# needs_sphinx = '1.0'
# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
'recommonmark',
'sphinx.ext.autodoc',
]
html_logo = 'nstatic/liminal_logo.png'
# Add any paths that contain templates here, relative to this directory.
templates_path = ['ntemplates']
# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
#
# source_suffix = ['.rst', '.md']
source_suffix = '.rst'
# The master toctree document.
master_doc = 'index'
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
language = None
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This pattern also affects html_static_path and html_extra_path.
exclude_patterns = ['README.md', '**/README.md']
# The name of the Pygments (syntax highlighting) style to use.
pygments_style = None
# -- Options for HTML output -------------------------------------------------
# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
#
html_theme = 'classic'
# Theme options are theme-specific and customize the look and feel of a theme
# further. For a list of options available for each theme, see the
# documentation.
#
# html_theme_options = {}
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['nstatic']
# Custom sidebar templates, must be a dictionary that maps document names
# to template names.
#
# The default sidebars (for documents that don't match any pattern) are
# defined by theme itself. Builtin themes are using these templates by
# default: ``['localtoc.html', 'relations.html', 'sourcelink.html',
# 'searchbox.html']``.
#
# html_sidebars = {}
# -- Options for HTMLHelp output ---------------------------------------------
# Output file base name for HTML help builder.
htmlhelp_basename = 'ApchaeLiminaldoc'
# -- Options for LaTeX output ------------------------------------------------
latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#
# 'papersize': 'letterpaper',
# The font size ('10pt', '11pt' or '12pt').
#
# 'pointsize': '10pt',
# Additional stuff for the LaTeX preamble.
#
# 'preamble': '',
# Latex figure (float) alignment
#
# 'figure_align': 'htbp',
}
# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
# author, documentclass [howto, manual, or own class]).
latex_documents = [
(master_doc, 'ApchaeLiminal.tex', u'Apchae Liminal Documentation', u'Apache Software Foundation', 'manual'),
]
# -- Options for manual page output ------------------------------------------
# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [(master_doc, 'apchaeliminal', u'Apchae Liminal Documentation', [author], 1)]
# -- Options for Texinfo output ----------------------------------------------
# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
# dir menu entry, description, category)
texinfo_documents = [
(
master_doc,
'ApchaeLiminal',
u'Apchae Liminal Documentation',
author,
'ApchaeLiminal',
'One line description of project.',
'Miscellaneous',
),
]
# -- Options for Epub output -------------------------------------------------
# Bibliographic Dublin Core info.
epub_title = project
# The unique identifier of the text. This can be a ISBN number
# or the project homepage.
#
# epub_identifier = ''
# A unique identification for the text.
#
# epub_uid = ''
# A list of files that should not be packed into the epub file.
epub_exclude_files = ['search.html']
def on_missing_reference(app, env, node, contnode):
if node['reftype'] == 'any':
return contnode
else:
return None
def setup(app):
app.connect('missing-reference', on_missing_reference)
================================================
FILE: docs/getting-started/hello_world.md
================================================
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
# Getting started / ***Hello World***
This guide will allow you to set up your first Apache Liminal environment and allow you to create
some simple ML pipelines. These will be very similar to the ones you are going to build for real
production scenarios.
## Prerequisites
Python 3 (3.6 and up)
[Docker Desktop](https://www.docker.com/products/docker-desktop)
*Note: Make sure kubernetes cluster is running in docker desktop (or custom kubernetes installation
on your machine).*
## Deploying the Example
In this tutorial, we will go through setting up Liminal for the first time on your local machine.
### First, let’s build our example project:
In the dev folder, just clone the example code from liminal:
```BASH
git clone https://github.com/apache/incubator-liminal
```
***Note:*** *You just cloned the entire Liminal Project, you actually only need examples folder.*
Create a python virtual environment to isolate your runs:
```BASH
cd incubator-liminal/examples/liminal-getting-started
python3 -m venv env
```
Activate your virtual environment:
```BASH
source env/bin/activate
```
Now we are ready to install liminal:
```BASH
pip install apache-liminal
```
Let's build the images you need for the example:
```BASH
liminal build
```
##### The build will create docker images based on the liminal.yml file in the `images` section.
```BASH
liminal deploy --clean
```
The deploy command deploys a liminal server and deploys any liminal.yml files in your working
directory or any of its subdirectories to your liminal home directory.
*Note: liminal home directory is located in the path defined in LIMINAL_HOME env variable.
If the LIMINAL_HOME environemnet variable is not defined, home directory defaults to
~/liminal_home directory.*
Now lets runs liminal:
```BASH
liminal start
```
The start command spins up the liminal server containers which will run pipelines based on your
deployed liminal.yml files.
It runs the following three containers:
* liminal-postgress
* liminal-webserver
* liminal-scheduler
Once liminal server has completed starting up, you can navigate to admin UI in your browser:
[http://localhost:8080](http://localhost:8080)
By default liminal server starts Apache Airflow servers and admin UI will be that of Apache Airflow.
***Important:** Set off/on toggle to activate your pipeline (DAG), nothing will happen otherwise!*
You can go to graph view to see all the tasks configured in the liminal.yml file:
[http://localhost:8080/admin/airflow/graph?dag_id=example_pipeline](
http://localhost:8080/admin/airflow/graph?dag_id=example_pipeline
)
#### Now lets see what actually happened to our task:
#### Click on “hello_world_example” and you will get this popup:
#### Click on “view log” button and you can see the log of the current task run:
## Mounted volumes
All tasks use a mounted volume as defined in the pipeline YAML:
```YAML
name: GettingStartedPipeline
volumes:
- volume: gettingstartedvol
claim_name: gettingstartedvol-pvc
local:
path: .
```
In our case the mounted volume will point to the liminal hello world example. \
The hello world task will read the **hello_world.json** file from the mounted volume and will write
the **hello_world_output.json** to it.
*Note:* Each task will internally mount the volume defined above to an internal representation,
described under the task section in the yml:
```YAML
task:
...
mounts:
- mount: taskmount
volume: gettingstartedvol
path: /mnt/vol1
```
## Here are the entire list of commands, if you want to start from scratch:
```
git clone https://github.com/apache/incubator-liminal
cd examples/liminal-getting-started
python3 -m venv env
source env/bin/activate
pip uninstall apache-liminal
pip install apache-liminal
Liminal build
liminal deploy --clean
liminal start
```
## Closing up
To make sure liminal containers are stopped use:
```
liminal stop
```
To deactivate the python virtual env use:
```
deactivate
```
================================================
FILE: docs/getting-started/iris_classification.md
================================================
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
# Getting started / ***Iris Classification***
* [Setup your local environment](#Setup-your-local-environment)
* [Setup liminal](#setup-liminal)
* [Liminal build](#Liminal-build)
* [Liminal create](#Liminal-create)
* [Liminal deploy](#Liminal-deploy)
* [Liminal start](#Liminal-start)
* [Liminal YAML walkthrough](#Liminal-YAML-walkthrough)
* [Evaluate the Iris Classification model](#Evaluate-the-iris-classification-model)
* [Debugging Kubernetes Deployments](#Debugging-Kubernetes-Deployments)
* [Closing up](#Closing-up)
In this tutorial, we will guide you through setting up Apache Liminal on your local machine and run a simple machine-learning workflow, based on the classic Iris dataset classification example. \
More details in this [link](https://scikit-learn.org/stable/auto_examples/datasets/plot_iris_dataset.html).
#### Prerequisites
* [Python 3 (3.6 and up)](https://www.python.org/downloads)
* [Python Virtual Environments](https://pypi.org/project/virtualenv)
* [Docker Desktop](https://www.docker.com/products/docker-desktop)
* [Kubernetes CLI (kubectl)](https://kubernetes.io/docs/tasks/tools/install-kubectl-macos)
*Note: Make sure kubernetes cluster is running in docker desktop*
We will define the following steps and services to implement the Iris classification example: \
Train, Validate & Deploy - Training and validation execution is managed by Liminal Airflow extension. The training task trains a regression model using a public dataset. \
We then validate the model and deploy it to a model-store in mounted volume. \
Inference - online inference is done using a Python Flask service running on the local Kubernetes in docker desktop. The service exposes the `/predict` endpoint. It reads the model stored in the mounted drive and uses it to evaluate the request.
## Setup your local env environment
In the dev folder, clone the example code from liminal:
```BASH
git clone https://github.com/apache/incubator-liminal
```
***Note:*** *You just cloned the entire Liminal Project, you actually only need examples folder.*
Create a python virtual environment to isolate your runs:
```BASH
cd incubator-liminal/examples/aws-ml-app-demo
python3 -m venv env
```
Activate your virtual environment:
```BASH
source env/bin/activate
```
Now we are ready to install liminal:
```BASH
pip install apache-liminal
```
## Setup liminal
### Liminal build
The build will create docker images based on the liminal.yml file in the `images` section and will create a kubernetes local volume.
Be informed that all tasks use a mounted volume as defined in the pipeline YAML. \
In our case the mounted volume will point to the liminal Iris Classification example.
The training task trains a regression model using a public dataset. We then validate the model and deploy it to a model-store in the mounted volume.
```BASH
liminal build
```
### Liminal deploy
The deploy command deploys a liminal server and deploys any liminal.yml files in your working directory or any of its subdirectories to your liminal home directory.
```BASH
liminal deploy --clean
```
*Note: liminal home directory is located in the path defined in LIMINAL_HOME env variable.
If the LIMINAL_HOME environemnet variable is not defined, home directory defaults to
~/liminal_home directory.*
### Liminal start
The start command spins up 3 containers that load the Apache Airflow stack. Liminal's Airflow extension is responsible to execute the workflows defined in the liminal.yml file as standard Airflow DAGs.
```BASH
liminal start
```
It runs the following three containers:
* liminal-postgress
* liminal-webserver
* liminal-scheduler
Once liminal server has completed starting up, you can navigate to admin UI in your browser:
[http://localhost:8080](http://localhost:8080)
***Important:** Set off/on toggle to activate your pipeline (DAG), nothing will happen otherwise!*
You can go to graph view to see all the tasks configured in the liminal.yml file:
[http://localhost:8080/admin/airflow/graph?dag_id=my_datascience_pipeline](
http://localhost:8080/admin/airflow/graph?dag_id=my_datascience_pipeline
)
#### Now lets see what actually happened to our task:
#### Click on “train” and you will get this popup:
#### Click on “view log” button and you can see the log of the current task run:
## Liminal YAML walkthrough
* [Mounted volumes](#Mounted-volumes)
* [Pipeline flow](#Pipeline-flow)
### Mounted volumes
Declaration of the mounted volume in your liminal YAML:
```YAML
name: MyDataScienceApp
owner: Bosco Albert Baracus
volumes:
- volume: gettingstartedvol
claim_name: gettingstartedvol-pvc
local:
path: .
```
### Pipeline flow
Declaration of the pipeline tasks flow in your liminal YAML:
```YAML
pipelines:
- pipeline: my_datascience_pipeline
...
schedule: 0 * 1 * *
tasks:
- task: train
type: python
description: train model
image: myorg/mydatascienceapp
cmd: python -u training.py train
...
- task: validate
type: python
description: validate model and deploy
image: myorg/mydatascienceapp
cmd: python -u training.py validate
...
```
##### Each task will internally mount the volume defined above to an internal representation, described under the task section in the yml:
```YAML
pipelines:
...
tasks:
- task: train
...
env:
MOUNT_PATH: /mnt/gettingstartedvol
mounts:
- mount: mymount
volume: gettingstartedvol
path: /mnt/gettingstartedvol
```
###### We specify the `MOUNT_PATH` in which we store the trained model.
## Evaluate the iris classification model
Once the Iris Classification model trainging is completed and model is deployed (to the mounted volume), you can launch a pod of the pre-built image which contains a flask server, by applying the following Kubernetes manifest configuration:
```BASH
kubectl apply -f manifests/aws-ml-app-demo.yaml
```
Alternatively, create a Kubernetes pod from stdin:
```YAML
cat <<EOF | kubectl apply -f -
---
apiVersion: v1
kind: Pod
metadata:
name: aws-ml-app-demo
spec:
volumes:
- name: task-pv-storage
persistentVolumeClaim:
claimName: gettingstartedvol-pvc
containers:
- name: task-pv-container
imagePullPolicy: Never
image: myorg/mydatascienceapp
lifecycle:
postStart:
exec:
command: ["/bin/bash", "-c", "apt update && apt install curl -y"]
ports:
- containerPort: 80
name: "http-server"
volumeMounts:
- mountPath: "/mnt/gettingstartedvol"
name: task-pv-storage
EOF
```
Check that the service is running:
```BASH
kubectl get pods --namespace=default
```
Check that the service is up:
```BASH
kubectl exec -it --namespace=default aws-ml-app-demo -- /bin/bash -c "curl localhost/healthcheck"
```
Check the prediction:
```BASH
kubectl exec -it --namespace=default aws-ml-app-demo -- /bin/bash -c "curl -X POST -d '{\"petal_width\": \"2.1\"}' localhost/predict"
```
## Debugging Kubernetes Deployments
kubectl get pods will help you check your pod status:
```BASH
kubectl get pods --namespace=default
```
kubectl logs will help you check your pods log:
```BASH
kubectl logs --namespace=default aws-ml-app-demo
```
kubectl exec to get a shell to a running container:
```BASH
kubectl exec --namespace=default aws-ml-app-demo -- bash
```
Then you can check the mounted volume `df -h` and to verify the result of the model.
## Here are the entire list of commands, if you want to start from scratch:
```
git clone https://github.com/apache/incubator-liminal
cd examples/aws-ml-app-demo
python3 -m venv env
source env/bin/activate
rm -rf ~/liminal_home
pip uninstall apache-liminal
pip install apache-liminal
Liminal build
Liminal create
liminal deploy --clean
liminal start
```
## Closing up
To make sure liminal containers are stopped use:
```
liminal stop
```
To deactivate the python virtual env use:
```
deactivate
```
To terminate the kubernetes pod:
```
kubectl delete pod --namespace=default aws-ml-app-demo
```
================================================
FILE: docs/getting-started/spark_app_demo.md
================================================
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
# Getting started / ***Spark application***
* [Setup your local environment](#Setup-your-local-environment)
* [Spark on K8S](#Spark-On-K8S)
* [Setup liminal](#setup-liminal)
* [Liminal YAML walkthrough](#Liminal-YAML-walkthrough)
* [Evaluate the Iris Classification model](#Evaluate-the-iris-classification-model)
* [Debugging Kubernetes Deployments](#Debugging-Kubernetes-Deployments)
* [Closing up](#Closing-up)
In this tutorial, we will guide you through setting up Apache Liminal on your local machine and run
a simple machine-learning workflow, based on the classic Iris dataset classification example
including a feature engineering with Spark engine. \
More details in
this [link](https://scikit-learn.org/stable/auto_examples/datasets/plot_iris_dataset.html).
#### Prerequisites
* [Python 3 (3.6 and up)](https://www.python.org/downloads)
* [Python Virtual Environments](https://pypi.org/project/virtualenv)
* [Docker Desktop](https://www.docker.com/products/docker-desktop)
* [Kubernetes CLI (kubectl)](https://kubernetes.io/docs/tasks/tools/install-kubectl-macos)
*Note: Make sure kubernetes cluster is running in docker desktop*
## Setup your local env environment
In the dev folder, clone the example code from liminal:
```BASH
git clone https://github.com/apache/incubator-liminal
```
***Note:*** *You just cloned the entire Liminal Project, you actually only need examples folder.*
Create a python virtual environment to isolate your runs:
```BASH
cd incubator-liminal/examples/spark-app-demo/
python3 -m venv env
```
Activate your virtual environment:
```BASH
source env/bin/activate
```
Now we are ready to install liminal:
```BASH
pip install apache-liminal
```
## Spark On K8S
We will define the following steps and services to implement the Iris classification example
including a simple feature engineering with Apache Spark: \
Clean data, Train, Validate & Deploy - Cleaning the input data set and prepare it for training and
validation execution is managed by Liminal Airflow extension. The training task trains a regression
model using a public dataset. \
We then validate the model and deploy it to a model-store in mounted volume. \
Inference - online inference is done using a Python Flask service running on the local Kubernetes in
docker desktop. The service exposes the `/predict` endpoint. It reads the model stored in the
mounted drive and uses it to evaluate the request.
### Setup liminal
```BASH
cd incubator-liminal/examples/spark-app-demo/k8s
```
#### Liminal build
The build will create docker images based on the liminal.yml file in the `images` section and will create a kubernetes local volume.
Be informed that all tasks use a mounted volume as defined in the pipeline YAML. \
In our case the mounted volume will point to the liminal Iris Classification example.
The training task trains a regression model using a public dataset. We then validate the model and deploy it to a model-store in the mounted volume.
```BASH
liminal build
```
#### Liminal deploy
The deploy command deploys a liminal server and deploys any liminal.yml files in your working
directory or any of its subdirectories to your liminal home directory.
```BASH
liminal deploy --clean
```
*Note: liminal home directory is located in the path defined in LIMINAL_HOME env variable. If the
LIMINAL_HOME environemnet variable is not defined, home directory defaults to
~/liminal_home directory.*
#### Liminal start
The start command spins up 3 containers that load the Apache Airflow stack. Liminal's Airflow
extension is responsible to execute the workflows defined in the liminal.yml file as standard
Airflow DAGs.
```BASH
liminal start
```
You can go to graph view to see all the tasks configured in the liminal.yml file:
[http://localhost:8080/admin/airflow/graph?dag_id=my_first_spark_pipeline](
http://localhost:8080/admin/airflow/graph?dag_id=my_first_spark_pipeline
)
You should see the following dag:
### Liminal YAML walkthrough
* [Local archetype](#Local-archetype)
* [Pipeline flow](#Pipeline-flow)
#### Local archetype
A superliminal for an easy local development
```YAML
name: InfraSpark
owner: Bosco Albert Baracus
type: super
executors:
- executor: k8s
type: kubernetes
variables:
output_root_dir: /mnt/gettingstartedvol
input_root_dir: ''
images:
- image: my_spark_image
type: spark
source: .
no_cache: True
task_defaults:
spark:
executor: k8s
executors: 2
application_source: '{{application}}'
mounts:
- mount: mymount
volume: gettingstartedvol
path: /mnt/gettingstartedvol
```
###### We specify the `MOUNT_PATH` in which we store the trained model.
You can read more about `superliminal` `variables` and `defaults`
in [advanced.liminal.yml](../liminal/advanced.liminal.yml.md)
#### Pipeline flow
Declaration of the pipeline tasks flow in your liminal YAML:
```YAML
name: MyFirstLiminalSparkApp
super: InfraSpark
owner: Bosco Albert Baracus
variables:
output_path: '{{output_root_dir}}/my_first_liminal_spark_app_outputs/'
application: data_cleanup.py
task_defaults:
python:
mounts:
- mount: mymount
volume: gettingstartedvol
path: /mnt/gettingstartedvol
pipelines:
- pipeline: my_first_spark_pipeline
start_date: 1970-01-01
timeout_minutes: 45
schedule: 0 * 1 * *
tasks:
- task: data_preprocessing
type: spark
description: prepare the data for training
application_arguments:
- '{{input_root_dir}}data/iris.csv'
- '{{output_path}}'
- task: train
type: python
description: train model
image: myorg/mydatascienceapp
cmd: python -u training.py train '{{output_path}}'
env:
MOUNT_PATH: /mnt/gettingstartedvol
...
```
### Evaluate the iris classification model
Once the Iris Classification model trainging is completed and model is deployed (to the mounted
volume), you can launch a pod of the pre-built image which contains a flask server, by applying the
following Kubernetes manifest configuration:
```BASH
kubectl apply -f manifests/spark-app-demo.yaml
```
Alternatively, create a Kubernetes pod from stdin:
```YAML
cat <<EOF | kubectl apply -f -
---
apiVersion: v1
kind: Pod
metadata:
name: spark-app-demo
spec:
volumes:
- name: task-pv-storage
persistentVolumeClaim:
claimName: gettingstartedvol-pvc
containers:
- name: task-pv-container
imagePullPolicy: Never
image: myorg/mydatascienceapp
lifecycle:
postStart:
exec:
command: [ "/bin/bash", "-c", "apt update && apt install curl -y" ]
ports:
- containerPort: 80
name: "http-server"
volumeMounts:
- mountPath: "/mnt/gettingstartedvol"
name: task-pv-storage
EOF
```
Check that the service is running:
```BASH
kubectl get pods --namespace=default
```
Check that the service is up:
```BASH
kubectl exec -it --namespace=default spark-app-demo -- /bin/bash -c "curl localhost/healthcheck"
```
Check the prediction:
```BASH
kubectl exec -it --namespace=default spark-app-demo -- /bin/bash -c "curl -X POST -d '{\"petal_width\": [1.1,1.1,1.1,1.1]}' localhost/predict"
```
## Debugging Kubernetes Deployments
kubectl get pods will help you check your pod status:
```BASH
kubectl get pods --namespace=default
```
kubectl logs will help you check your pods log:
```BASH
kubectl logs --namespace=default spark-app-demo
```
kubectl exec to get a shell to a running container:
```BASH
kubectl exec --namespace=default spark-app-demo -- bash
```
Then you can check the mounted volume `df -h` and to verify the result of the model.
You can go to graph view to see all the tasks configured in the liminal.yml file:
[http://localhost:8080/admin/airflow/graph?dag_id=my_first_pipeline](
http://localhost:8080/admin/airflow/graph?dag_id=my_first_pipeline
## Here are the entire list of commands, if you want to start from scratch:
```
git clone https://github.com/apache/incubator-liminal
cd examples/spark-app-demo/k8s
# cd examples/spark-app-demo/emr
python3 -m venv env
source env/bin/activate
rm -rf ~/liminal_home
pip uninstall apache-liminal
pip install apache-liminal
Liminal build
Liminal create
liminal deploy --clean
liminal start
```
## Closing up
To make sure liminal containers are stopped use:
```
liminal stop
```
To deactivate the python virtual env use:
```
deactivate
```
To terminate the kubernetes pod:
```
kubectl delete pod --namespace=default spark-app-demo
```
================================================
FILE: docs/index.rst
================================================
.. Apchae Liminal documentation master file, created by
sphinx-quickstart on Sun Nov 15 08:45:27 2020.
You can adapt this file completely to your liking, but it should at least
contain the root `toctree` directive.
..
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
..
.. http://www.apache.org/licenses/LICENSE-2.0
..
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
..
Apache Liminal
==============
Apache Liminal is a data systems orchestration platform. Liminal enables data scientists and data
engineers to define their data systems and flow using configuration. From feature engineering to
production monitoring - and the framework takes care of the infra behind the scenes and seamlessly
integrates with the infrastructure behind the scenes.
.. toctree::
:maxdepth: 1
getting_started
liminal/index
architecture
Indices and tables
==================
* :ref:`genindex`
* :ref:`modindex`
* :ref:`search`
================================================
FILE: docs/liminal/README.md
================================================
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
# Apache Liminal Documentation
## Overview
Apache Liminal is an end-to-end platform for data engineers & scientists, allowing them to build,
train and deploy machine learning models in a robust and agile way.
The platform provides the abstractions and declarative capabilities for
data extraction & feature engineering followed by model training and serving.
Liminal's goal is to operationalize the machine learning process, allowing data scientists to
quickly transition from a successful experiment to an automated pipeline of model training,
validation, deployment and inference in production, freeing them from engineering and
non-functional tasks, and allowing them to focus on machine learning code and artifacts.
## Chapters
1. [liminal.yml](liminal.yml.md)
2. [Images](images)
3. [Pipelines](pipelines.md)
4. [Tasks](tasks)
5. [Services](services.md)
6. [Monitoring](monitoring.md)
7. [Metrics Backends](metrics_backends)
8. [Alerts Backends](alerts_backends)
9. [Advanced liminal.yml](advanced.liminal.yml.md)
10. [Executors](executors)
================================================
FILE: docs/liminal/advanced.liminal.yml.md
================================================
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
# Advanced liminal.yml
In this section you will learn about advanced features of liminal.yml
## Variables
Much like in programming languages, you can define variables for re-use across your liminal.yml
file.
```yaml
variables:
myvar1: myvalue1
myvar2: myvalue2
```
In the `variables` section of your liminal.yml you can define your variables as key value pairs
## Placeholders
You can use placeholders of format `{{myplaceholder}}` in most any string value in your liminal.yml
Make sure that any string that includes placeholders is surrounded by single or double quotes.
For example:
```yaml
variables:
image_name: myorg/myrepo:myapp
images:
- image: '{{image_name}}'
type: python
source: .
pipelines:
- pipeline: my_pipeline
tasks:
- task: my_python_task
type: python
image: "{{image_name}}"
cmd: python -u my_module.py
```
### Placeholders rendering
The process of rendering placeholders checks for values in several places, by order.
#### Pipeline placeholder rendering
When running pipelines placeholders will be rendered by replacing values from sources in the
following order:
1. Current [DAG run conf](https://airflow.apache.org/docs/apache-airflow/stable/dag-run.html)
(Airflow).
2. liminal.yml [variables](#variables) section.
3. [Airflow variables](https://airflow.apache.org/docs/apache-airflow/stable/howto/variable.html)
(Airflow).
4. [Airflow macros](https://airflow.apache.org/docs/apache-airflow/stable/macros-ref.html) (Airflow)
. For example: `"{{yesterday_ds}}"`. For more information see:
5.
Airflow [Jinja Templating](https://airflow.apache.org/docs/apache-airflow/stable/concepts.html#jinja-templating)
(Airflow).
#### Build placeholder rendering
When using `liminal build` placeholders will be rendered by replacing values from sources in the
following order:
1. Environment variables.
2. liminal.yml [variables](#variables) section.
## Task variables
In addition to the variables section you can also set specific variables for specific `task`s using
the `variables`
attribute of that task. For example:
```yaml
- task: my_task
type: python
cmd: python -u my_module.py
variables:
myvar1: myvalue1
myvar2: myvalue2
```
You can also pass a reference to a variable map (dictionary) set in your variables section:
```yaml
variables:
my_variable_map:
myvar1: myvalue1
myvar2: myvalue2
my_variable_map2:
myvar1: myvalue_a
myvar2: myvalue_x
pipelines:
- pipeline: my_pipeline
tasks:
- task: my_task1
type: python
cmd: python -u my_module.py
variables: my_variable_map1
- task: my_task2
type: python
cmd: python -u my_module.py
variables: my_variable_map2
```
## Executors
For fully detailed information on executors see: [executors](executors).
Each `task` in your pipelines has a reference to an executor from the `executors` section of your
liminal.yml file. If not specified, the appropriate default executor for that task type is used.
```yaml
executors:
- executor: my_kubernetes_executor
type: kubernetes
resources:
request_memory: 128Mi
pipelines:
pipeline: my_pipeline
tasks:
- task: my_python_task
type: python
image: myorg/myrepo:mypythonapp
executor: my_kubernetes_executor
cmd: python -u my_module.py
```
In the example above we define an `executor` of type `kubernetes` with custom resources
configuration.
`executors` is a section in the root of your liminal.yml file and is a list of `executor`s defined
by the following attributes:
### executor attributes
`executor`: name of your executor.
`type`: type of the executor. The current available image types are: `kubernetes` and `emr`.
Different executor types support their own additional configuration.
## Task Defaults
`task_defaults` is a section in the root of your liminal.yml file in which default attributes can be
set for each `task`
type.
```yaml
task_defaults:
python:
executor: my_kubernetes_executor
env_vars:
env: {{env}}
foo: bar
```
In the example above we set default attributes for any `python` task in any pipeline in the liminal
system defined by our liminal.yml file. Each `python` task that does not set these attributes will
default to the setting in `task_defaults`.
If the same attribute is defined in both `task_defaults` and in the `task`, definitions from the
`task` take precedence. If a map (dictionary) of values (for example `env_vars`) is defined in both
`task_defaults` the two maps will be merged, with definitions in the `task` taking precedence in
case key is defined in both maps.
## Pipeline Defaults
`pipeline_defaults` is a section in the root of your liminal.yml file in which default attributes
can be set for each `pipeline` in the liminal system defined in your liminal.yml file.
```yaml
pipeline_defaults:
start_date: 2021-01-01
timeout_minutes: 30
```
In the example above we set default attributes for any `pipeline` defined in our liminal.yml file.
Each `pipeline` that does not set these attributes will default to the setting in
`pipeline_defaults`.
If the same attribute is defined in both `pipeline_defaults` and in the `pipeline`, definitions from
the `pipeline` take precedence.
If `tasks` section is defined in `pipeline_defaults` each pipeline defined in our liminal.yml file
will have the tasks defined in `pipeline_defaults`.
A special `task` type `pipeline` may be used in `pipeline_defaults` `tasks` section. This task type
is interpreted as "
tasks defined in pipeline go here". This allows flexibility of defining common tasks to be run
before and after the tasks of each pipeline defined in our liminal.yml file. For example:
```yaml
pipeline_defaults:
before_tasks:
- task: my_common_setup_task
type: python
image: myorg/myrepo:mypythonapp
cmd: python -u my_setup_module.py
after_tasks:
- task: my_common_teardown_task1
type: python
image: myorg/myrepo:mypythonapp
cmd: python -u my_teardown_module1.py
- task: my_common_teardown_task2
type: python
image: myorg/myrepo:mypythonapp
cmd: python -u my_teardown_module2.py
```
In the example above we set a list of `tasks` in `pipeline_defaults` which leads to each pipeline
defined in our liminal.yml file will have `my_common_setup_task` run before its tasks and
`my_common_teardown_task1` and `my_common_teardown_task2` after its tasks.
## Inheritence
Each liminal.yml defines a subliminal layer of a liminal system. A subliminal layer can inherit
attributes of a superliminal layer by specifying `super: name_of_super` in the root of the yml.
```yaml
name: my_system
super: my_super
```
A super is found by this name if a liminal.yml file in your environment exists with that name. A
superliminal layer liminal.yml file needs to define its `type` as `super`:
```yaml
name: my_super
type: super
```
If not specified, the `type` of a liminal.yml file defaults to `sub` (subliminal).
A superliminal can also inherit from another superliminal by defining its own `super` attribute.
```yaml
name: my_super
type: super
super: my_other_super
```
A liminal system can have 1 subliminal layer but many superliminal layers using inheritence. This
allows us to chain common behaviors of our systems into several superliminal layers.
If no `super` is defined for a liminal.yml file then it defaults to having the
[base](https://github.com/incubator-liminal/liminal/core/config/defaults/base/liminal.yml)
be its super.
### superliminal attribtues
A superliminal layer can define the following attributes:
```
variables
executors
monitoring
task_defaults
pipeline_defaults
```
If the same attribute section is defined in both a superliminal and a lower layer of the system they
will be merged, with key collisions favoring the lower level layer.
================================================
FILE: docs/liminal/executors/README.md
================================================
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
# Executors
Each `task` in your pipelines has a reference to an executor from the `executors` section of your
liminal.yml file. If not specified, the appropriate default executor for that task type is used.
```yaml
executors:
- executor: my_kubernetes_executor
type: kubernetes
resources:
request_memory: 128Mi
pipelines:
pipeline: my_pipeline
tasks:
- task: my_python_task
type: python
image: myorg/myrepo:mypythonapp
executor: my_kubernetes_executor
cmd: python -u my_module.py
```
In the example above we define an `executor` of type `kubernetes` with custom resources
configuration.
`executors` is a section in the root of your liminal.yml file and is a list of `executor`s defined
by the following attributes:
## executor attributes
`executor`: name of your executor.
`type`: type of the executor. The current available image types are: `kubernetes` and `emr`.
Different executor types support their own additional configuration.
## task types
1. [kubernetes](kubernetes.md)
2. [emr](emr.md)
================================================
FILE: docs/liminal/executors/emr.md
================================================
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
# emr executor
The `emr` executor allows you to run tasks on AWS EMR cluster.
```yaml
- executor: my_emr_cluster
type: emr
cluser_name: liminal-cluster
```
## attributes
`cluster_name`: the name of the cluster uses by the executor
OR
`cluster_id`: the unique identifier of the cluster used by the executor.
`aws_conn_id`: a reference to the emr connection. default: `aws_default`
`cluster_states`: the cluster state filters to apply when searching for an existing cluster.
default: `['RUNNING', 'WAITING']`
`properties`: any attribute of [aws-resource-emr-step-properties](
https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-emr-step.html#aws-resource-emr-step-properties)
(Optional)
## supported task types
- [spark](../tasks/spark.md)
- [sql](../tasks/sql.md)
================================================
FILE: docs/liminal/executors/index.rst
================================================
.. Apchae Liminal documentation master file, created by
sphinx-quickstart on Sun Nov 15 08:45:27 2020.
You can adapt this file completely to your liking, but it should at least
contain the root `toctree` directive.
..
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
..
.. http://www.apache.org/licenses/LICENSE-2.0
..
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
..
Executors
=========
Each ``task`` in your pipelines has a reference to an executor from the ``executors`` section of
your liminal.yml file. If not specified, the appropriate default executor for that task type is
used.
.. code-block:: yaml
executors:
- executor: my_kubernetes_executor
type: kubernetes
resources:
request_memory: 128Mi
pipelines:
pipeline: my_pipeline
tasks:
- task: my_python_task
type: python
image: myorg/myrepo:mypythonapp
executor: my_kubernetes_executor
cmd: python -u my_module.py
..
In the example above we define an ``executor`` of type ``kubernetes`` with custom resources
configuration.
``executors`` is a section in the root of your liminal.yml file and is a list of ``executor`` s
defined by the following attributes:
executor attributes
'''''''''''''''''''
``executor``: name of your executor.
``type``: type of the executor. The current available image types are: ``kubernetes`` and ``emr``.
Different executor types support their own additional configuration.
executor types
''''''''''''''
.. toctree::
:maxdepth: 1
kubernetes
================================================
FILE: docs/liminal/executors/kubernetes.md
================================================
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
# kubernetes executor
The `kubernetes` executor allows you to run tasks on kubernetes.
```yaml
- executor: my_kubernetes_executor
type: kubernetes
```
## attributes
If running pipelines on Airflow, any attribute of
[KubernetesPodOperator](https://airflow.apache.org/docs/apache-airflow/1.10.12/_api/airflow/contrib/operators/kubernetes_pod_operator/index.html)
can be set as an attribute of the executor.
Note that some of these attributes are set in the `task`:
```
image
cmd
mounts
name
env_vars
```
For example, see: [python task](../tasks/python.md)
================================================
FILE: docs/liminal/extensibility.md
================================================
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
# Extensibility
The extensibility designed to have an easy way for the users to add their tasks, executors, image
builders and extend the library so that it fits the level of abstraction that suits the user
environment.
## Location
Tasks folder: `{LIMINAL_HOME}/plugins/tasks`
Executors folder: `{LIMINAL_HOME}/plugins/executors`
Image builders folder: `{LIMINAL_HOME}/plugins/images`
## Example
### Prerequisites
Apache Liminal
### Guide
Check out the examples for each one of the extensible item
in [examples/extensibility](../../examples/extensibility)
Copy the extensible items to the plugin location:
```shell
cp -r ../../examples/extensibility/executors/* $LIMINAL_HOME/liminal/plugins/executors/
```
```shell
cp -r ../../examples/extensibility/tasks/* $LIMINAL_HOME/liminal/plugins/tasks/
```
```shell
cp -r ../../examples/extensibility/images/* $LIMINAL_HOME/liminal/plugins/images/
```
```shell
liminal build .
liminal deploy --clean
liminal start
```
================================================
FILE: docs/liminal/images/README.md
================================================
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
# Images
In the `images` section you can configure how to pack your code into docker images. This can be
achieved in liminal easily, without any Docker knowledge needed by setting just a few attributes:
```yaml
images:
- image: myorg/myrepo:mypythonapp
type: python
source: .
- image: myorg/myrepo:myserver
type: python_server
source: path/to/my/server/code
endpoints:
- endpoint: /myendpoint
module: my_module
```
`images` is a section in the root lof your liminal.yml file and is a list of `image`s, defined
by the following attributes:
## image attributes
`image`: name of your image, usually will be in the form of `user/repository:tag` common in docker
image repositories such as Docker Hub, but for local development this can be any string.
`type`: type of the image. Usually this will match your coding language, or your coding langauge
followed by an `_` and a specific style of application (such as a server).
`source`: location of source files to include in the image, this can be a relative path within your
project, or even `.` which means the entire project should be packaged in the image.
`build_cmd`: certain languages require to be built or compiled, here you can provide your build
command to be executed in order to build your code.
Other image types might require additional configuration, for example, servers require `endpoints`
to be configured.
## image types
1. [python](python.md)
2. [python server](python_server.md)
================================================
FILE: docs/liminal/images/index.rst
================================================
.. Apchae Liminal documentation master file, created by
sphinx-quickstart on Sun Nov 15 08:45:27 2020.
You can adapt this file completely to your liking, but it should at least
contain the root `toctree` directive.
..
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
..
.. http://www.apache.org/licenses/LICENSE-2.0
..
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
..
Images
======
In the ``images`` section you can configure how to pack your code into docker images. This can be
achieved in liminal easily, without any Docker knowledge needed by setting just a few attributes:
.. code-block:: yaml
images:
- image: myorg/myrepo:mypythonapp
type: python
source: .
- image: myorg/myrepo:myserver
type: python_server
source: path/to/my/server/code
endpoints:
- endpoint: /myendpoint
module: my_module
..
``images`` is a section in the root lof your liminal.yml file and is a list of
``image`` s, defined by the following attributes:
image attributes
''''''''''''''''
``image``: name of your image, usually will be in the form of ``user/repository:tag`` common in
docker image repositories such as Docker Hub, but for local development this can be any string.
``type``: type of the image. Usually this will match your coding language, or your coding langauge
followed by an ``_`` and a specific style of application (such as a server).
``source``: location of source files to include in the image, this can be a relative path within
your project, or even ``.`` which means the entire project should be packaged in the image.
``build_cmd``: certain languages require to be built or compiled, here you can provide your build
command to be executed in order to build your code.
Other image types might require additional configuration, for example, servers require ``endpoints``
to be configured.
image types
'''''''''''
.. toctree::
:maxdepth: 1
python
python_server
================================================
FILE: docs/liminal/images/python.md
================================================
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
# python image
The `python` image builder packages your python code as a docker image.
```yaml
- image: myorg/myrepo:mypythonapp
type: python
source: relative/path/to/source
```
## attributes
`image`: name of your image.
`source`: location of source files to include in the image, this can be a relative path within your
project, or even `.` which means the entire project should be packaged in the image.
================================================
FILE: docs/liminal/images/python_server.md
================================================
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
# python_server image
The `python_server` image builder packages your python code as a docker image which starts a web
server to serve your code via HTTP calls.
```yaml
- image: myorg/myrepo:mypythonserver
type: python_server
source: relative/path/to/source
endpoints:
- endpoint: /myendpoint
module: my_module
function: my_function
- endpoint: /myotherendpoint
module: my_module2
function: my_function2
```
## attributes
`image`: name of your image.
`source`: location of source files to include in the image, this can be a relative path within your
project, or even `.` which means the entire project should be packaged in the image.
`endpoint`: list of `endpoint`s to expose in the server.
## endpoint attributes
`endpoint`: path to your endpoint in the server.
`module`: module containing your user code.
`function`: name of the function within the module that should be called when handling this
endpoint. the function should expect a dictionary as an input (or None if no input) and should
return a string to return in the response to the HTTP call.
================================================
FILE: docs/liminal/index.rst
================================================
.. Apchae Liminal documentation master file, created by
sphinx-quickstart on Sun Nov 15 08:45:27 2020.
You can adapt this file completely to your liking, but it should at least
contain the root `toctree` directive.
..
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
..
.. http://www.apache.org/licenses/LICENSE-2.0
..
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
..
Apache Liminal Documentation
============================
Chapters:
.. toctree::
:maxdepth: 1
liminal.yml
images/index.rst
pipelines
tasks/index.rst
services
monitoring
metrics_backends/index.rst
advanced.liminal.yml
executors/index.rst
================================================
FILE: docs/liminal/kubernetes/secret_util.md
================================================
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
# Secret Utils
Each `task` in your pipelines has a reference to an secret from the `secrets` section of your
liminal.yml file.
```yaml
---
name: k8s_secret_example
secrets:
- secret: aws
local_path_file: "~/.aws/credentials"
executors:
- executor: k8s
type: kubernetes
variables:
AWS_CONFIG_FILE: /mnt/credentials
task_defaults:
python:
executor: k8s
image: amazon/aws-cli:2.7.23
executors: 2
env_vars:
AWS_CONFIG_FILE: "/secret/credentials"
secrets:
- secret: aws
remote_path: "/secret"
pipelines:
- pipeline: k8s_secret_example
owner: Bosco Albert Baracus
start_date: 1970-01-01
timeout_minutes: 10
schedule: 0 * 1 * *
tasks:
- task: my_python_task
type: python
cmd: aws s3 ls
```
That example manifest defines a Secret Opaque for AWS credentials used. The values are Base64 strings in the manifest; however, when you use the Secret with a Pod then the kubelet provides the decoded data to the Pod and its containers.
In order to make use of the AWS credentials we define an environment variable `AWS_CONFIG_FILE` to authenticate our requests.
For example, the files generated by the AWS CLI for a default profile configured with aws configure looks similar to the following.
`~/.aws/credentials`
```
[default]
aws_access_key_id=<aws_access_key_id>
aws_secret_access_key=<aws_secret_access_key>
region = us-east-1
aws_session_token=<aws_session_token>
```
================================================
FILE: docs/liminal/liminal.yml.md
================================================
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
## liminal.yml
Definition of your own liminal system is done via yml configuration file named liminal.yml in
your project. This yml is the one place in which you define all characteristics and behavior of your
liminal system.
In this section we'll go over the anatomy of the liminal.yml file.
### root attributes
At the root level of your limimal.yml you can define the most basic root characteristics of your
liminal system such as name and owner:
```yaml
name: MyLiminalSystem
owner: Bosco Albert Baracus
```
### images
For fully detailed information on pipelines see: [images](images).
In the `images` section you can configure how to pack your code into docker images. This can be
achieved in liminal easily, without any Docker knowledge needed by setting just a few attributes:
```yaml
images:
- image: myorg/myrepo:mypythonapp
type: python
source: .
- image: myorg/myrepo:myserver
type: python_server
source: path/to/my/server/code
endpoints:
- endpoint: /myendpoint
module: my_module
function: my_function
```
`images` is a section in the root lof your liminal.yml file and is a list of `image`s, defined
by the following attributes:
#### image attributes
`image`: name of your image, usually will be in the form of `user/repository:tag` common in docker
image repositories such as Docker Hub, but for local development this can be any string.
`type`: type of the image. Usually this will match your coding language, or your coding langauge
followed by an `_` and a specific style of application (such as a server).
The current available image types are: `python`.
`source`: location of source files to include in the image, this can be a relative path within your
project, or even `.` which means the entire project should be packaged in the image.
`build_cmd`: certain languages require to be built or compiled, here you can provide your build
command to be executed in order to build your code.
Other image types might require additional configuration, for example, servers require `endpoints`
to be configured.
### pipelines
For fully detailed information on pipelines see: [pipelines](pipelines.md).
In the `pipelines` section you can configure scheduled/manually run pipelines of tasks to be
executed sequentially:
```yaml
pipelines:
- pipeline: my_data_pipeline
timeout_minutes: 30
start_date: 2020-03-01
schedule: 0 10 * * *
tasks:
- task: my_sql_task
type: sql
query: "SELECT * FROM
{{my_database_name}}.{{my_table_name}}
WHERE event_date_prt >=
'{{yesterday_ds}}'"
AND cms_platform = 'xsite'
output_table: my_db.my_out_table
output_path: s3://my_bky/{{env}}/mydir
- task: my_python_task
type: python
image: myorg/myrepo:pythonapp
cmd: python my_python_app.py
env_vars:
env: {{env}}
fizz: buzz
- task: my_python_task
type: python
image: myorg/myrepo:mypythonapp
cmd: python -u my_module.py
env_vars:
env: {{env}}
fizz: buzz
```
`pipelines` is a section in the root lof your liminal.yml file and is a list of `pipeline`s defined
by the following attributes:
#### pipeline attributes
`pipeline`: name of your pipeline (must be unique per liminal server).
`timeout_minutes`: maximum allowed pipeline run time in minutes, if run exceeds this time, pipeline
and all running tasks will fail.
`start_date`: start date for the pipeline.
`schedule`: to be configured if the pipeline should run on a schedule. Format is
[cron expression](https://en.wikipedia.org/wiki/Cron#CRON_expression).
`tasks`: list of `task`s, defined by the following attributes:
##### task attributes
For fully detailed information on tasks see: [tasks](tasks).
`task`: name of your task (must be made of alphanumeric, dash and/or underscore characters only).
`type`: type of the task. Examples of available task types are: `python`.
and more..
Different task types require their own additional configuration. For example, `python` task requires
`image` to be configured.
### services
For fully detailed information on services see: [services](services.md).
In the `services` section you can configure constantly running applications such as
servers.
```yaml
services:
- service: my_server
image: myorg/myrepo:myserver
```
`services` is a section in the root lof your liminal.yml file and is a list of `service`s, defined
by the following attributes:
#### service attributes
`service`: name of your service.
`image`: the service's docker image.
### monitoring
For fully detailed information on monitoring see: [monitoring](monitoring.md).
In the `monitoring` section you can configure monitoring for your pipelines and services.
```yaml
monitoring:
metrics_backends:
- metrics_backend: cloudwatch_metrics
type: aws_cloudwatch
namespace: DataPipeline
AWS_REGION_NAME: us-east-1
alerts_backends:
- alerts_backend: cloudwatch_alerts
type: aws_cloudwatch
metrics_backend: cloudwatch_metrics
ok_actions: ['arn:aws:sns:...']
alarm_actions: ['arn:aws:sns:...']
```
`monitoring` is a section in the root lof your liminal.yml file and is a list of `metrics_backend`s and
a list of `alerts_backend`s.
`metrics_backend`s are where metrics for your pipelines are automatically sent.
`alerts_backends`s automatically register alerts based on `metrics_backend` they are paired with.
#### metrics_backend attributes
`metrics_backend`: name of your metrics backend
`type`: type of the metrics backend. The current available metrics backends are: `aws_cloudwatch`.
Different metrics backend types require their own additional configuration. For example,
`aws_cloudwatch` metrics backend requires `namespace` to be configured.
#### alerts_backend attributes
`alerts_backend`: name of your alerts backend
`type`: type of the alerts backend. The current available alerts backends are: `aws_cloudwatch`.
`metrics_backend`: name of the metrics backends to register alerts for.
Different alerts backend types require their own additional configuration. For example,
`aws_cloudwatch` alerts backend requires `alarm_actions` to be configured.
### advanced topics
For documentation of more advanced features of liminal.yml see:
[advanced liminal.yml](advanced.liminal.yml.md)
================================================
FILE: docs/liminal/metrics_backends/README.md
================================================
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
# Metrics Backends
`metrics_backend` is the definition of a metrics backend to which all automatically
generated metrics from your pipeline are sent to and is part of the `metrics_backends` list in the
`monitoring` section of your liminal.yml
```yaml
- metrics_backend: cloudwatch_metrics
type: aws_cloudwatch
namespace: DataPipeline
AWS_REGION_NAME: us-east-1
```
A `metrics_backend` is defined by the following attribtues:
## metrics_backend attributes
`metrics_backend`: name of your metrics backend
`type`: type of the metrics backend. The current available metrics backends are: `aws_cloudwatch`.
Different metrics backend types require their own additional configuration. For example,
`aws_cloudwatch` metrics backend requires `namespace` to be configured.
## metrics_backend types
1. [aws cloudwatch](aws_cloudwatch.md)
================================================
FILE: docs/liminal/metrics_backends/aws_cloudwatch.md
================================================
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
# aws_cloudwatch metrics backend
The `aws_cloudwatch` metrics backend allows you to send all automatically generated metrics from
your pipelines to AWS CloudWatch.
```yaml
- metrics_backend: cloudwatch_metrics
type: aws_cloudwatch
namespace: DataPipeline
AWS_REGION_NAME: us-east-1
```
## attributes
`metrics_backend`: name of your metrics backend.
`type`: type of the metrics backend. The current available metrics backends are: `aws_cloudwatch`.
`namespace`: target namespace on AWS CloudWatch.
`AWS_REGION_NAME`: target AWS region to report metrics to.
`AWS_ACCESS_KEY_ID`: AWS access key id (optional).
`AWS_SECRET_ACCESS_KEY`: AWS secret access key (optional).
================================================
FILE: docs/liminal/metrics_backends/index.rst
================================================
.. Apchae Liminal documentation master file, created by
sphinx-quickstart on Sun Nov 15 08:45:27 2020.
You can adapt this file completely to your liking, but it should at least
contain the root `toctree` directive.
..
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
..
.. http://www.apache.org/licenses/LICENSE-2.0
..
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
..
Metrics Backends
================
``metrics_backend`` is the definition of a metrics backend to which all automatically
generated metrics from your pipeline are sent to and is part of the ``metrics_backends`` list in the
``monitoring`` section of your liminal.yml
.. code-block:: yaml
- metrics_backend: cloudwatch_metrics
type: aws_cloudwatch
namespace: DataPipeline
AWS_REGION_NAME: us-east-1
..
A ``metrics_backend`` is defined by the following attributes:
metrics_backend attributes
''''''''''''''''''''''''''
``metrics_backend``: name of your metrics backend
``type``: type of the metrics backend. The current available metrics backends are: ``aws_cloudwatch``.
Different metrics backend types require their own additional configuration. For example,
``aws_cloudwatch`` metrics backend requires ``namespace`` to be configured.
metrics_backend types
'''''''''''''''''''''
.. toctree::
:maxdepth: 1
aws_cloudwatch
================================================
FILE: docs/liminal/monitoring.md
================================================
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
# Monitoring
In the `monitoring` section you can configure monitoring for your pipelines and services.
```yaml
monitoring:
metrics_backends:
- metrics_backend: cloudwatch_metrics
type: aws_cloudwatch
namespace: DataPipeline
AWS_REGION_NAME: us-east-1
```
`monitoring` is a section in the root lof your liminal.yml file and is a list of `metrics_backend`s and
a list of `alerts_backend`s.
`metrics_backend`s are where metrics for your pipelines are automatically sent.
`alerts_backends`s automatically register alerts based on `metrics_backend` they are paired with.
## metrics_backend attributes
For fully detailed information on metrics backends see: [metrics backends](metrics_backends).
`metrics_backend`: name of your metrics backend
`type`: type of the metrics backend. The current available metrics backends are: `aws_cloudwatch`.
Different metrics backend types require their own additional configuration. For example,
`aws_cloudwatch` metrics backend requires `namespace` to be configured.
================================================
FILE: docs/liminal/pipelines.md
================================================
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
# Pipelines
In the `pipelines` section you can configure scheduled/manually run pipelines of tasks to be
executed sequentially:
```yaml
pipelines:
- pipeline: my_data_pipeline
timeout_minutes: 30
start_date: 2020-03-01
schedule: 0 10 * * *
tasks:
- task: my_sql_task
type: sql
query: "SELECT * FROM
{{my_database_name}}.{{my_table_name}}
WHERE event_date_prt >=
'{{yesterday_ds}}'"
AND cms_platform = 'xsite'
output_table: my_db.my_out_table
output_path: s3://my_bky/{{env}}/mydir
- task: my_python_task
type: python
image: myorg/myrepo:pythonapp
cmd: python my_python_app.py
env_vars:
env: {{env}}
fizz: buzz
```
`pipelines` is a section in the root lof your liminal.yml file and is a list of `pipeline`s defined
by the following attributes:
## pipeline attributes
`pipeline`: name of your pipeline (must be unique per liminal server).
`timeout_minutes`: maximum allowed pipeline run time in minutes, if run exceeds this time, pipeline
and all running tasks will fail.
`start_date`: start date for the pipeline.
`schedule`: to be configured if the pipeline should run on a schedule. Format is
[cron expression](https://en.wikipedia.org/wiki/Cron#CRON_expression).
`tasks`: list of `task`s, defined by the following attributes:
## task attributes
For fully detailed information on tasks see: [tasks](tasks).
`task`: name of your task (must be made of alphanumeric, dash and/or underscore characters only).
`type`: type of the task. Examples of available task types are: `python`.
and more..
Different task types require their own additional configuration. For example, `python` task requires
`image` to be configured.
================================================
FILE: docs/liminal/services.md
================================================
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
# Services
In the `services` section you can configure constantly running applications such as
servers.
```yaml
services:
- service: my_server
image: myorg/myrepo:myserver
```
`services` is a section in the root lof your liminal.yml file and is a list of `service`s, defined
by the following attributes:
## service attributes
`service`: name of your service.
`image`: the service's docker image.
================================================
FILE: docs/liminal/tasks/README.md
================================================
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
# Tasks
`task` is the definition of a specific step in your pipeline, and is part of the `tasks` list
in your pipeline definition.
For fully detailed information on pipelines see: [pipelines](../pipelines.md).
```yaml
- task: my_python_task
image: myorg/myrepo:mypythonapp
cmd: python -u my_module.py
env_vars:
env: {{env}}
fizz: buzz
```
A `task` is defined by the following attributes:
## task attributes
`task`: name of your task (must be made of alphanumeric, dash and/or underscore characters only).
`type`: type of the task. Examples of available task types are: `python`
and more..
Different task types require their own additional configuration. For example, `python` task requires
`image` to be configured.
## task types
1. [python](python.md)
2. [spark](spark.md)
3. [create_cloudformation_stack](create_cloudformation_stack.md)
4. [delete_cloudformation_stack](delete_cloudformation_stack.md)
================================================
FILE: docs/liminal/tasks/create_cloudformation_stack.md
================================================
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
# create_cloudformation_stack task
The `create_cloudformation_stack` task allows you to create cloudformation stack on AWS
```yaml
- task: create_emr
type: create_cloudformation_stack
stack_name: liminal_document_emr
properties:
OnFailure: DO_NOTHING
TimeoutInMinutes: 25
Capabilities: [ 'CAPABILITY_NAMED_IAM' ]
TemplateURL: s3://liminal-doc/emr/template.yml
Parameters:
Environment: Production
CoreServerCount: '2'
```
## attributes
`task`: name of your task (must be made of alphanumeric, dash and/or underscore characters only).
`stack_name`: the name of the stack
`properties`: any attribute of [aws-create-stack-request-parameters](
https://docs.aws.amazon.com/AWSCloudFormation/latest/APIReference/API_CreateStack.html#API_CreateStack_RequestParameters)
================================================
FILE: docs/liminal/tasks/delete_cloudformation_stack.md
================================================
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
# delete_cloudformation_stack task
The `delete_cloudformation_stack` task allows you to delete stack
```yaml
- task: delete_emr
type: delete_cloudformation_stack
stack_name: liminal_emr_stack_name
description: delete stack
```
## attributes
`task`: name of your task (must be made of alphanumeric, dash and/or underscore characters only).
`stack_name`: the name of the stack to delete
================================================
FILE: docs/liminal/tasks/index.rst
================================================
.. Apchae Liminal documentation master file, created by
sphinx-quickstart on Sun Nov 15 08:45:27 2020.
You can adapt this file completely to your liking, but it should at least
contain the root `toctree` directive.
..
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
..
.. http://www.apache.org/licenses/LICENSE-2.0
..
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
..
Tasks
=====
``task`` is the definition of a specific step in your pipeline, and is part of the ``tasks`` list
in your pipeline definition.
For fully detailed information on pipelines see: `pipelines`_.
.. _pipelines: ../pipelines.html
.. code-block:: yaml
- task: my_python_task
type: python
image: myorg/myrepo:mypythonapp
cmd: python -u my_module.py
env_vars:
env: {{env}}
fizz: buzz
..
A ``task`` is defined by the following attributes:
In the ``images`` section you can configure how to pack your code into docker images. This can be
achieved in liminal easily, without any Docker knowledge needed by setting just a few attributes:
task attributes
''''''''''''''''
``task``: name of your task (must be made of alphanumeric, dash and/or underscore characters only).
``type``: type of the task.
Different task types require their own additional configuration. For example, ``python`` task
requires ``image`` to be configured.
task types
''''''''''
.. toctree::
:maxdepth: 1
python
================================================
FILE: docs/liminal/tasks/python.md
================================================
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
# python task
The `python` task allows you to run python code packaged as docker images.
```yaml
- task: my_python_task
type: python
image: myorg/myrepo:mypythonapp
cmd: python my_python_app.py
env_vars:
env: '{{env}}'
fizz: buzz
mounts:
- mount: mymount
volume: myvol1
path: /mnt/vol1
```
## attributes
`task`: name of your task (must be made of alphanumeric, dash and/or underscore characters only).
`image`: name of image to run.
`cmd`: command to run when running the image.
`env_vars`: environment variables to set when running the image.
`mounts`: list of `mount`s defined by the following attributes:
### mount attributes
`mount`: name of the mount.
`volume`: volume to mount. volumes are defined in the `volumes` section of liminal.yml
`path`: path in which to mount the volume. this is the path accessible to user code.
================================================
FILE: docs/liminal/tasks/spark.md
================================================
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
# spark task
The `spark` task allows you to submit a spark application
```yaml
- task: my_spark_app
type: spark
executor: emr_executor
application_source: 'my_app.py',
class: 'com.mycompany.MySparkApp',
master: 'yarn',
conf:
spark.driver.memory: '1g',
spark.driver.maxResultSize: '1g',
spark.yarn.executor.memoryOverhead: '500M'
application_arguments:
--query: "select * from my_db.my_input_table where my_date_col >= "
"'{{yesterday_ds}}'",
--output: 'my_output_table'
```
## attributes
`task`: name of your task (must be made of alphanumeric, dash and/or underscore characters only).
`executor`: executor name ([supported executors](#supported-executors))
`application_source`: the location of the spark application (jar location / python file)
`class`: the entry point for your application
`master`: the cluster manager to connect to.See the list
of [allowed master URL's](https://spark.apache.org/docs/latest/submitting-applications.html#master-urls)
(Optional)
`conf`: arbitrary Spark configuration properties (Optional)
`application_arguments`: arguments passed to the main method of your main class (Optional)
## supported executors
- [emr](../executors/emr.md)
================================================
FILE: docs/make.bat
================================================
REM
REM Licensed to the Apache Software Foundation (ASF) under one
REM or more contributor license agreements. See the NOTICE file
REM distributed with this work for additional information
REM regarding copyright ownership. The ASF licenses this file
REM to you under the Apache License, Version 2.0 (the
REM "License"); you may not use this file except in compliance
REM with the License. You may obtain a copy of the License at
REM
REM http://www.apache.org/licenses/LICENSE-2.0
REM
REM Unless required by applicable law or agreed to in writing,
REM software distributed under the License is distributed on an
REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
REM KIND, either express or implied. See the License for the
REM specific language governing permissions and limitations
REM under the License.
@ECHO OFF
pushd %~dp0
REM Command file for Sphinx documentation
if "%SPHINXBUILD%" == "" (
set SPHINXBUILD=sphinx-build
)
set SOURCEDIR=.
set BUILDDIR=build
if "%1" == "" goto help
%SPHINXBUILD% >NUL 2>NUL
if errorlevel 9009 (
echo.
echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
echo.installed, then set the SPHINXBUILD environment variable to point
echo.to the full path of the 'sphinx-build' executable. Alternatively you
echo.may add the Sphinx directory to PATH.
echo.
echo.If you don't have Sphinx installed, grab it from
echo.http://sphinx-doc.org/
exit /b 1
)
%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%
goto end
:help
%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%
:end
popd
================================================
FILE: docs/source/How_to_install_liminal_in_airflow_on_kubernetes.md
================================================
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
# Install Liminal in Airflow
* [Workflow](#workflow)
* [Prerequisites](#prerequisites)
* [One time setup](#One-time-environment-setup)
* [Deploying your Yaml files](#Deploying-your-Yaml-files)
* [References and other resources](#references-and-other-resources)
## Workflow
To deploy and run liminal on airflow, we need to complete two tasks:
1. Ensure that the Liminal python package is installed inside each one of Airflow's pods (schedulers, Workers and Web Server). \
This is a standard airflow management task, which can be achieved in multiple ways depending on your setup (we will cover a few later in this guide).
2. Ensure that the user's Liminal code (i.e. Yamls) are present in each of the pods' DAGs folder (typically located in /opt/airflow/dags). \
Here, our recommended approach is to use a folder on a shared filesystem (EFS) to host the airflow DAGs (and Liminal Yamls). \
This folder will be mounted into Airflow pods' DAGs folder - thus enabling each of the pods to pick the files up and run the Liminal DAGs.
Below is a description of how to achieve this task using a commonly used Helm chart for Airflow on Kubernetes.
## Prerequisites
### Airflow on Kubernetes
This guide assumes you have successfully installed Airflow on Kubernetes.
If you haven't done so yet, we found the following Helm chart useful for achieving this goal:
[airflow-helm-chart-7.16.0]
You'll need to add the chart to your Helm installation and follow the instructions from the README.md
```sh
helm repo add airflow-stable https://airflow-helm.github.io/charts
helm repo update
```
### A provisioned EFS file system
EFS is an AWS solution which offers an NFS as a service. \
You can read up about EFS [here](https://aws.amazon.com/efs/features/) and set it up via AWS console or CLI.
## One time environment setup
### Adding the Apache Liminal package to Apache Airflow pods
There are multiple ways to install liminal into the Airflow pods, depending on how you deployed Airflow on Kubernetes. \
In principle, it requires the package to be pip-installed into the Airflow docker - either during build time or in run time.
1. If you are rolling out your own Airflow Docker images based on the [official docker image](https://github.com/apache/airflow),
you can add apache-liminal installation during the build:
```docker build --build-arg ADDITIONAL_PYTHON_DEPS="apache-liminal"```
2. If you already have a running Airflow on Kubernetes, you can run a command which iterates over Airflow pods and executes a ```pip install apache-liminal```
on each of the pods.
3. If you used the [airflow-helm-chart-7.16.0], you just need to specifiy apache-liminal inside the ```requirements.txt``` file as per the instructions [here][airflow-helm-chart-7.16.0]
### Preparing a "deployment box" and an EFS folder to host the Liminal Yaml files
The "deployment box" is the machine which has access to the Yamls you want to deploy. \
This machine will also mount the same EFS folder as Airflow, and use ```liminal deploy``` to push the Yamls into that folder.
1. Provision an EC2 instance which has access to the EFS
3. Mount the EFS onto the EC2 instance
```sudo mount -t nfs4 -o nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,noresvport <EFS_ID>:/ /mnt/efs```
3. Create a folder on an EFS drive. \
This will be our target for deployment
```mkdir -p /mnt/efs/opt/airflow/liminal_home```
4. ```export LIMINAL_HOME=/mnt/efs/opt/airflow/liminal_home```
### Mounting the EFS folder into Airflow pods
Now that we have an EFS drive, and we've created the shared folder on it, it's time to mount it onto the Airflow pods` Dags folder.
The [guide][airflowInstallation] includes details on how to expose EFS as a Persistent Volume for kubernetes.
Make sure to point the pods' PV to the right EFS folder, like so:
```
volumeMounts:
- name: efs-data
mountPath: /opt/airflow/dags
```
where efs-data is a PVC pointing to `<EFS_ID>:/opt/airflow/liminal_home`
## Deploying your Yaml files
Finally, the one time setup is done, and by this point, you should have:
1. A deployment box which can deploy Liminal to an EFS folder
2. Airflow pods which will pick up the Yamls from that folder automatically.
Deploying the actual Yaml files is the simple part.
Each time you want to deploy the Liminal Yamls from the deployment box:
```
liminal deploy --path <<<path to liminal user code>>>
```
## References and other resources
### Setting up Airflow on Kubernetes with EFS
[Setting up Airflow on Kubernetes with AWS EFS][airflowInstallation]
### Example script
In order to make it easier to undernstad all the steps, we include a script which performs the Liminal-specific steps in the deployment.
This script should be run from an EC2 machine which has access to the EFS you've provisioned. \
The below steps runs the example project of [liminal-getting-started][liminal-getting-started].
* Mount the EFS and setting environment parameters:
* [`liminal_aws_deploy.sh`][liminal-aws-deploy] -o installation
* Build dockers from your business logic:
* [`liminal_aws_deploy.sh`][liminal-aws-deploy] -o build
* Deploy the Yaml onto the EFS folder:
* [`liminal_aws_deploy.sh`][liminal-aws-deploy] -o deployment
##### Seting up AWS ECR with kubernetes
[How to configure and use AWS ECR with kubernetes][AWS-ECR-with-kubernetes]
1. `liminal_aws_deploy.sh -o build` will build dockers from the given liminal project path. \
The [liminal.yml][liminal-yml] defindes a docker image :
```
tasks:
- task: python_hello_world_example
type: python
description: static input task
image: python_hello_world_example_image
```
The Yaml defines task that will pull the docker image from your predefined registry. \
Therefore, you need to use a docker registry in your kubernetes cluster. \
One way to achieve it is to use [Amazon ECR][amazon-ecr].
[AWS-ECR-with-kubernetes]: <https://medium.com/@damitj07/how-to-configure-and-use-aws-ecr-with-kubernetes-rancher2-0-6144c626d42c>
[amazon-ecr]: <https://aws.amazon.com/ecr/>
[liminal-yml]: <https://github.com/apache/incubator-liminal/blob/master/examples/liminal-getting-started/liminal.yml>
[liminal-getting-started]: <https://github.com/apache/incubator-liminal/tree/master/examples/liminal-getting-started>
[airflow-helm-chart-7.16.0]: <https://github.com/airflow-helm/charts/tree/airflow-7.16.0>
[homebrew-kubectl]: <https://formulae.brew.sh/formula/kubernetes-cli>
[cluster-access-kubeconfig]: <https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/#context>
[liminal-aws-deploy]: <https://github.com/apache/incubator-liminal/tree/master/docs/source/liminal_aws_deploy.sh>
[airflowChart]: <https://github.com/airflow-helm/charts/tree/main/charts/airflow>
[airflowInstallation]: <https://medium.com/terragoneng/setting-up-airflow-on-kubernetes-with-aws-efs-c659f3a16292>
[airflowImage]: <https://hub.docker.com/layers/apache/airflow/1.10.12-python3.6/images/sha256-9ea9e5ca66bd17632241889ab248fe3852c9f3c830ed299a8ecaa8a13ac2082f?context=explore>
================================================
FILE: docs/source/liminal_aws_deploy.sh
================================================
#! /bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
help() {
echo "$0: Get Started"
echo "Usage: $0 -o option"
echo "Liminal available options:"
echo "install"
echo "build"
echo "deploy"
}
if [[ "$#" -lt 2 ]]; then
help
exit
fi
# Arguments processing
while getopts ":o:" opt; do #install, build, deploy
case $opt in
o)
option=$OPTARG
case $option in
install)
ACTION="install"
;;
build)
ACTION="build"
;;
deploy)
ACTION="deploy"
;;
esac
esac
done
mount_efs() {
EFS_ID=$1
echo "The EFS_ID is: ${EFS_ID}"
echo "Create /mnt/efs"
mkdir /mnt/efs
echo "Mount the /mnt/efs"
sudo mount -t nfs4 -o nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,noresvport ${EFS_ID}:/ /mnt/efs
}
install_liminal() {
echo "Installing liminal on Airflow components"
docker ps | grep k8s_airflow | awk '{print $1}' | xargs -I {} docker exec {} bash -c 'pip install apache-liminal'
echo "Installing liminal locally"
pip install apache-liminal
}
deploy_yaml() {
DEPLOY_PATH=$1
echo "The DEPLOY_PATH is: ${DEPLOY_PATH}"
echo 'Export the liminal home'
export LIMINAL_HOME=$(find /mnt/efs/ -name "liminal_home")
liminal deploy --path "${DEPLOY_PATH}"
}
build() {
BUILD_PATH=$1
echo "The BUILD_PATH is: ${BUILD_PATH}"
liminal build --path "${BUILD_PATH}"
}
case $ACTION in
install)
read -r -p "Please enter the EFS ID: " EFS_ID
mount_efs $EFS_ID
install_liminal
exit 0
;;
build)
read -r -p "Please enter the path to the liminal project: " BUILD_PATH
build $BUILD_PATH
exit 0
;;
deploy)
read -r -p "Please enter the liminal yaml path: " DEPLOY_PATH
deploy_yaml $DEPLOY_PATH
exit 0
;;
*)
help
exit
;;
esac
================================================
FILE: examples/aws-ml-app-demo/__init__.py
================================================
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
================================================
FILE: examples/aws-ml-app-demo/liminal.yml
================================================
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
---
name: MyDataScienceApp
owner: Bosco Albert Baracus
volumes:
- volume: gettingstartedvol
claim_name: gettingstartedvol-pvc
local:
path: .
images:
- image: myorg/mydatascienceapp
type: python_server
source: .
endpoints:
- endpoint: /predict
module: serving
function: predict
- endpoint: /healthcheck
module: serving
function: healthcheck
- endpoint: /version
module: serving
function: version
services:
- service: my_datascience_server
image: myorg/mydatascienceapp
pipelines:
- pipeline: my_datascience_pipeline
start_date: 1970-01-01
timeout_minutes: 45
schedule: 0 * 1 * *
metrics:
namespace: DataScience
backends: []
tasks:
- task: train
type: python
description: train model
image: myorg/mydatascienceapp
cmd: python -u training.py train
env:
MOUNT_PATH: /mnt/gettingstartedvol
mounts:
- mount: mymount
volume: gettingstartedvol
path: /mnt/gettingstartedvol
- task: validate
type: python
description: validate model and deploy
image: myorg/mydatascienceapp
cmd: python -u training.py validate
env:
MOUNT_PATH: /mnt/gettingstartedvol
mounts:
- mount: mymount
volume: gettingstartedvol
path: /mnt/gettingstartedvol
================================================
FILE: examples/aws-ml-app-demo/manifests/aws-ml-app-demo.yaml
================================================
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
---
apiVersion: v1
kind: Pod
metadata:
name: aws-ml-app-demo
spec:
volumes:
- name: task-pv-storage
persistentVolumeClaim:
claimName: gettingstartedvol-pvc
containers:
- name: task-pv-container
imagePullPolicy: Never
image: myorg/mydatascienceapp
lifecycle:
postStart:
exec:
command: [/bin/bash, -c, apt update && apt install curl -y]
ports:
- containerPort: 80
name: http-server
volumeMounts:
- mountPath: /mnt/gettingstartedvol
name: task-pv-storage
================================================
FILE: examples/aws-ml-app-demo/model_store.py
================================================
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
import glob
import os
import pickle
import time
MOUNT_PATH = os.environ.get('MOUNT_PATH', '/mnt/gettingstartedvol')
PRODUCTION = 'production'
CANDIDATE = 'candidate'
_ONE_HOUR = 60 * 60
class ModelStore:
def __init__(self, env):
self.env = env
self._latest_model = None
self._latest_version = None
self._last_check = time.time()
def load_latest_model(self, force=False):
if not self._latest_model or time.time() - self._last_check > _ONE_HOUR or force:
self._latest_model, self._latest_version = self._download_latest_model()
return self._latest_model, self._latest_version
def save_model(self, model, version):
key = 'model.p'
path = f'{MOUNT_PATH}/{self.env}/{version}'
os.makedirs(f'{path}', exist_ok=True)
pickle.dump(model, open(f'{path}/{key}', "wb"))
def _download_latest_model(self):
objects = glob.glob(f'{MOUNT_PATH}/{self.env}/**/*')
models = list(reversed(sorted(obj for obj in objects if obj.endswith('.p'))))
latest_key = models[0]
version = latest_key.rsplit('/')[-2]
print(f'Loading model version {version}')
return pickle.load(open(latest_key, 'rb')), version
================================================
FILE: examples/aws-ml-app-demo/requirements.txt
================================================
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
scikit-learn==0.23.2
apache-liminal==0.0.2
Werkzeug==1.0.1
itsdangerous==1.1.0
MarkupSafe==1.1.1
Flask
================================================
FILE: examples/aws-ml-app-demo/serving.py
================================================
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
import json
import model_store
from model_store import ModelStore
_MODEL_STORE = ModelStore(model_store.PRODUCTION)
_PETAL_WIDTH = 'petal_width'
def predict(input_json):
try:
input_dict = json.loads(input_json)
model, version = _MODEL_STORE.load_latest_model()
result = str(model.predict_proba([[float(input_dict[_PETAL_WIDTH])]])[0][1])
return json.dumps({"result": result, "version": version})
except IndexError:
return 'Failure: the model is not ready yet'
except Exception as e:
print(e)
return 'Failure'
def healthcheck(self):
return 'Server is up!'
def version(self):
try:
model, version = _MODEL_STORE.load_latest_model()
print(f'version={version}')
return version
except Exception as e:
return e
================================================
FILE: examples/aws-ml-app-demo/training.py
================================================
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
import sys
import time
import model_store
import numpy as np
from model_store import ModelStore
from sklearn import datasets
from sklearn.linear_model import LogisticRegression
_CANDIDATE_MODEL_STORE = ModelStore(model_store.CANDIDATE)
_PRODUCTION_MODEL_STORE = ModelStore(model_store.PRODUCTION)
def train_model():
iris = datasets.load_iris()
X = iris["data"][:, 3:] # petal width
y = (iris["target"] == 2).astype(np.int)
model = LogisticRegression()
model.fit(X, y)
version = round(time.time())
print(f'Saving model with version {version} to candidate model store.')
_CANDIDATE_MODEL_STORE.save_model(model, version)
def validate_model():
model, version = _CANDIDATE_MODEL_STORE.load_latest_model()
print(f'Validating model with version {version} to candidate model store.')
if not isinstance(model.predict([[1]]), np.ndarray):
raise ValueError('Invalid model')
print(f'Deploying model with version {version} to production model store.')
_PRODUCTION_MODEL_STORE.save_model(model, version)
if __name__ == '__main__':
cmd = sys.argv[1]
if cmd == 'train':
train_model()
elif cmd == 'validate':
validate_model()
else:
raise ValueError(f"Unknown command {cmd}")
================================================
FILE: examples/extensibility/__init__.py
================================================
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
================================================
FILE: examples/extensibility/executors/__init__.py
================================================
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
================================================
FILE: examples/extensibility/executors/custom_executor.py
================================================
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
from plugins.tasks import custom_task
from liminal.runners.airflow.model import executor
class CustomExecutor(executor.Executor):
supported_task_types = [custom_task.CustomTask]
def _apply_executor_task_to_dag(self, **kwargs):
print("Hello from custom executor")
return kwargs['task'].custom_task_logic()
================================================
FILE: examples/extensibility/images/__init__.py
================================================
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.
gitextract_29sbg3vw/ ├── .asf.yaml ├── .bumpversion.cfg ├── .github/ │ ├── PULL_REQUEST_TEMPLATE.md │ └── workflows/ │ ├── pre_commits.yml │ ├── unittests.yml │ └── versioning.yml ├── .gitignore ├── .pre-commit-config.yaml ├── CONTRIBUTING.md ├── DISCLAIMER-WIP ├── LICENSE ├── MANIFEST.in ├── NOTICE ├── README.md ├── RETIRED.txt ├── dev/ │ ├── LICENSE.txt │ ├── RELEASE.md │ └── sign.sh ├── docs/ │ ├── Makefile │ ├── README.md │ ├── architecture.md │ ├── conf.py │ ├── getting-started/ │ │ ├── hello_world.md │ │ ├── iris_classification.md │ │ └── spark_app_demo.md │ ├── index.rst │ ├── liminal/ │ │ ├── README.md │ │ ├── advanced.liminal.yml.md │ │ ├── executors/ │ │ │ ├── README.md │ │ │ ├── emr.md │ │ │ ├── index.rst │ │ │ └── kubernetes.md │ │ ├── extensibility.md │ │ ├── images/ │ │ │ ├── README.md │ │ │ ├── index.rst │ │ │ ├── python.md │ │ │ └── python_server.md │ │ ├── index.rst │ │ ├── kubernetes/ │ │ │ └── secret_util.md │ │ ├── liminal.yml.md │ │ ├── metrics_backends/ │ │ │ ├── README.md │ │ │ ├── aws_cloudwatch.md │ │ │ └── index.rst │ │ ├── monitoring.md │ │ ├── pipelines.md │ │ ├── services.md │ │ └── tasks/ │ │ ├── README.md │ │ ├── create_cloudformation_stack.md │ │ ├── delete_cloudformation_stack.md │ │ ├── index.rst │ │ ├── python.md │ │ └── spark.md │ ├── make.bat │ └── source/ │ ├── How_to_install_liminal_in_airflow_on_kubernetes.md │ └── liminal_aws_deploy.sh ├── examples/ │ ├── aws-ml-app-demo/ │ │ ├── __init__.py │ │ ├── liminal.yml │ │ ├── manifests/ │ │ │ └── aws-ml-app-demo.yaml │ │ ├── model_store.py │ │ ├── requirements.txt │ │ ├── serving.py │ │ └── training.py │ ├── extensibility/ │ │ ├── __init__.py │ │ ├── executors/ │ │ │ ├── __init__.py │ │ │ └── custom_executor.py │ │ ├── images/ │ │ │ ├── __init__.py │ │ │ ├── custom_image.py │ │ │ └── custom_image_builder/ │ │ │ ├── Dockerfile │ │ │ └── __init__.py │ │ ├── liminal.yml │ │ └── tasks/ │ │ ├── __init__.py │ │ └── custom_task.py │ ├── liminal-getting-started/ │ │ ├── __init__.py │ │ ├── hello_world.json │ │ ├── helloworld/ │ │ │ ├── __init__.py │ │ │ └── hello_world.py │ │ ├── liminal.yml │ │ ├── myserver/ │ │ │ ├── __init__.py │ │ │ └── my_server.py │ │ ├── pip.conf │ │ └── requirements.txt │ ├── sagemaker_example/ │ │ ├── README.md │ │ ├── archetype/ │ │ │ └── liminal.yaml │ │ ├── data_preparation/ │ │ │ ├── __init__.py │ │ │ ├── data_preparation.py │ │ │ └── data_uploader.py │ │ ├── data_train/ │ │ │ ├── __init__.py │ │ │ └── train.py │ │ ├── inference.py │ │ ├── liminal.yaml │ │ ├── liminal_sm.py │ │ ├── requirements.txt │ │ └── sm_ops.py │ └── spark-app-demo/ │ └── k8s/ │ ├── __init__.py │ ├── archetype/ │ │ └── liminal.yml │ ├── data/ │ │ └── iris.csv │ ├── data_cleanup.py │ ├── liminal.yml │ ├── manifests/ │ │ └── spark-app-demo.yaml │ ├── model_store.py │ ├── requirements.txt │ ├── serving.py │ └── training.py ├── install.sh ├── liminal/ │ ├── __init__.py │ ├── build/ │ │ ├── __init__.py │ │ ├── image/ │ │ │ ├── __init__.py │ │ │ ├── python/ │ │ │ │ ├── Dockerfile │ │ │ │ ├── __init__.py │ │ │ │ └── python.py │ │ │ ├── python_server/ │ │ │ │ ├── Dockerfile │ │ │ │ ├── __init__.py │ │ │ │ ├── liminal_python_server.py │ │ │ │ ├── python_server.py │ │ │ │ └── python_server_requirements.txt │ │ │ └── spark/ │ │ │ ├── Dockerfile │ │ │ ├── __init__.py │ │ │ └── spark.py │ │ ├── image_builder.py │ │ ├── liminal_apps_builder.py │ │ └── python.py │ ├── core/ │ │ ├── __init__.py │ │ ├── config/ │ │ │ ├── __init__.py │ │ │ ├── config.py │ │ │ └── defaults/ │ │ │ ├── __init__.py │ │ │ ├── base/ │ │ │ │ ├── __init__.py │ │ │ │ └── liminal.yml │ │ │ └── default_configs.py │ │ ├── environment.py │ │ └── util/ │ │ ├── __init__.py │ │ ├── class_util.py │ │ ├── dict_util.py │ │ ├── env_util.py │ │ ├── extensible.py │ │ └── files_util.py │ ├── docker/ │ │ └── __init__.py │ ├── kubernetes/ │ │ ├── __init__.py │ │ ├── secret_util.py │ │ └── volume_util.py │ ├── logging/ │ │ ├── __init__.py │ │ └── logging_setup.py │ ├── monitoring/ │ │ └── __init__.py │ ├── runners/ │ │ ├── __init__.py │ │ └── airflow/ │ │ ├── __init__.py │ │ ├── config/ │ │ │ ├── __init__.py │ │ │ └── standalone_variable_backend.py │ │ ├── dag/ │ │ │ ├── __init__.py │ │ │ ├── liminal_dags.py │ │ │ └── liminal_register_dags.py │ │ ├── executors/ │ │ │ ├── __init__.py │ │ │ ├── airflow.py │ │ │ ├── emr.py │ │ │ └── kubernetes.py │ │ ├── model/ │ │ │ ├── __init__.py │ │ │ ├── executor.py │ │ │ └── task.py │ │ ├── operators/ │ │ │ ├── __init__.py │ │ │ ├── cloudformation.py │ │ │ ├── job_status_operator.py │ │ │ └── operator_with_variable_resolving.py │ │ └── tasks/ │ │ ├── __init__.py │ │ ├── airflow.py │ │ ├── containerable.py │ │ ├── create_cloudformation_stack.py │ │ ├── delete_cloudformation_stack.py │ │ ├── hadoop.py │ │ ├── job_end.py │ │ ├── job_start.py │ │ ├── python.py │ │ ├── spark.py │ │ └── sql.py │ ├── settings.py │ └── sql/ │ └── __init__.py ├── liminal-arch.md ├── pyproject.toml ├── requirements.txt ├── run_tests.sh ├── scripts/ │ ├── Dockerfile-airflow │ ├── __init__.py │ ├── docker-compose.yml │ ├── liminal │ ├── requirements-airflow.txt │ └── webserver_config.py ├── setup.py ├── tests/ │ ├── __init__.py │ ├── liminal/ │ │ ├── __init__.py │ │ ├── core/ │ │ │ ├── __init__.py │ │ │ ├── config/ │ │ │ │ ├── __init__.py │ │ │ │ ├── defaults/ │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── test_apply_variables_substitution.py │ │ │ │ │ ├── test_defaults_pipeline_config.py │ │ │ │ │ ├── test_defaults_service_config.py │ │ │ │ │ └── test_defaults_tasks_config.py │ │ │ │ └── test_config.py │ │ │ └── util/ │ │ │ ├── __init__.py │ │ │ └── test_dict_utils.py │ │ └── kubernetes/ │ │ ├── __init__.py │ │ └── test_volume_util.py │ ├── runners/ │ │ ├── __init__.py │ │ ├── airflow/ │ │ │ ├── __init__.py │ │ │ ├── build/ │ │ │ │ ├── __init__.py │ │ │ │ ├── http/ │ │ │ │ │ ├── __init__.py │ │ │ │ │ └── python/ │ │ │ │ │ ├── __init__.py │ │ │ │ │ └── test_python_server_image_builder.py │ │ │ │ ├── python/ │ │ │ │ │ ├── __init__.py │ │ │ │ │ └── test_python_image_builder.py │ │ │ │ ├── spark/ │ │ │ │ │ ├── __init__.py │ │ │ │ │ └── test_spark_image_builder.py │ │ │ │ └── test_liminal_apps_builder.py │ │ │ ├── compiler/ │ │ │ │ └── __init__.py │ │ │ ├── dag/ │ │ │ │ ├── __init__.py │ │ │ │ └── test_liminal_dags.py │ │ │ ├── executors/ │ │ │ │ ├── __init__.py │ │ │ │ ├── test_airflow_executor.py │ │ │ │ └── test_emr.py │ │ │ ├── liminal/ │ │ │ │ ├── __init__.py │ │ │ │ ├── liminal.yml │ │ │ │ ├── myserver/ │ │ │ │ │ ├── __init__.py │ │ │ │ │ └── my_server.py │ │ │ │ ├── pip.conf │ │ │ │ ├── requirements.txt │ │ │ │ ├── write_inputs/ │ │ │ │ │ ├── __init__.py │ │ │ │ │ └── write_inputs.py │ │ │ │ └── write_outputs/ │ │ │ │ ├── __init__.py │ │ │ │ └── write_outputs.py │ │ │ ├── operators/ │ │ │ │ ├── __init__.py │ │ │ │ └── test_operator_with_variable_resolving.py │ │ │ └── tasks/ │ │ │ ├── __init__.py │ │ │ ├── test_create_cloudformation_stack.py │ │ │ ├── test_delete_cloudformation_stack.py │ │ │ ├── test_job_end.py │ │ │ ├── test_job_start.py │ │ │ ├── test_python.py │ │ │ └── test_spark_task.py │ │ └── apps/ │ │ ├── __init__.py │ │ ├── test/ │ │ │ └── liminal.yml │ │ ├── test_app/ │ │ │ ├── __init__.py │ │ │ ├── defaults/ │ │ │ │ ├── __init__.py │ │ │ │ └── liminal.yml │ │ │ ├── extra/ │ │ │ │ ├── __init__.py │ │ │ │ └── liminal.yml │ │ │ ├── helloworld/ │ │ │ │ ├── __init__.py │ │ │ │ └── hellp_world.py │ │ │ ├── liminal.yml │ │ │ └── my_server/ │ │ │ ├── __init__.py │ │ │ └── my_server.py │ │ └── test_spark_app/ │ │ ├── __init__.py │ │ ├── liminal.yml │ │ └── wordcount/ │ │ ├── __init__.py │ │ ├── requirements.txt │ │ ├── wordcount.py │ │ └── words.txt │ ├── test_licenses.py │ └── util/ │ ├── __init__.py │ ├── dag_test_utils.py │ ├── test_class_utils.py │ └── test_pkg_1/ │ ├── __init__.py │ ├── test_clazz_base.py │ └── test_pkg_1_1/ │ ├── __init__.py │ ├── test_clazz_child_1.py │ ├── test_clazz_child_2.py │ ├── test_pkg_1_1_1/ │ │ ├── __init__.py │ │ └── test_clazz_leaf_1.py │ └── test_pkg_1_1_2/ │ ├── __init__.py │ └── test_clazz_leaf_2.py └── yamllint-config.yml
SYMBOL INDEX (464 symbols across 90 files)
FILE: docs/conf.py
function on_missing_reference (line 191) | def on_missing_reference(app, env, node, contnode):
function setup (line 198) | def setup(app):
FILE: examples/aws-ml-app-demo/model_store.py
class ModelStore (line 31) | class ModelStore:
method __init__ (line 32) | def __init__(self, env):
method load_latest_model (line 38) | def load_latest_model(self, force=False):
method save_model (line 44) | def save_model(self, model, version):
method _download_latest_model (line 51) | def _download_latest_model(self):
FILE: examples/aws-ml-app-demo/serving.py
function predict (line 28) | def predict(input_json):
function healthcheck (line 43) | def healthcheck(self):
function version (line 47) | def version(self):
FILE: examples/aws-ml-app-demo/training.py
function train_model (line 32) | def train_model():
function validate_model (line 47) | def validate_model():
FILE: examples/extensibility/executors/custom_executor.py
class CustomExecutor (line 23) | class CustomExecutor(executor.Executor):
method _apply_executor_task_to_dag (line 26) | def _apply_executor_task_to_dag(self, **kwargs):
FILE: examples/extensibility/images/custom_image.py
class CustomImageBuilder (line 23) | class CustomImageBuilder(ImageBuilder):
method __init__ (line 24) | def __init__(self, config, base_path, relative_source_path, tag):
method _dockerfile_path (line 28) | def _dockerfile_path():
FILE: examples/extensibility/tasks/custom_task.py
class CustomTask (line 23) | class CustomTask(task.Task):
method custom_task_logic (line 24) | def custom_task_logic(self):
FILE: examples/liminal-getting-started/myserver/my_server.py
function myendpoint1func (line 20) | def myendpoint1func():
FILE: examples/sagemaker_example/data_preparation/data_preparation.py
function transform (line 36) | def transform(data):
function extract (line 49) | def extract(input_uri):
function load (line 53) | def load(train, test, output_uri_base, data_uploader):
function data_pipeline (line 61) | def data_pipeline(input_uri, output_uri_base, data_uploader):
FILE: examples/sagemaker_example/data_preparation/data_uploader.py
function get_uploader (line 28) | def get_uploader(output_uri_base):
class DataUploader (line 37) | class DataUploader(ABC):
method __init__ (line 38) | def __init__(self):
method upload (line 42) | def upload(self, local_path, output_uri_base):
class LocalDataUploader (line 46) | class LocalDataUploader(DataUploader):
method upload (line 49) | def upload(self, local_path, output_uri_base):
class S3DataUploader (line 56) | class S3DataUploader(DataUploader):
method __init__ (line 59) | def __init__(self):
method upload (line 62) | def upload(self, local_path, output_uri_base):
class SagemakerDataUploader (line 67) | class SagemakerDataUploader(DataUploader):
method __init__ (line 73) | def __init__(self):
method upload (line 79) | def upload(self, local_path, output_uri_base):
FILE: examples/sagemaker_example/data_train/train.py
function train (line 45) | def train(train_df, test_df, n_jobs, model_dir):
FILE: examples/sagemaker_example/inference.py
function model_fn (line 35) | def model_fn(model_dir):
function input_fn (line 40) | def input_fn(input_data, content_type):
function predict_fn (line 52) | def predict_fn(input_data, model):
function df_to_inference_input (line 57) | def df_to_inference_input():
FILE: examples/sagemaker_example/liminal_sm.py
function read_message (line 31) | def read_message(name):
function forward_message (line 37) | def forward_message(name, lines):
FILE: examples/sagemaker_example/sm_ops.py
function get_role (line 42) | def get_role():
function sm_train (line 50) | def sm_train(train_path, test_path, base_job_name="Liminal-sm-training-j...
function sm_deploy (line 72) | def sm_deploy(artifact, model_name="Liminal-sm-demo-model", instance_typ...
function sm_validate (line 87) | def sm_validate(predictor, test_path):
function sm_data_prep (line 101) | def sm_data_prep(input_uri, output_uri_base):
function sm_main (line 109) | def sm_main(args):
function create_sm_args_parser (line 125) | def create_sm_args_parser():
FILE: examples/spark-app-demo/k8s/data_cleanup.py
function transform (line 28) | def transform(data):
function extract (line 44) | def extract(spark, input_uri):
function load (line 48) | def load(data, output_uri):
function data_pipeline (line 52) | def data_pipeline(input_uri, output_uri):
FILE: examples/spark-app-demo/k8s/model_store.py
class ModelStore (line 31) | class ModelStore:
method __init__ (line 32) | def __init__(self, env):
method load_latest_model (line 38) | def load_latest_model(self, force=False):
method save_model (line 44) | def save_model(self, model, version):
method _download_latest_model (line 51) | def _download_latest_model(self):
FILE: examples/spark-app-demo/k8s/serving.py
function predict (line 28) | def predict(input_json):
function healthcheck (line 43) | def healthcheck(self):
function version (line 47) | def version(self):
FILE: examples/spark-app-demo/k8s/training.py
function load_iris_from_csv_file (line 39) | def load_iris_from_csv_file(f):
function get_dataset (line 47) | def get_dataset(d):
function load_and_split (line 57) | def load_and_split(input_uri):
function train_model (line 66) | def train_model(input_uri):
function validate_model (line 83) | def validate_model(input_uri):
FILE: liminal/build/image/python/python.py
class PythonImageBuilder (line 24) | class PythonImageBuilder(BasePythonImageBuilder):
method __init__ (line 25) | def __init__(self, config, base_path, relative_source_path, tag):
method _dockerfile_path (line 29) | def _dockerfile_path():
FILE: liminal/build/image/python_server/liminal_python_server.py
function __get_module (line 23) | def __get_module(kls):
function __get_endpoint_function (line 32) | def __get_endpoint_function(endpoint_config):
function no_content (line 49) | def no_content():
function show (line 54) | def show(endpoint):
FILE: liminal/build/image/python_server/python_server.py
class PythonServerImageBuilder (line 26) | class PythonServerImageBuilder(BasePythonImageBuilder):
method __init__ (line 27) | def __init__(self, config, base_path, relative_source_path, tag):
method _dockerfile_path (line 31) | def _dockerfile_path():
method _additional_files_from_paths (line 35) | def _additional_files_from_paths():
method _additional_files_from_filename_content_pairs (line 41) | def _additional_files_from_filename_content_pairs(self):
FILE: liminal/build/image/spark/spark.py
class SparkImageBuilder (line 23) | class SparkImageBuilder(BasePythonImageBuilder):
method __init__ (line 24) | def __init__(self, config, base_path, relative_source_path, tag):
method _dockerfile_path (line 28) | def _dockerfile_path():
FILE: liminal/build/image_builder.py
class ImageBuilder (line 27) | class ImageBuilder:
method __init__ (line 34) | def __init__(self, config, base_path, relative_source_path, tag):
method build (line 46) | def build(self):
method __copy_source_code (line 103) | def __copy_source_code(self, temp_dir):
method _write_additional_files (line 106) | def _write_additional_files(self, temp_dir):
method __temp_dir (line 114) | def __temp_dir(self):
method __remove_dir (line 121) | def __remove_dir(temp_dir):
method __copy_dir (line 125) | def __copy_dir(source_path, destination_path):
method __copy_file (line 129) | def __copy_file(source_file_path, destination_file_path):
method _dockerfile_path (line 133) | def _dockerfile_path():
method _additional_files_from_paths (line 140) | def _additional_files_from_paths():
method _additional_files_from_filename_content_pairs (line 146) | def _additional_files_from_filename_content_pairs(self):
method _build_flags (line 152) | def _build_flags(self):
method _use_buildkit (line 158) | def _use_buildkit(self):
FILE: liminal/build/liminal_apps_builder.py
function build_liminal_apps (line 26) | def build_liminal_apps(path):
function __build_image (line 50) | def __build_image(base_path, builder_config, builder):
function __get_image_builder_class (line 60) | def __get_image_builder_class(task_type):
FILE: liminal/build/python.py
class PythonImageVersions (line 24) | class PythonImageVersions:
method default_version (line 30) | def default_version(self):
method supported_versions (line 34) | def supported_versions(self):
method get_image_name (line 37) | def get_image_name(self, python_version):
class BasePythonImageBuilder (line 58) | class BasePythonImageBuilder(ImageBuilder):
method __init__ (line 66) | def __init__(self, config, base_path, relative_source_path, tag, base_...
method _dockerfile_path (line 71) | def _dockerfile_path():
method _write_additional_files (line 74) | def _write_additional_files(self, temp_dir):
method _additional_files_from_filename_content_pairs (line 82) | def _additional_files_from_filename_content_pairs(self):
method __mount_pip_conf (line 91) | def __mount_pip_conf(self, data):
method __add_python_base_version (line 102) | def __add_python_base_version(self, data):
method _build_flags (line 107) | def _build_flags(self):
method _use_buildkit (line 113) | def _use_buildkit(self):
FILE: liminal/core/config/config.py
class ConfigUtil (line 28) | class ConfigUtil:
method __init__ (line 59) | def __init__(self, configs_path):
method safe_load (line 66) | def safe_load(self, is_render_variables, soft_merge=False):
method __merge_configs (line 92) | def __merge_configs(self, subliminal, superliminal, is_render_variable...
method __get_superliminal (line 111) | def __get_superliminal(self, liminal, soft_merge):
method __get_base_config (line 130) | def __get_base_config(self):
method __is_base_config (line 133) | def __is_base_config(self, config):
method __is_subliminal (line 136) | def __is_subliminal(self, config):
method __get_config (line 142) | def __get_config(self, config_name):
method __merge_sub_and_super (line 145) | def __merge_sub_and_super(self, sub, supr, is_render_variables):
method __merge_superliminals (line 159) | def __merge_superliminals(self, super1, super2):
method snapshot_final_liminal_configs (line 176) | def snapshot_final_liminal_configs(self):
method __merge_section (line 179) | def __merge_section(self, subliminal, superliminal, section):
method __apply_pipeline_defaults (line 183) | def __apply_pipeline_defaults(subliminal, superliminal, pipeline):
method __deep_list_keyword_merge (line 187) | def __deep_list_keyword_merge(unique_key_name, subliminal_list_conf, s...
FILE: liminal/core/config/defaults/default_configs.py
function apply_variable_substitution (line 28) | def apply_variable_substitution(subliminal, superliminal, is_render_vari...
function apply_service_defaults (line 51) | def apply_service_defaults(subliminal, superliminal):
function apply_pipeline_defaults (line 66) | def apply_pipeline_defaults(subliminal, superliminal, pipeline):
function apply_task_defaults (line 91) | def apply_task_defaults(subliminal, superliminal, pipeline, superliminal...
function __apply_task_defaults (line 109) | def __apply_task_defaults(
function __enrich_tasks (line 120) | def __enrich_tasks(sub_tasks, super_before_tasks, super_after_tasks, tas...
FILE: liminal/core/environment.py
function get_liminal_home (line 31) | def get_liminal_home():
function get_dags_dir (line 41) | def get_dags_dir():
function get_liminal_version (line 47) | def get_liminal_version():
function get_airflow_home_dir (line 67) | def get_airflow_home_dir():
FILE: liminal/core/util/class_util.py
function find_subclasses_in_packages (line 23) | def find_subclasses_in_packages(packages, parent_class):
function import_module (line 48) | def import_module(package, recursive=True):
function __get_class (line 71) | def __get_class(the_module, the_class):
FILE: liminal/core/util/dict_util.py
function merge_dicts (line 27) | def merge_dicts(dict1, dict2, recursive=False):
function __merge_dicts (line 37) | def __merge_dicts(dict1, dict2):
function replace_placeholders (line 58) | def replace_placeholders(dct, variables):
function replace_placholders_in_string (line 65) | def replace_placholders_in_string(string_value, variables, pattern=__PLA...
function __replace_placeholders (line 69) | def __replace_placeholders(dct, variables):
function __replace_placeholder_in_list (line 82) | def __replace_placeholder_in_list(lst, variables):
function __repl (line 94) | def __repl(matched, variables):
function __try_backend_variables (line 100) | def __try_backend_variables(key, default):
FILE: liminal/core/util/env_util.py
function is_running_on_jenkins (line 22) | def is_running_on_jenkins():
FILE: liminal/core/util/extensible.py
function __generate_extra_paths (line 28) | def __generate_extra_paths(plugin_type, extra_paths):
function load_executors (line 32) | def load_executors(extra_paths=None):
function load_tasks (line 40) | def load_tasks(extra_paths=None):
function load_image_builders (line 48) | def load_image_builders(extra_paths=None):
FILE: liminal/core/util/files_util.py
function resolve_pipeline_source_file (line 29) | def resolve_pipeline_source_file(config_name):
function load (line 36) | def load(path):
function find_config_files (line 57) | def find_config_files(path):
function dump_liminal_configs (line 72) | def dump_liminal_configs(liminal_configs, path):
function dump_liminal_config (line 82) | def dump_liminal_config(liminal_config, file_path):
FILE: liminal/kubernetes/secret_util.py
function get_secret_configs (line 41) | def get_secret_configs(liminal_config):
function create_local_secrets (line 52) | def create_local_secrets(liminal_config):
function create_secret (line 60) | def create_secret(conf, namespace='default') -> None:
function _create_secret (line 77) | def _create_secret(namespace, conf, name):
function delete_local_secrets (line 98) | def delete_local_secrets(liminal_config, base_dir):
function delete_local_secret (line 106) | def delete_local_secret(name, namespace='default'):
FILE: liminal/kubernetes/volume_util.py
function get_volume_configs (line 39) | def get_volume_configs(liminal_config, base_dir):
function create_local_volumes (line 53) | def create_local_volumes(liminal_config, base_dir):
function create_local_volume (line 61) | def create_local_volume(conf, namespace='default') -> None:
function delete_local_volumes (line 94) | def delete_local_volumes(liminal_config, base_dir):
function delete_local_volume (line 102) | def delete_local_volume(name, namespace='default'):
function _list_persistent_volume_claims (line 127) | def _list_persistent_volume_claims(name):
function _list_persistent_volumes (line 133) | def _list_persistent_volumes(name):
function _create_persistent_volume_claim (line 137) | def _create_persistent_volume_claim(pvc_name, volume_name, namespace):
function _create_local_volume (line 155) | def _create_local_volume(conf, name):
FILE: liminal/logging/logging_setup.py
function logging_initialization (line 30) | def logging_initialization():
FILE: liminal/runners/airflow/__init__.py
class DummyDagRun (line 25) | class DummyDagRun:
method __init__ (line 26) | def __init__(self) -> None:
method get_task_instances (line 32) | def get_task_instances():
method get_task_instance (line 35) | def get_task_instance(self, _):
class DummyDag (line 44) | class DummyDag:
method __init__ (line 45) | def __init__(self, dag_id, task_id):
method get_dagrun (line 63) | def get_dagrun(self):
method xcom_push (line 67) | def xcom_push(key, value):
FILE: liminal/runners/airflow/config/standalone_variable_backend.py
function get_variable (line 30) | def get_variable(key, default_val):
function liminal_local_mode (line 46) | def liminal_local_mode():
FILE: liminal/runners/airflow/dag/__init__.py
function register_dags (line 26) | def register_dags():
FILE: liminal/runners/airflow/dag/liminal_register_dags.py
function register_dags (line 33) | def register_dags(configs_path):
function __initialize_executors (line 107) | def __initialize_executors(liminal_config):
function __initialize_dag (line 116) | def __initialize_dag(default_args, pipeline, owner):
function __default_args (line 147) | def __default_args(pipeline):
function get_task_class (line 168) | def get_task_class(task_type):
function get_executor_class (line 172) | def get_executor_class(executor_type):
FILE: liminal/runners/airflow/executors/airflow.py
class AirflowExecutor (line 23) | class AirflowExecutor(executor.Executor):
method _apply_executor_task_to_dag (line 30) | def _apply_executor_task_to_dag(self, **kwargs):
FILE: liminal/runners/airflow/executors/emr.py
class EMRExecutor (line 26) | class EMRExecutor(executor.Executor):
method __init__ (line 33) | def __init__(self, executor_id, liminal_config, executor_config):
method _apply_executor_task_to_dag (line 40) | def _apply_executor_task_to_dag(self, **kwargs):
method __generate_emr_step (line 76) | def __generate_emr_step(self, task_id, args):
FILE: liminal/runners/airflow/executors/kubernetes.py
class KubernetesPodExecutor (line 39) | class KubernetesPodExecutor(executor.Executor):
method __init__ (line 46) | def __init__(self, task_id, liminal_config, executor_config):
method _apply_executor_task_to_dag (line 52) | def _apply_executor_task_to_dag(self, **kwargs):
method _volumes (line 67) | def _volumes(self):
method __kubernetes_kwargs (line 86) | def __kubernetes_kwargs(self, task: ContainerTask):
method __jenkins_kubernetes_affinity (line 149) | def __jenkins_kubernetes_affinity():
FILE: liminal/runners/airflow/model/executor.py
function add_variables_to_operator (line 28) | def add_variables_to_operator(operator, task) -> BaseOperator:
class Executor (line 44) | class Executor(ABC):
method __init__ (line 52) | def __init__(self, executor_id, liminal_config, executor_config):
method apply_task_to_dag (line 57) | def apply_task_to_dag(self, **kwargs):
method _apply_executor_task_to_dag (line 65) | def _apply_executor_task_to_dag(self, **kwargs):
method _validate_task_type (line 68) | def _validate_task_type(self, task):
FILE: liminal/runners/airflow/model/task.py
class Task (line 30) | class Task(ABC):
method __init__ (line 35) | def __init__(
method serialize (line 47) | def serialize(self) -> str:
method _add_variables_to_operator (line 64) | def _add_variables_to_operator(self, operator) -> BaseOperator:
FILE: liminal/runners/airflow/operators/cloudformation.py
class CloudFormationHook (line 32) | class CloudFormationHook(AwsBaseHook):
method __init__ (line 37) | def __init__(self, region_name=None, *args, **kwargs):
method get_conn (line 42) | def get_conn(self):
class BaseCloudFormationOperator (line 47) | class BaseCloudFormationOperator(BaseOperator):
method __init__ (line 62) | def __init__(self, params, aws_conn_id='aws_default', *args, **kwargs):
method execute (line 67) | def execute(self, context):
method cloudformation_op (line 72) | def cloudformation_op(self, cloudformation):
class CloudFormationCreateStackOperator (line 79) | class CloudFormationCreateStackOperator(BaseCloudFormationOperator):
method __init__ (line 94) | def __init__(self, params, aws_conn_id='aws_default', *args, **kwargs):
method cloudformation_op (line 97) | def cloudformation_op(self, cloudformation):
class CloudFormationDeleteStackOperator (line 101) | class CloudFormationDeleteStackOperator(BaseCloudFormationOperator):
method __init__ (line 117) | def __init__(self, params, aws_conn_id='aws_default', *args, **kwargs):
method cloudformation_op (line 120) | def cloudformation_op(self, cloudformation):
class BaseCloudFormationSensor (line 124) | class BaseCloudFormationSensor(BaseSensorOperator):
method __init__ (line 137) | def __init__(
method poke (line 154) | def poke(self, context):
method get_hook (line 180) | def get_hook(self):
method allow_non_existing_stack_status (line 189) | def allow_non_existing_stack_status(self):
class CloudFormationCreateStackSensor (line 196) | class CloudFormationCreateStackSensor(BaseCloudFormationSensor):
method __init__ (line 212) | def __init__(self, stack_name, aws_conn_id='aws_default', poke_interva...
class CloudFormationDeleteStackSensor (line 224) | class CloudFormationDeleteStackSensor(BaseCloudFormationSensor):
method __init__ (line 240) | def __init__(self, stack_name, aws_conn_id='aws_default', poke_interva...
method allow_non_existing_stack_status (line 251) | def allow_non_existing_stack_status(self):
FILE: liminal/runners/airflow/operators/job_status_operator.py
class JobStatusOperator (line 32) | class JobStatusOperator(BaseOperator):
method __init__ (line 40) | def __init__(self, backends, *args, **kwargs):
method execute (line 45) | def execute(self, context):
method metrics (line 53) | def metrics(self, context):
method send_metric_to_cloudwatch (line 56) | def send_metric_to_cloudwatch(self, metric):
method get_cloudwatch (line 61) | def get_cloudwatch(self):
class JobStartOperator (line 67) | class JobStartOperator(JobStatusOperator):
method __init__ (line 70) | def __init__(self, namespace, application_name, backends, *args, **kwa...
method metrics (line 75) | def metrics(self, context):
class JobEndOperator (line 79) | class JobEndOperator(JobStatusOperator):
method __init__ (line 82) | def __init__(self, namespace, application_name, backends, *args, **kwa...
method execute (line 88) | def execute(self, context):
method metrics (line 92) | def metrics(self, context):
method __log_and_get_state (line 106) | def __log_and_get_state(self, task_instance):
method __calculate_job_result (line 113) | def __calculate_job_result(self, context):
method post_execute (line 128) | def post_execute(self, context: Any, result: Any = None, session=None):
class CloudWatchHook (line 140) | class CloudWatchHook(AwsHook):
method __init__ (line 145) | def __init__(self, region_name=None, *args, **kwargs):
method get_conn (line 150) | def get_conn(self):
method put_metric_data (line 153) | def put_metric_data(self, metric):
class Metric (line 176) | class Metric:
method __init__ (line 189) | def __init__(self, namespace, name, value, tags):
class Tag (line 196) | class Tag:
method __init__ (line 197) | def __init__(self, name, value):
FILE: liminal/runners/airflow/operators/operator_with_variable_resolving.py
class OperatorWithVariableResolving (line 37) | class OperatorWithVariableResolving(BaseOperator):
method __init__ (line 42) | def __init__(self, dag, task_config: dict, variables: dict = None, lim...
method execute (line 53) | def execute(self, context):
method post_execute (line 66) | def post_execute(self, context, result=None):
method _get_operator_delegate_attributes (line 69) | def _get_operator_delegate_attributes(self):
method pre_execute (line 79) | def pre_execute(self, context: Any):
method on_kill (line 82) | def on_kill(self) -> None:
method render_template_fields (line 85) | def render_template_fields(self, context: Dict, jinja_env: Optional[ji...
method render_template (line 88) | def render_template(
method get_template_env (line 100) | def get_template_env(self) -> jinja2.Environment:
method prepare_template (line 103) | def prepare_template(self) -> None:
method resolve_template_files (line 106) | def resolve_template_files(self) -> None:
method clear (line 109) | def clear(
method run (line 119) | def run(
class LiminalEnvironment (line 130) | class LiminalEnvironment(Environment):
method __init__ (line 131) | def __init__(self, variables, task_config=None):
method from_string (line 145) | def from_string(self, val, **kwargs):
method render (line 149) | def render(self, *_, **kwargs):
method __render (line 156) | def __render(self, val: str, dag_run_conf: dict, unresolved_tags: set):
function add_variables_to_operator (line 177) | def add_variables_to_operator(operator, task) -> BaseOperator:
FILE: liminal/runners/airflow/tasks/airflow.py
class AirflowTask (line 24) | class AirflowTask(task.Task, ABC):
method apply_task_to_dag (line 30) | def apply_task_to_dag(self):
FILE: liminal/runners/airflow/tasks/containerable.py
class ContainerTask (line 35) | class ContainerTask(task.Task, ABC):
method __init__ (line 40) | def __init__(
method _kubernetes_cmds_and_arguments (line 51) | def _kubernetes_cmds_and_arguments(self):
method __get_local_env_params_from_env_file (line 59) | def __get_local_env_params_from_env_file():
method __env_vars (line 75) | def __env_vars(self, env) -> Dict:
FILE: liminal/runners/airflow/tasks/create_cloudformation_stack.py
class CreateCloudFormationStackTask (line 34) | class CreateCloudFormationStackTask(airflow.AirflowTask):
method __init__ (line 39) | def __init__(
method apply_task_to_dag (line 45) | def apply_task_to_dag(self):
method __cloudformation_stack_running_branch (line 82) | def __cloudformation_stack_running_branch(self, stack_name):
method __reformatted_params (line 101) | def __reformatted_params(self, **kwargs):
FILE: liminal/runners/airflow/tasks/delete_cloudformation_stack.py
class DeleteCloudFormationStackTask (line 30) | class DeleteCloudFormationStackTask(airflow.AirflowTask):
method __init__ (line 35) | def __init__(
method apply_task_to_dag (line 41) | def apply_task_to_dag(self):
method __queued_dag_runs_exists (line 77) | def __queued_dag_runs_exists(self, **kwargs):
FILE: liminal/runners/airflow/tasks/hadoop.py
class HadoopTask (line 24) | class HadoopTask(task.Task, ABC):
method get_runnable_command (line 30) | def get_runnable_command(self):
FILE: liminal/runners/airflow/tasks/job_end.py
class JobEndTask (line 23) | class JobEndTask(airflow.AirflowTask):
method __init__ (line 28) | def __init__(
method apply_task_to_dag (line 36) | def apply_task_to_dag(self):
FILE: liminal/runners/airflow/tasks/job_start.py
class JobStartTask (line 23) | class JobStartTask(airflow.AirflowTask):
method __init__ (line 28) | def __init__(
method apply_task_to_dag (line 36) | def apply_task_to_dag(self):
FILE: liminal/runners/airflow/tasks/python.py
class PythonTask (line 22) | class PythonTask(containerable.ContainerTask):
method _kubernetes_cmds_and_arguments (line 27) | def _kubernetes_cmds_and_arguments(self):
method __cmd (line 33) | def __cmd(self):
FILE: liminal/runners/airflow/tasks/spark.py
class SparkTask (line 26) | class SparkTask(hadoop.HadoopTask, containerable.ContainerTask):
method __init__ (line 31) | def __init__(
method get_runnable_command (line 39) | def get_runnable_command(self):
method __generate_spark_submit (line 45) | def __generate_spark_submit(self):
method __spark_args (line 56) | def __spark_args(self):
method __additional_arguments (line 76) | def __additional_arguments(self):
method __parse_spark_arguments (line 82) | def __parse_spark_arguments(self, spark_arguments):
method __interleaving (line 90) | def __interleaving(keys, values):
method _kubernetes_cmds_and_arguments (line 93) | def _kubernetes_cmds_and_arguments(self):
FILE: liminal/runners/airflow/tasks/sql.py
class SqlTask (line 22) | class SqlTask(task.Task):
FILE: liminal/settings.py
function prepare_syspath (line 26) | def prepare_syspath():
function initialize (line 31) | def initialize():
FILE: tests/liminal/core/config/defaults/test_apply_variables_substitution.py
class TestApplyVariablesSubstitution (line 24) | class TestApplyVariablesSubstitution(TestCase):
method test_apply (line 25) | def test_apply(self):
method test_apply_superliminal_is_missing_variables (line 33) | def test_apply_superliminal_is_missing_variables(self):
method test_apply_subliminal_is_missing_variables (line 41) | def test_apply_subliminal_is_missing_variables(self):
FILE: tests/liminal/core/config/defaults/test_defaults_pipeline_config.py
class TestDefaultsPipelineConfig (line 24) | class TestDefaultsPipelineConfig(TestCase):
method test_apply (line 25) | def test_apply(self):
FILE: tests/liminal/core/config/defaults/test_defaults_service_config.py
class TestDefaultsServiceConfig (line 24) | class TestDefaultsServiceConfig(TestCase):
method test_apply (line 25) | def test_apply(self):
FILE: tests/liminal/core/config/defaults/test_defaults_tasks_config.py
class TestDefaultsTaskConfig (line 24) | class TestDefaultsTaskConfig(TestCase):
method test_apply (line 25) | def test_apply(self):
method test_missing_tasks_from_supr (line 62) | def test_missing_tasks_from_supr(self):
method test_missing_tasks_from_sub (line 95) | def test_missing_tasks_from_sub(self):
FILE: tests/liminal/core/config/test_config.py
class TestConfigUtil (line 26) | class TestConfigUtil(TestCase):
method test_safe_load (line 28) | def test_safe_load(self, find_config_files_mock):
method test_get_config (line 190) | def test_get_config(self, find_config_files_mock):
method test_get_superliminal (line 203) | def test_get_superliminal(self, find_config_files_mock):
method test_merge_superliminals (line 241) | def test_merge_superliminals(self, find_config_files_mock):
method test_safe_load_with_variables (line 285) | def test_safe_load_with_variables(self, find_config_files_mock):
method test_liminal_config_snapshot (line 467) | def test_liminal_config_snapshot(self, find_config_files_mock, get_air...
method test_soft_merge_load (line 535) | def test_soft_merge_load(self, find_config_files_mock):
method test_non_soft_merge_load (line 543) | def test_non_soft_merge_load(self):
FILE: tests/liminal/core/util/test_dict_utils.py
class TestDictUtils (line 25) | class TestDictUtils(TestCase):
method setUp (line 26) | def setUp(self) -> None:
method test_merge_dicts (line 31) | def test_merge_dicts(self):
method test_recursive_merge_dicts (line 36) | def test_recursive_merge_dicts(self):
method test_merge_with_empty (line 40) | def test_merge_with_empty(self):
method test_replace_variables_simple_case (line 47) | def test_replace_variables_simple_case(self):
method test_replace_variables_empty_var (line 68) | def test_replace_variables_empty_var(self):
method test_replace_variables_flat_replace (line 80) | def test_replace_variables_flat_replace(self, airflow_variable_mock):
method test_replace_variables_with_nested_list (line 109) | def test_replace_variables_with_nested_list(self, airflow_variable_mock):
method test_replace_variables_from_env (line 140) | def test_replace_variables_from_env(self):
method test_replace_variables_from_variable_and_not_env (line 157) | def test_replace_variables_from_variable_and_not_env(self):
method test_replace_variables_from_airflow_and_not_enc (line 175) | def test_replace_variables_from_airflow_and_not_enc(self, airflow_vari...
FILE: tests/liminal/kubernetes/test_volume_util.py
class TestKubernetesVolume (line 33) | class TestKubernetesVolume(unittest.TestCase):
method setUp (line 34) | def setUp(self) -> None:
method test_volume_config (line 45) | def test_volume_config(self):
method test_create_volume (line 49) | def test_create_volume(self):
method test_delete_volume (line 58) | def test_delete_volume(self):
method _create_volumes (line 64) | def _create_volumes(self):
method _delete_volumes (line 67) | def _delete_volumes(self):
FILE: tests/runners/airflow/build/http/python/test_python_server_image_builder.py
class TestPythonServer (line 36) | class TestPythonServer(TestCase):
method setUp (line 37) | def setUp(self) -> None:
method tearDown (line 43) | def tearDown(self) -> None:
method test_build_python_server (line 47) | def test_build_python_server(self):
method test_build_python_server_with_pip_conf (line 53) | def test_build_python_server_with_pip_conf(self):
method __test_build_python_server (line 61) | def __test_build_python_server(self, use_pip_conf=False, python_versio...
method __remove_containers (line 107) | def __remove_containers(self):
method __get_docker_containers (line 122) | def __get_docker_containers(self):
method __run_container (line 125) | def __run_container(self):
method __create_conf (line 134) | def __create_conf():
FILE: tests/runners/airflow/build/python/test_python_image_builder.py
class TestPythonImageBuilder (line 31) | class TestPythonImageBuilder(TestCase):
method setUp (line 34) | def setUp(self) -> None:
method tearDown (line 40) | def tearDown(self) -> None:
method test_build (line 45) | def test_build(self):
method test_build_with_pip_conf (line 52) | def test_build_with_pip_conf(self):
method test_with_unsupported_python_version (line 62) | def test_with_unsupported_python_version(self):
method __test_build (line 66) | def __test_build(self, use_pip_conf=False, python_version=None):
method __test_image (line 85) | def __test_image(self):
method __create_conf (line 118) | def __create_conf(self, task_id):
method __temp_dir (line 128) | def __temp_dir():
method __remove_dir (line 133) | def __remove_dir(temp_dir):
FILE: tests/runners/airflow/build/spark/test_spark_image_builder.py
class TestSparkImageBuilder (line 30) | class TestSparkImageBuilder(TestCase):
method setUp (line 33) | def setUp(self) -> None:
method tearDown (line 39) | def tearDown(self) -> None:
method test_build (line 44) | def test_build(self):
method __test_build (line 50) | def __test_build(self):
method __test_image (line 63) | def __test_image(self):
method __create_conf (line 88) | def __create_conf(self, task_id):
method __temp_dir (line 98) | def __temp_dir():
method __remove_dir (line 103) | def __remove_dir(temp_dir):
FILE: tests/runners/airflow/build/test_liminal_apps_builder.py
class TestLiminalAppsBuilder (line 28) | class TestLiminalAppsBuilder(TestCase):
method setUp (line 31) | def setUp(self) -> None:
method tearDown (line 35) | def tearDown(self) -> None:
method __remove_images (line 39) | def __remove_images(self):
method test_build_liminal (line 44) | def test_build_liminal(self):
FILE: tests/runners/airflow/dag/test_liminal_dags.py
class Test (line 31) | class Test(TestCase):
method test_register_dags (line 32) | def test_register_dags(self):
method test_default_start_task (line 42) | def test_default_start_task(self):
method test_default_end_task (line 49) | def test_default_end_task(self):
method test_default_args (line 56) | def test_default_args(self):
method get_register_dags (line 67) | def get_register_dags(mock_snapshot_final_liminal_configs):
FILE: tests/runners/airflow/executors/test_airflow_executor.py
class TestAirflowExecutorTask (line 26) | class TestAirflowExecutorTask(TestCase):
method test_apply_task_to_dag (line 31) | def test_apply_task_to_dag(self):
FILE: tests/runners/airflow/executors/test_emr.py
class TestEMRExecutorTask (line 39) | class TestEMRExecutorTask(TestCase):
method setUp (line 44) | def setUp(self) -> None:
method test_apply_task_to_dag (line 87) | def test_apply_task_to_dag(self):
method test_add_step (line 98) | def test_add_step(self, mock_emr_hook_get_conn):
method test_watch_step (line 124) | def test_watch_step(self):
FILE: tests/runners/airflow/liminal/myserver/my_server.py
function myendpoint1func (line 21) | def myendpoint1func(input_json):
FILE: tests/runners/airflow/operators/test_operator_with_variable_resolving.py
class TestKubernetesPodOperatorWithAutoImage (line 30) | class TestKubernetesPodOperatorWithAutoImage(TestCase):
method test_value_from_environment_and_liminal_config (line 32) | def test_value_from_environment_and_liminal_config(self):
method test_value_from_task_variables (line 43) | def test_value_from_task_variables(self):
method test_value_from_inline_task_variables (line 58) | def test_value_from_inline_task_variables(self):
method test_nested_variables (line 73) | def test_nested_variables(self):
method test_dag_run_conf_parameters (line 92) | def test_dag_run_conf_parameters(self):
method test_duplicated_params (line 111) | def test_duplicated_params(self):
method test_non_existing_param (line 147) | def test_non_existing_param(self):
class BaseTestOperator (line 158) | class BaseTestOperator(BaseOperator):
method execute (line 159) | def execute(self, context):
method __init__ (line 162) | def __init__(self, field, expected, *args, **kwargs):
class TestOperator (line 168) | class TestOperator(BaseTestOperator):
method __init__ (line 169) | def __init__(self, *args, **kwargs):
method execute (line 172) | def execute(self, context):
FILE: tests/runners/airflow/tasks/test_create_cloudformation_stack.py
class TestCreateCloudFormationStackTask (line 43) | class TestCreateCloudFormationStackTask(TestCase):
method setUp (line 48) | def setUp(self) -> None:
method test_apply_task_to_dag (line 93) | def test_apply_task_to_dag(self):
method test_cloudformation_does_not_exist (line 104) | def test_cloudformation_does_not_exist(self):
method test_cloudformation_exist_and_running (line 118) | def test_cloudformation_exist_and_running(self):
method test_cloudformation_exists_and_not_running (line 132) | def test_cloudformation_exists_and_not_running(self):
method test_cloudformation_create_stack_operator_task (line 147) | def test_cloudformation_create_stack_operator_task(self):
FILE: tests/runners/airflow/tasks/test_delete_cloudformation_stack.py
class TestDeleteCloudFormationStackTask (line 43) | class TestDeleteCloudFormationStackTask(TestCase):
method setUp (line 48) | def setUp(self) -> None:
method test_apply_task_to_dag (line 71) | def test_apply_task_to_dag(self):
method test_check_dags_queued_task (line 81) | def test_check_dags_queued_task(self):
FILE: tests/runners/airflow/tasks/test_job_end.py
class TestJobEndTask (line 28) | class TestJobEndTask(TestCase):
method test_apply_task_to_dag (line 29) | def test_apply_task_to_dag(self):
method test_apply_task_to_dag_missing_metrics (line 51) | def test_apply_task_to_dag_missing_metrics(self):
method test_apply_task_to_dag_with_partial_configuration (line 74) | def test_apply_task_to_dag_with_partial_configuration(self):
FILE: tests/runners/airflow/tasks/test_job_start.py
class TestJobStartTask (line 28) | class TestJobStartTask(TestCase):
method test_apply_task_to_dag (line 29) | def test_apply_task_to_dag(self):
method test_apply_task_to_dag_missing_metrics (line 51) | def test_apply_task_to_dag_missing_metrics(self):
method test_apply_task_to_dag_with_partial_configuration (line 74) | def test_apply_task_to_dag_with_partial_configuration(self):
FILE: tests/runners/airflow/tasks/test_python.py
class TestPythonTask (line 32) | class TestPythonTask(TestCase):
method setUp (line 36) | def setUp(self) -> None:
method test_apply_task_to_dag (line 56) | def test_apply_task_to_dag(self):
method __create_python_task (line 130) | def __create_python_task(self, dag, task_id, parent, image, cmd, env_v...
FILE: tests/runners/airflow/tasks/test_spark_task.py
class TestSparkTask (line 31) | class TestSparkTask(TestCase):
method test_spark_on_k8s (line 38) | def test_spark_on_k8s(self):
method test_get_runnable_command (line 103) | def test_get_runnable_command(self):
method test_missing_spark_arguments (line 150) | def test_missing_spark_arguments(self):
method test_partially_missing_spark_arguments (line 180) | def test_partially_missing_spark_arguments(self):
FILE: tests/runners/apps/test_app/my_server/my_server.py
function myendpoint1func (line 23) | def myendpoint1func(input_json):
FILE: tests/test_licenses.py
class TestLicenses (line 155) | class TestLicenses(TestCase):
method test_licenses (line 156) | def test_licenses(self):
method check_license (line 186) | def check_license(file):
FILE: tests/util/dag_test_utils.py
function create_dag (line 25) | def create_dag():
FILE: tests/util/test_class_utils.py
class Test (line 29) | class Test(TestCase):
method test_find_full_hierarchy_from_root (line 30) | def test_find_full_hierarchy_from_root(self):
method hierarchy_check (line 39) | def hierarchy_check(self, clazz, expected_dict):
FILE: tests/util/test_pkg_1/test_clazz_base.py
class A (line 20) | class A:
FILE: tests/util/test_pkg_1/test_pkg_1_1/test_clazz_child_1.py
class B (line 23) | class B(A):
class M (line 27) | class M:
FILE: tests/util/test_pkg_1/test_pkg_1_1/test_clazz_child_2.py
class C (line 23) | class C(A):
FILE: tests/util/test_pkg_1/test_pkg_1_1/test_pkg_1_1_1/test_clazz_leaf_1.py
class D (line 23) | class D(B):
FILE: tests/util/test_pkg_1/test_pkg_1_1/test_pkg_1_1_2/test_clazz_leaf_2.py
class E (line 21) | class E(D):
Condensed preview — 266 files, each showing path, character count, and a content snippet. Download the .json file or copy for the full structured content (649K chars).
[
{
"path": ".asf.yaml",
"chars": 1477,
"preview": "---\n#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NO"
},
{
"path": ".bumpversion.cfg",
"chars": 1086,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": ".github/PULL_REQUEST_TEMPLATE.md",
"chars": 728,
"preview": "## Changes Title (replace this with a logical title for your changes)\n\n*Issue #, if available:*\n\n### *Description of cha"
},
{
"path": ".github/workflows/pre_commits.yml",
"chars": 1561,
"preview": "---\nname: PreCommitChecks\n\non:\n push:\n branches:\n - master\n # - '!release*'\n pull_request:\n branches:\n"
},
{
"path": ".github/workflows/unittests.yml",
"chars": 1370,
"preview": "---\nname: Running unittest\n\non:\n # Trigger the workflow on push or pull request, but only for the master branch\n push:"
},
{
"path": ".github/workflows/versioning.yml",
"chars": 805,
"preview": "---\nname: Versioning RC\n\non:\n workflow_run:\n workflows: [Running unittest]\n branches: [master]\n types:\n -"
},
{
"path": ".gitignore",
"chars": 935,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": ".pre-commit-config.yaml",
"chars": 6245,
"preview": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE f"
},
{
"path": "CONTRIBUTING.md",
"chars": 4138,
"preview": "<!--\nLicensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE "
},
{
"path": "DISCLAIMER-WIP",
"chars": 1265,
"preview": "Apache Liminal is an effort undergoing incubation at the Apache Software\nFoundation (ASF), sponsored by the Apache Incub"
},
{
"path": "LICENSE",
"chars": 10850,
"preview": " Apache License\n Version 2.0, January 2004\n http"
},
{
"path": "MANIFEST.in",
"chars": 884,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "NOTICE",
"chars": 207,
"preview": "Apache Liminal\nCopyright 2020-2023 The Apache Software Foundation\n\nThis product includes software developed at\nThe Apach"
},
{
"path": "README.md",
"chars": 7714,
"preview": "<!--\nLicensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE "
},
{
"path": "RETIRED.txt",
"chars": 98,
"preview": "This podling has been retired. Please see http://incubator.apache.org/projects/index.html#liminal\n"
},
{
"path": "dev/LICENSE.txt",
"chars": 755,
"preview": "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\n"
},
{
"path": "dev/RELEASE.md",
"chars": 24627,
"preview": "<!--\nLicensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE "
},
{
"path": "dev/sign.sh",
"chars": 1160,
"preview": "#!/usr/bin/env bash\n#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreeme"
},
{
"path": "docs/Makefile",
"chars": 1366,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "docs/README.md",
"chars": 2061,
"preview": "<!--\nLicensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE "
},
{
"path": "docs/architecture.md",
"chars": 5886,
"preview": "<!--\nLicensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE "
},
{
"path": "docs/conf.py",
"chars": 6376,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "docs/getting-started/hello_world.md",
"chars": 4940,
"preview": "<!--\nLicensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE "
},
{
"path": "docs/getting-started/iris_classification.md",
"chars": 9134,
"preview": "<!--\nLicensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE "
},
{
"path": "docs/getting-started/spark_app_demo.md",
"chars": 9390,
"preview": "<!--\nLicensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE "
},
{
"path": "docs/index.rst",
"chars": 1588,
"preview": ".. Apchae Liminal documentation master file, created by\n sphinx-quickstart on Sun Nov 15 08:45:27 2020.\n You can ada"
},
{
"path": "docs/liminal/README.md",
"chars": 1816,
"preview": "<!--\nLicensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE "
},
{
"path": "docs/liminal/advanced.liminal.yml.md",
"chars": 8672,
"preview": "<!--\nLicensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE "
},
{
"path": "docs/liminal/executors/README.md",
"chars": 1820,
"preview": "<!--\nLicensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE "
},
{
"path": "docs/liminal/executors/emr.md",
"chars": 1572,
"preview": "<!--\nLicensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE "
},
{
"path": "docs/liminal/executors/index.rst",
"chars": 2200,
"preview": ".. Apchae Liminal documentation master file, created by\n sphinx-quickstart on Sun Nov 15 08:45:27 2020.\n You can ada"
},
{
"path": "docs/liminal/executors/kubernetes.md",
"chars": 1330,
"preview": "<!--\nLicensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE "
},
{
"path": "docs/liminal/extensibility.md",
"chars": 1741,
"preview": "<!--\nLicensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE "
},
{
"path": "docs/liminal/images/README.md",
"chars": 2258,
"preview": "<!--\nLicensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE "
},
{
"path": "docs/liminal/images/index.rst",
"chars": 2618,
"preview": ".. Apchae Liminal documentation master file, created by\n sphinx-quickstart on Sun Nov 15 08:45:27 2020.\n You can ada"
},
{
"path": "docs/liminal/images/python.md",
"chars": 1186,
"preview": "<!--\nLicensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE "
},
{
"path": "docs/liminal/images/python_server.md",
"chars": 1889,
"preview": "<!--\nLicensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE "
},
{
"path": "docs/liminal/index.rst",
"chars": 1304,
"preview": ".. Apchae Liminal documentation master file, created by\n sphinx-quickstart on Sun Nov 15 08:45:27 2020.\n You can ada"
},
{
"path": "docs/liminal/kubernetes/secret_util.md",
"chars": 2229,
"preview": "<!--\nLicensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE "
},
{
"path": "docs/liminal/liminal.yml.md",
"chars": 7176,
"preview": "<!--\nLicensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE "
},
{
"path": "docs/liminal/metrics_backends/README.md",
"chars": 1615,
"preview": "<!--\nLicensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE "
},
{
"path": "docs/liminal/metrics_backends/aws_cloudwatch.md",
"chars": 1454,
"preview": "<!--\nLicensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE "
},
{
"path": "docs/liminal/metrics_backends/index.rst",
"chars": 1983,
"preview": ".. Apchae Liminal documentation master file, created by\n sphinx-quickstart on Sun Nov 15 08:45:27 2020.\n You can ada"
},
{
"path": "docs/liminal/monitoring.md",
"chars": 1798,
"preview": "<!--\nLicensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE "
},
{
"path": "docs/liminal/pipelines.md",
"chars": 2555,
"preview": "<!--\nLicensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE "
},
{
"path": "docs/liminal/services.md",
"chars": 1173,
"preview": "<!--\nLicensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE "
},
{
"path": "docs/liminal/tasks/README.md",
"chars": 1705,
"preview": "<!--\nLicensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE "
},
{
"path": "docs/liminal/tasks/create_cloudformation_stack.md",
"chars": 1592,
"preview": "<!--\nLicensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE "
},
{
"path": "docs/liminal/tasks/delete_cloudformation_stack.md",
"chars": 1177,
"preview": "<!--\nLicensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE "
},
{
"path": "docs/liminal/tasks/index.rst",
"chars": 2068,
"preview": ".. Apchae Liminal documentation master file, created by\n sphinx-quickstart on Sun Nov 15 08:45:27 2020.\n You can ada"
},
{
"path": "docs/liminal/tasks/python.md",
"chars": 1662,
"preview": "<!--\nLicensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE "
},
{
"path": "docs/liminal/tasks/spark.md",
"chars": 2022,
"preview": "<!--\nLicensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE "
},
{
"path": "docs/make.bat",
"chars": 1626,
"preview": "REM\r\nREM Licensed to the Apache Software Foundation (ASF) under one\r\nREM or more contributor license agreements. See th"
},
{
"path": "docs/source/How_to_install_liminal_in_airflow_on_kubernetes.md",
"chars": 7914,
"preview": "<!--\nLicensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE "
},
{
"path": "docs/source/liminal_aws_deploy.sh",
"chars": 2478,
"preview": "#! /bin/bash\n#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. S"
},
{
"path": "examples/aws-ml-app-demo/__init__.py",
"chars": 787,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "examples/aws-ml-app-demo/liminal.yml",
"chars": 2238,
"preview": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE f"
},
{
"path": "examples/aws-ml-app-demo/manifests/aws-ml-app-demo.yaml",
"chars": 1363,
"preview": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE f"
},
{
"path": "examples/aws-ml-app-demo/model_store.py",
"chars": 2030,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "examples/aws-ml-app-demo/requirements.txt",
"chars": 891,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "examples/aws-ml-app-demo/serving.py",
"chars": 1614,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "examples/aws-ml-app-demo/training.py",
"chars": 2060,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "examples/extensibility/__init__.py",
"chars": 787,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "examples/extensibility/executors/__init__.py",
"chars": 787,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "examples/extensibility/executors/custom_executor.py",
"chars": 1120,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "examples/extensibility/images/__init__.py",
"chars": 787,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "examples/extensibility/images/custom_image.py",
"chars": 1171,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "examples/extensibility/images/custom_image_builder/Dockerfile",
"chars": 831,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "examples/extensibility/images/custom_image_builder/__init__.py",
"chars": 787,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "examples/extensibility/liminal.yml",
"chars": 1215,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "examples/extensibility/tasks/__init__.py",
"chars": 787,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "examples/extensibility/tasks/custom_task.py",
"chars": 1199,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "examples/liminal-getting-started/__init__.py",
"chars": 787,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "examples/liminal-getting-started/hello_world.json",
"chars": 905,
"preview": "{\n \"\": [\n \"Licensed to the Apache Software Foundation (ASF) under one\",\n \"or more contributor license agreements."
},
{
"path": "examples/liminal-getting-started/helloworld/__init__.py",
"chars": 787,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "examples/liminal-getting-started/helloworld/hello_world.py",
"chars": 1003,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "examples/liminal-getting-started/liminal.yml",
"chars": 2526,
"preview": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE f"
},
{
"path": "examples/liminal-getting-started/myserver/__init__.py",
"chars": 787,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "examples/liminal-getting-started/myserver/my_server.py",
"chars": 827,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "examples/liminal-getting-started/pip.conf",
"chars": 787,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "examples/liminal-getting-started/requirements.txt",
"chars": 787,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "examples/sagemaker_example/README.md",
"chars": 1533,
"preview": "<!--\nLicensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE "
},
{
"path": "examples/sagemaker_example/archetype/liminal.yaml",
"chars": 1597,
"preview": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE f"
},
{
"path": "examples/sagemaker_example/data_preparation/__init__.py",
"chars": 787,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "examples/sagemaker_example/data_preparation/data_preparation.py",
"chars": 2582,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "examples/sagemaker_example/data_preparation/data_uploader.py",
"chars": 2454,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "examples/sagemaker_example/data_train/__init__.py",
"chars": 787,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "examples/sagemaker_example/data_train/train.py",
"chars": 3301,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "examples/sagemaker_example/inference.py",
"chars": 2545,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "examples/sagemaker_example/liminal.yaml",
"chars": 1783,
"preview": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE f"
},
{
"path": "examples/sagemaker_example/liminal_sm.py",
"chars": 3061,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "examples/sagemaker_example/requirements.txt",
"chars": 859,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "examples/sagemaker_example/sm_ops.py",
"chars": 4846,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "examples/spark-app-demo/k8s/__init__.py",
"chars": 787,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "examples/spark-app-demo/k8s/archetype/liminal.yml",
"chars": 1324,
"preview": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE f"
},
{
"path": "examples/spark-app-demo/k8s/data/iris.csv",
"chars": 5474,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "examples/spark-app-demo/k8s/data_cleanup.py",
"chars": 2408,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "examples/spark-app-demo/k8s/liminal.yml",
"chars": 2794,
"preview": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE f"
},
{
"path": "examples/spark-app-demo/k8s/manifests/spark-app-demo.yaml",
"chars": 1362,
"preview": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE f"
},
{
"path": "examples/spark-app-demo/k8s/model_store.py",
"chars": 2030,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "examples/spark-app-demo/k8s/requirements.txt",
"chars": 846,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "examples/spark-app-demo/k8s/serving.py",
"chars": 1611,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "examples/spark-app-demo/k8s/training.py",
"chars": 3538,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "install.sh",
"chars": 1178,
"preview": "#!/bin/sh\n#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See "
},
{
"path": "liminal/__init__.py",
"chars": 839,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "liminal/build/__init__.py",
"chars": 787,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "liminal/build/image/__init__.py",
"chars": 787,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "liminal/build/image/python/Dockerfile",
"chars": 1679,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "liminal/build/image/python/__init__.py",
"chars": 787,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "liminal/build/image/python/python.py",
"chars": 1164,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "liminal/build/image/python_server/Dockerfile",
"chars": 1642,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "liminal/build/image/python_server/__init__.py",
"chars": 787,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "liminal/build/image/python_server/liminal_python_server.py",
"chars": 2065,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "liminal/build/image/python_server/python_server.py",
"chars": 1619,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "liminal/build/image/python_server/python_server_requirements.txt",
"chars": 801,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "liminal/build/image/spark/Dockerfile",
"chars": 912,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "liminal/build/image/spark/__init__.py",
"chars": 787,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "liminal/build/image/spark/spark.py",
"chars": 1162,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "liminal/build/image_builder.py",
"chars": 5233,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "liminal/build/liminal_apps_builder.py",
"chars": 2525,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "liminal/build/python.py",
"chars": 3872,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "liminal/core/__init__.py",
"chars": 787,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "liminal/core/config/__init__.py",
"chars": 787,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "liminal/core/config/config.py",
"chars": 7498,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "liminal/core/config/defaults/__init__.py",
"chars": 787,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "liminal/core/config/defaults/base/__init__.py",
"chars": 905,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "liminal/core/config/defaults/base/liminal.yml",
"chars": 1424,
"preview": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE f"
},
{
"path": "liminal/core/config/defaults/default_configs.py",
"chars": 5341,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "liminal/core/environment.py",
"chars": 2978,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "liminal/core/util/__init__.py",
"chars": 787,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "liminal/core/util/class_util.py",
"chars": 2554,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "liminal/core/util/dict_util.py",
"chars": 3296,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "liminal/core/util/env_util.py",
"chars": 923,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "liminal/core/util/extensible.py",
"chars": 1997,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "liminal/core/util/files_util.py",
"chars": 2647,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "liminal/docker/__init__.py",
"chars": 787,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "liminal/kubernetes/__init__.py",
"chars": 787,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "liminal/kubernetes/secret_util.py",
"chars": 3681,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "liminal/kubernetes/volume_util.py",
"chars": 5978,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "liminal/logging/__init__.py",
"chars": 787,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "liminal/logging/logging_setup.py",
"chars": 1625,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "liminal/monitoring/__init__.py",
"chars": 787,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "liminal/runners/__init__.py",
"chars": 787,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "liminal/runners/airflow/__init__.py",
"chars": 1988,
"preview": "##\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTIC"
},
{
"path": "liminal/runners/airflow/config/__init__.py",
"chars": 787,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "liminal/runners/airflow/config/standalone_variable_backend.py",
"chars": 1743,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "liminal/runners/airflow/dag/__init__.py",
"chars": 1069,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "liminal/runners/airflow/dag/liminal_dags.py",
"chars": 1041,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "liminal/runners/airflow/dag/liminal_register_dags.py",
"chars": 6026,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "liminal/runners/airflow/executors/__init__.py",
"chars": 787,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "liminal/runners/airflow/executors/airflow.py",
"chars": 1130,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "liminal/runners/airflow/executors/emr.py",
"chars": 3200,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "liminal/runners/airflow/executors/kubernetes.py",
"chars": 6080,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "liminal/runners/airflow/model/__init__.py",
"chars": 787,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "liminal/runners/airflow/model/executor.py",
"chars": 2242,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "liminal/runners/airflow/model/task.py",
"chars": 2236,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "liminal/runners/airflow/operators/__init__.py",
"chars": 787,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "liminal/runners/airflow/operators/cloudformation.py",
"chars": 8458,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "liminal/runners/airflow/operators/job_status_operator.py",
"chars": 6283,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "liminal/runners/airflow/operators/operator_with_variable_resolving.py",
"chars": 7532,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "liminal/runners/airflow/tasks/__init__.py",
"chars": 787,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "liminal/runners/airflow/tasks/airflow.py",
"chars": 1009,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "liminal/runners/airflow/tasks/containerable.py",
"chars": 3500,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "liminal/runners/airflow/tasks/create_cloudformation_stack.py",
"chars": 4426,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "liminal/runners/airflow/tasks/delete_cloudformation_stack.py",
"chars": 3139,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "liminal/runners/airflow/tasks/hadoop.py",
"chars": 1010,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "liminal/runners/airflow/tasks/job_end.py",
"chars": 1893,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "liminal/runners/airflow/tasks/job_start.py",
"chars": 1911,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "liminal/runners/airflow/tasks/python.py",
"chars": 1136,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "liminal/runners/airflow/tasks/spark.py",
"chars": 3448,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "liminal/runners/airflow/tasks/sql.py",
"chars": 912,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "liminal/settings.py",
"chars": 1104,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "liminal/sql/__init__.py",
"chars": 787,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "liminal-arch.md",
"chars": 6012,
"preview": "<!--\nLicensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE "
},
{
"path": "pyproject.toml",
"chars": 1105,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "requirements.txt",
"chars": 1470,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "run_tests.sh",
"chars": 817,
"preview": "#!/bin/sh\n#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See "
},
{
"path": "scripts/Dockerfile-airflow",
"chars": 1266,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "scripts/__init__.py",
"chars": 787,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "scripts/docker-compose.yml",
"chars": 2963,
"preview": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE f"
},
{
"path": "scripts/liminal",
"chars": 10299,
"preview": "#!/usr/bin/env python3\n#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agre"
},
{
"path": "scripts/requirements-airflow.txt",
"chars": 921,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "scripts/webserver_config.py",
"chars": 4697,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "setup.py",
"chars": 2101,
"preview": "#!/usr/bin/env python3\n#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agre"
},
{
"path": "tests/__init__.py",
"chars": 787,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "tests/liminal/__init__.py",
"chars": 787,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "tests/liminal/core/__init__.py",
"chars": 787,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "tests/liminal/core/config/__init__.py",
"chars": 787,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "tests/liminal/core/config/defaults/__init__.py",
"chars": 787,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "tests/liminal/core/config/defaults/test_apply_variables_substitution.py",
"chars": 2017,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "tests/liminal/core/config/defaults/test_defaults_pipeline_config.py",
"chars": 3182,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "tests/liminal/core/config/defaults/test_defaults_service_config.py",
"chars": 1898,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "tests/liminal/core/config/defaults/test_defaults_tasks_config.py",
"chars": 4623,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "tests/liminal/core/config/test_config.py",
"chars": 23459,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "tests/liminal/core/util/__init__.py",
"chars": 787,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "tests/liminal/core/util/test_dict_utils.py",
"chars": 7427,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "tests/liminal/kubernetes/__init__.py",
"chars": 787,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "tests/liminal/kubernetes/test_volume_util.py",
"chars": 2419,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "tests/runners/__init__.py",
"chars": 787,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
},
{
"path": "tests/runners/airflow/__init__.py",
"chars": 787,
"preview": "#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE"
}
]
// ... and 66 more files (download for full content)
About this extraction
This page contains the full source code of the apache/incubator-liminal GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 266 files (596.4 KB), approximately 150.4k tokens, and a symbol index with 464 extracted functions, classes, methods, constants, and types. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.