Showing preview only (1,247K chars total). Download the full file or copy to clipboard to get everything.
Repository: stitchfix/flotilla-os
Branch: master
Commit: d16becadf8cb
Files: 224
Total size: 1.1 MB
Directory structure:
gitextract_unzartjt/
├── .circleci/
│ └── config.yml
├── .github/
│ ├── CODEOWNERS
│ └── PULL_REQUEST_TEMPLATE.md
├── .gitignore
├── .migrations/
│ ├── V20200123054713__initial_table_create.sql
│ ├── V20200123054714__add_spark_extension.sql
│ ├── V20200205133700__executable.sql
│ ├── V20200206115000__template.sql
│ ├── V20200210154600__template_refactor.sql
│ ├── V20200211160100__task_col_fix.sql
│ ├── V20200211161900__template_indicies.sql
│ ├── V20200212101900__template.sql
│ ├── V20200213101400__task_indexes.sql
│ ├── V20200213125200__rename_default_payload.sql
│ ├── V20200225125200__add_limits.sql
│ ├── V20200325125200__add_attempts.sql
│ ├── V20200325125201__add_spawned.sql
│ ├── V20200625125201__add_run_exceptions.sql
│ ├── V20210083054714__metrics_uri.sql
│ ├── V20210427125201__add_active_deadline_seconds.sql
│ ├── V20210807125201__drop_index_container_name.sql
│ ├── V20211007125201__add_description.sql
│ ├── V20220907125201__add_idempotence.sql
│ ├── V20220907125202__add_arch.sql
│ ├── V20221215125203__add_labels.sql
│ ├── V20230718115000__add_ephemeral_storage.sql
│ ├── V20231013191711__add_requires_docker.sql
│ ├── V20231122141100__add_target_cluster.sql
│ ├── V20240205132100__add_service_account.sql
│ ├── V20250122141100__add_cluster_routing.sql
│ └── dev.conf
├── ARA_METRICS_COMPARISON.md
├── Dockerfile
├── LICENSE
├── README.html
├── README.md
├── ara-impact-report-staging.md
├── ara-impact-report.md
├── clients/
│ ├── cluster/
│ │ ├── cluster.go
│ │ └── eks_cluster_client.go
│ ├── httpclient/
│ │ ├── client.go
│ │ └── client_test.go
│ ├── logs/
│ │ ├── eks_cloudwatch_logs_client.go
│ │ ├── eks_s3_logs_client.go
│ │ └── logs.go
│ ├── metrics/
│ │ ├── datadog_metrics_client.go
│ │ └── metrics.go
│ └── middleware/
│ └── client.go
├── conf/
│ └── config.yml
├── config/
│ ├── config.go
│ └── config_test.go
├── datadog-ara-dashboard-api.json
├── docker-compose.yml
├── docs/
│ ├── ara-command-hash-bug-report.md
│ ├── ara-command-hash-fix-locations.md
│ ├── ara-command-hash-history.md
│ ├── ara-instrumentation.md
│ └── ara.md
├── exceptions/
│ └── errors.go
├── execution/
│ ├── adapter/
│ │ ├── eks_adapter.go
│ │ └── eks_adapter_test.go
│ └── engine/
│ ├── dcm.go
│ ├── eks_engine.go
│ ├── emr_engine.go
│ └── engine.go
├── flotilla/
│ ├── app.go
│ ├── endpoints.go
│ ├── endpoints_test.go
│ └── router.go
├── go.mod
├── go.sum
├── log/
│ ├── event.go
│ ├── event_test.go
│ ├── logger.go
│ └── logger_test.go
├── main.go
├── queue/
│ ├── manager.go
│ ├── sqs_manager.go
│ └── sqs_manager_test.go
├── services/
│ ├── definition.go
│ ├── definition_test.go
│ ├── execution.go
│ ├── execution_test.go
│ ├── logs.go
│ ├── logs_test.go
│ ├── template.go
│ └── worker.go
├── state/
│ ├── manager.go
│ ├── models.go
│ ├── models_test.go
│ ├── pg_queries.go
│ ├── pg_state_manager.go
│ └── pg_state_manager_test.go
├── testutils/
│ └── mocks.go
├── tracing/
│ └── tracing.go
├── ui/
│ ├── .gitignore
│ ├── .prettierrc
│ ├── Dockerfile
│ ├── README.md
│ ├── package.json
│ ├── public/
│ │ └── index.html
│ ├── src/
│ │ ├── api.ts
│ │ ├── components/
│ │ │ ├── ARASwitch.tsx
│ │ │ ├── App.tsx
│ │ │ ├── Attribute.tsx
│ │ │ ├── AutoscrollSwitch.tsx
│ │ │ ├── BaseTaskForm.tsx
│ │ │ ├── CloudtrailRecords.tsx
│ │ │ ├── ClusterSelect.tsx
│ │ │ ├── CreateTaskForm.tsx
│ │ │ ├── DeleteTaskButton.tsx
│ │ │ ├── Duration.tsx
│ │ │ ├── EngineTag.tsx
│ │ │ ├── EnvFieldArray.tsx
│ │ │ ├── EnvList.tsx
│ │ │ ├── EnvQueryFilter.tsx
│ │ │ ├── ErrorCallout.tsx
│ │ │ ├── FieldError.tsx
│ │ │ ├── GenericMultiSelect.tsx
│ │ │ ├── GroupNameSelect.tsx
│ │ │ ├── ISO8601AttributeValue.tsx
│ │ │ ├── ListFiltersDropdown.tsx
│ │ │ ├── ListRequest.tsx
│ │ │ ├── Log.tsx
│ │ │ ├── LogProcessor.tsx
│ │ │ ├── LogRequesterCloudWatchLogs.tsx
│ │ │ ├── LogRequesterS3.tsx
│ │ │ ├── LogVirtualized.tsx
│ │ │ ├── LogVirtualizedRow.tsx
│ │ │ ├── LogVirtualizedSearch.tsx
│ │ │ ├── Navigation.tsx
│ │ │ ├── NodeLifecycleSelect.tsx
│ │ │ ├── Pagination.tsx
│ │ │ ├── QueryParams.tsx
│ │ │ ├── Request.tsx
│ │ │ ├── ResourceUsageValue.tsx
│ │ │ ├── Run.tsx
│ │ │ ├── RunAttributes.tsx
│ │ │ ├── RunDebugAttributes.tsx
│ │ │ ├── RunEvents.tsx
│ │ │ ├── RunSidebar.tsx
│ │ │ ├── RunStatusSelect.tsx
│ │ │ ├── RunTag.tsx
│ │ │ ├── Runs.tsx
│ │ │ ├── SettingsButton.tsx
│ │ │ ├── SortableTh.tsx
│ │ │ ├── StopRunButton.tsx
│ │ │ ├── Table.tsx
│ │ │ ├── TagsSelect.tsx
│ │ │ ├── Task.tsx
│ │ │ ├── TaskDetails.tsx
│ │ │ ├── TaskExecutionForm.tsx
│ │ │ ├── TaskRuns.tsx
│ │ │ ├── Tasks.tsx
│ │ │ ├── Template.tsx
│ │ │ ├── TemplateDetails.tsx
│ │ │ ├── TemplateExecutionForm.tsx
│ │ │ ├── TemplateHistoryTable.tsx
│ │ │ ├── TemplateRunForm.tsx
│ │ │ ├── Templates.tsx
│ │ │ ├── Toaster.ts
│ │ │ ├── Toggler.tsx
│ │ │ ├── UpdateTaskForm.tsx
│ │ │ ├── ViewHeader.tsx
│ │ │ └── __tests__/
│ │ │ ├── BaseTaskForm.spec.tsx
│ │ │ ├── ClusterSelect.spec.tsx
│ │ │ ├── CreateTaskForm.spec.tsx
│ │ │ ├── DeleteTaskButton.spec.tsx
│ │ │ ├── EnvFieldArray.spec.tsx
│ │ │ ├── GroupNameSelect.spec.tsx
│ │ │ ├── ListRequest.spec.tsx
│ │ │ ├── LogProcessor.spec.tsx
│ │ │ ├── LogVirtualized.spec.tsx
│ │ │ ├── LogVirtualizedSearch.spec.tsx
│ │ │ ├── Pagination.spec.tsx
│ │ │ ├── QueryParams.spec.tsx
│ │ │ ├── Request.spec.tsx
│ │ │ ├── Run.spec.tsx
│ │ │ ├── Runs.spec.tsx
│ │ │ ├── StopRunButton.spec.tsx
│ │ │ ├── TaskRuns.spec.tsx
│ │ │ ├── Tasks.spec.tsx
│ │ │ └── UpdateTaskForm.spec.tsx
│ │ ├── constants.ts
│ │ ├── helpers/
│ │ │ ├── FlotillaClient.ts
│ │ │ ├── __mocks__/
│ │ │ │ └── FlotillaClient.ts
│ │ │ ├── __tests__/
│ │ │ │ ├── FlotillaClient.spec.ts
│ │ │ │ ├── getInitialValuesForTaskRun.spec.ts
│ │ │ │ └── pageToOffsetLimit.spec.ts
│ │ │ ├── calculateDuration.ts
│ │ │ ├── constructDefaultObjectFromJsonSchema.ts
│ │ │ ├── getEnhancedRunStatus.ts
│ │ │ ├── getInitialValuesForExecutionForm.ts
│ │ │ ├── getOwnerIdRunTagFromCookies.ts
│ │ │ ├── pageToOffsetLimit.ts
│ │ │ ├── runFormHelpers.ts
│ │ │ ├── selectHelpers.ts
│ │ │ ├── taskFormHelpers.ts
│ │ │ └── testHelpers.ts
│ │ ├── index.css
│ │ ├── index.tsx
│ │ ├── localstorage.ts
│ │ ├── react-app-env.d.ts
│ │ ├── setupTests.js
│ │ ├── state/
│ │ │ ├── runView.ts
│ │ │ ├── settings.ts
│ │ │ └── store.ts
│ │ ├── types.ts
│ │ └── workers/
│ │ ├── index.ts
│ │ └── log.worker.ts
│ └── tsconfig.json
├── utils/
│ ├── dd_tracing.go
│ └── utils.go
└── worker/
├── events_worker.go
├── events_worker_test.go
├── retry_worker.go
├── retry_worker_test.go
├── status_worker.go
├── status_worker_test.go
├── submit_worker.go
├── submit_worker_test.go
├── worker.go
├── worker_manager.go
└── worker_test.go
================================================
FILE CONTENTS
================================================
================================================
FILE: .circleci/config.yml
================================================
---
version: 2
jobs:
build:
working_directory: ~/go/src/github.com/stitchfix/flotilla-os
docker:
- image: cimg/go:1.24
environment:
FLOTILLA_MODE: test
DATABASE_URL: postgresql://flotilla:flotilla@localhost/flotilla?sslmode=disable
READONLY_DATABASE_URL: postgresql://flotilla:flotilla@localhost/flotilla?sslmode=disable
PG_USER: flotilla
PG_HOST: 127.0.0.1
GO111MODULE: "on"
- image: cimg/postgres:17.4
environment:
POSTGRES_USER: flotilla
POSTGRES_DB: flotilla
POSTGRES_PASSWORD: flotilla
steps:
- checkout
- run:
name: Installing Flyway
command: curl -sL https://repo1.maven.org/maven2/org/flywaydb/flyway-commandline/6.5.7/flyway-commandline-6.5.7-linux-x64.tar.gz | tar xz && sudo ln -s "$(pwd)/flyway-6.5.7/flyway" /usr/local/bin/flyway
- run:
name: Waiting for Postgres to be ready
command: dockerize -wait tcp://localhost:5432 -timeout 5m
- run:
name: Set Up DB
command: |
pwd
ls -a
flyway baseline -configFiles=./.migrations/dev.conf \
-user=flotilla \
-password=flotilla
flyway migrate -configFiles=./.migrations/dev.conf \
-locations=filesystem:./.migrations/ \
-user=flotilla \
-password=flotilla
- run: go get ./...
- run: go test -v ./...
================================================
FILE: .github/CODEOWNERS
================================================
# This file uses the GitHub CODEOWNERS convention to assign PR reviewers:
# https://help.github.com/articles/about-codeowners/
* @stitchfix/dev-platform
================================================
FILE: .github/PULL_REQUEST_TEMPLATE.md
================================================
## PROBLEM
## SOLUTION
================================================
FILE: .gitignore
================================================
# Binaries for programs and plugins
*.exe
*.dll
*.so
*.dylib
# Test binary, build with `go test -c`
*.test
# Output of the go coverage tool, specifically when used with LiteIDE
*.out
# Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736
.glide/
vendor/**
!vendor/vendor.json
.idea
*.iml
flotilla-os
# gh-pages and ui_branch
node_modules
.cache/
.DS_Store
yarn-error.log
ui/build/
.env
================================================
FILE: .migrations/V20200123054713__initial_table_create.sql
================================================
--
-- Definitions
--
CREATE TABLE IF NOT EXISTS task_def (
definition_id character varying PRIMARY KEY,
alias character varying,
image character varying NOT NULL,
group_name character varying NOT NULL,
memory integer,
cpu integer,
gpu integer,
command text,
env jsonb,
-- Refactor these
"user" character varying,
arn character varying,
container_name character varying NOT NULL,
task_type character varying,
privileged boolean,
adaptive_resource_allocation boolean,
-- Refactor these
CONSTRAINT task_def_alias UNIQUE(alias)
);
CREATE TABLE IF NOT EXISTS task_def_ports (
task_def_id character varying NOT NULL REFERENCES task_def(definition_id),
port integer NOT NULL,
CONSTRAINT task_def_ports_pkey PRIMARY KEY(task_def_id, port)
);
CREATE INDEX IF NOT EXISTS ix_task_def_alias ON task_def(alias);
CREATE INDEX IF NOT EXISTS ix_task_def_group_name ON task_def(group_name);
CREATE INDEX IF NOT EXISTS ix_task_def_image ON task_def(image);
CREATE INDEX IF NOT EXISTS ix_task_def_env ON task_def USING gin (env jsonb_path_ops);
--
-- Runs
--
CREATE TABLE IF NOT EXISTS task (
run_id character varying NOT NULL PRIMARY KEY,
definition_id character varying REFERENCES task_def(definition_id),
alias character varying,
image character varying,
cluster_name character varying,
exit_code integer,
exit_reason character varying,
status character varying,
queued_at timestamp with time zone,
started_at timestamp with time zone,
finished_at timestamp with time zone,
instance_id character varying,
instance_dns_name character varying,
group_name character varying,
env jsonb,
-- Refactor these --
task_arn character varying,
docker_id character varying,
"user" character varying,
task_type character varying,
-- Refactor these --
command text,
command_hash text,
memory integer,
cpu integer,
gpu integer,
ephemeral_storage integer,
node_lifecycle text,
engine character varying DEFAULT 'eks' NOT NULL,
container_name text,
pod_name text,
namespace text,
max_cpu_used integer,
max_memory_used integer,
pod_events jsonb,
cloudtrail_notifications jsonb
);
CREATE INDEX IF NOT EXISTS ix_task_definition_id ON task(definition_id);
CREATE INDEX IF NOT EXISTS ix_task_cluster_name ON task(cluster_name);
CREATE INDEX IF NOT EXISTS ix_task_status ON task(status);
CREATE INDEX IF NOT EXISTS ix_task_group_name ON task(group_name);
CREATE INDEX IF NOT EXISTS ix_task_env ON task USING gin (env jsonb_path_ops);
CREATE INDEX IF NOT EXISTS ix_task_definition_id ON task(definition_id);
CREATE INDEX IF NOT EXISTS ix_task_task_arn ON task(task_arn);
CREATE INDEX IF NOT EXISTS ix_task_definition_id_started_at_desc ON task(definition_id, started_at DESC NULLS LAST);
CREATE INDEX IF NOT EXISTS ix_task_definition_id_started_at_desc_engine ON task(definition_id, started_at DESC NULLS LAST, engine);
CREATE INDEX IF NOT EXISTS ix_finished_at_status_cluster_name ON task USING btree (cluster_name, status, finished_at DESC);
CREATE INDEX IF NOT EXISTS ix_task_definition_id_started_at_asc ON task USING btree (definition_id, started_at);
CREATE INDEX IF NOT EXISTS ix_task_pod_events ON task USING gin (pod_events jsonb_path_ops);
CREATE INDEX IF NOT EXISTS ix_task_queued_at_status_engine ON task USING btree (queued_at, status, engine);
CREATE INDEX IF NOT EXISTS task_definition_id_engine_started_at_index ON task USING btree (definition_id, engine, started_at DESC);
--
-- Status
--
CREATE TABLE IF NOT EXISTS task_status (
status_id integer NOT NULL PRIMARY KEY,
task_arn character varying,
status_version integer NOT NULL,
status character varying,
"timestamp" timestamp with time zone DEFAULT now()
);
CREATE INDEX IF NOT EXISTS ix_task_status_task_arn ON task_status(task_arn);
CREATE SEQUENCE IF NOT EXISTS task_status_status_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
ALTER TABLE ONLY task_status ALTER COLUMN status_id SET DEFAULT nextval('task_status_status_id_seq'::regclass);
--
-- Tags
--
CREATE TABLE IF NOT EXISTS tags (
text character varying NOT NULL PRIMARY KEY
);
CREATE TABLE IF NOT EXISTS task_def_tags (
tag_id character varying NOT NULL REFERENCES tags(text),
task_def_id character varying NOT NULL REFERENCES task_def(definition_id)
);
CREATE TABLE IF NOT EXISTS worker (
worker_type character varying,
engine character varying,
count_per_instance integer
);
================================================
FILE: .migrations/V20200123054714__add_spark_extension.sql
================================================
ALTER TABLE task ADD COLUMN IF NOT EXISTS spark_extension JSONB;
================================================
FILE: .migrations/V20200205133700__executable.sql
================================================
ALTER TABLE task
ADD COLUMN executable_id VARCHAR,
ADD COLUMN executable_type VARCHAR DEFAULT 'task_definition';
================================================
FILE: .migrations/V20200206115000__template.sql
================================================
CREATE TABLE template (
template_id VARCHAR PRIMARY KEY,
type VARCHAR NOT NULL,
version INTEGER NOT NULL,
schema JSONB NOT NULL,
command_template TEXT NOT NULL,
image VARCHAR NOT NULL,
memory INTEGER NOT NULL,
gpu INTEGER NOT NULL,
cpu INTEGER NOT NULL,
env JSONB,
privileged BOOLEAN,
adaptive_resource_allocation BOOLEAN,
container_name VARCHAR NOT NULL,
CONSTRAINT template_type_version UNIQUE(type, version)
);
ALTER TABLE task ADD COLUMN IF NOT EXISTS executable_request_custom JSONB;
================================================
FILE: .migrations/V20200210154600__template_refactor.sql
================================================
ALTER TABLE template DROP CONSTRAINT template_type_version;
ALTER TABLE template RENAME COLUMN type to template_name;
ALTER TABLE template ADD CONSTRAINT template_name_version UNIQUE(template_name, version);
================================================
FILE: .migrations/V20200211160100__task_col_fix.sql
================================================
ALTER TABLE task RENAME COLUMN executable_request_custom to execution_request_custom;
================================================
FILE: .migrations/V20200211161900__template_indicies.sql
================================================
CREATE INDEX IF NOT EXISTS ix_template_id ON template(template_id);
CREATE INDEX IF NOT EXISTS ix_template_name ON template(template_name);
================================================
FILE: .migrations/V20200212101900__template.sql
================================================
ALTER TABLE template ADD COLUMN default_payload JSONB;
ALTER TABLE template ADD COLUMN avatar_uri VARCHAR;
================================================
FILE: .migrations/V20200213101400__task_indexes.sql
================================================
CREATE INDEX IF NOT EXISTS ix_task_executable_id ON task(executable_id);
CREATE INDEX IF NOT EXISTS ix_task_executable_id_started_at_desc ON task(executable_id, started_at DESC NULLS LAST);
CREATE INDEX IF NOT EXISTS ix_task_executable_id_started_at_desc_engine ON task(executable_id, started_at DESC NULLS LAST, engine);
================================================
FILE: .migrations/V20200213125200__rename_default_payload.sql
================================================
ALTER TABLE template RENAME COLUMN default_payload to defaults;
================================================
FILE: .migrations/V20200225125200__add_limits.sql
================================================
ALTER TABLE task ADD COLUMN memory_limit integer;
ALTER TABLE task ADD COLUMN cpu_limit integer;
================================================
FILE: .migrations/V20200325125200__add_attempts.sql
================================================
ALTER TABLE task ADD COLUMN attempt_count integer;
================================================
FILE: .migrations/V20200325125201__add_spawned.sql
================================================
ALTER TABLE task ADD COLUMN spawned_runs jsonb;
================================================
FILE: .migrations/V20200625125201__add_run_exceptions.sql
================================================
ALTER TABLE task ADD COLUMN run_exceptions jsonb;
================================================
FILE: .migrations/V20210083054714__metrics_uri.sql
================================================
ALTER TABLE task ADD COLUMN IF NOT EXISTS metrics_uri varchar;
================================================
FILE: .migrations/V20210427125201__add_active_deadline_seconds.sql
================================================
ALTER TABLE task ADD COLUMN active_deadline_seconds integer;
================================================
FILE: .migrations/V20210807125201__drop_index_container_name.sql
================================================
alter table task_def alter column container_name drop not null;
================================================
FILE: .migrations/V20211007125201__add_description.sql
================================================
ALTER TABLE task ADD COLUMN IF NOT EXISTS description varchar;
================================================
FILE: .migrations/V20220907125201__add_idempotence.sql
================================================
ALTER TABLE task ADD COLUMN IF NOT EXISTS idempotence_key varchar;
================================================
FILE: .migrations/V20220907125202__add_arch.sql
================================================
ALTER TABLE task ADD COLUMN IF NOT EXISTS arch varchar;
================================================
FILE: .migrations/V20221215125203__add_labels.sql
================================================
ALTER TABLE task ADD COLUMN IF NOT EXISTS labels jsonb;
================================================
FILE: .migrations/V20230718115000__add_ephemeral_storage.sql
================================================
ALTER TABLE task_def ADD COLUMN IF NOT EXISTS ephemeral_storage INTEGER;
ALTER TABLE task ADD COLUMN IF NOT EXISTS ephemeral_storage INTEGER;
================================================
FILE: .migrations/V20231013191711__add_requires_docker.sql
================================================
ALTER TABLE task_def ADD COLUMN IF NOT EXISTS requires_docker BOOLEAN DEFAULT(false);
ALTER TABLE task ADD COLUMN IF NOT EXISTS requires_docker BOOLEAN DEFAULT(false);
================================================
FILE: .migrations/V20231122141100__add_target_cluster.sql
================================================
ALTER TABLE task_def ADD COLUMN IF NOT EXISTS target_cluster VARCHAR;
================================================
FILE: .migrations/V20240205132100__add_service_account.sql
================================================
ALTER TABLE task ADD COLUMN IF NOT EXISTS service_account VARCHAR;
================================================
FILE: .migrations/V20250122141100__add_cluster_routing.sql
================================================
DO $$
BEGIN
IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'cluster_status') THEN
CREATE TYPE cluster_status AS ENUM ('active', 'maintenance', 'offline');
END IF;
END$$;
CREATE TABLE IF NOT EXISTS cluster_state (
id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
name VARCHAR NOT NULL,
cluster_version VARCHAR NOT NULL DEFAULT '',
status cluster_status NOT NULL DEFAULT 'active',
status_reason VARCHAR,
status_since TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
capabilities VARCHAR[] NOT NULL DEFAULT '{}',
allowed_tiers VARCHAR[] NOT NULL DEFAULT '{}',
region VARCHAR NOT NULL,
updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
namespace VARCHAR NOT NULL DEFAULT '',
emr_virtual_cluster VARCHAR NOT NULL DEFAULT '',
spark_server_uri VARCHAR NOT NULL DEFAULT ''
);
CREATE INDEX IF NOT EXISTS ix_cluster_state_name ON cluster_state(name);
CREATE INDEX IF NOT EXISTS ix_cluster_state_status ON cluster_state(status);
DO $$
BEGIN
IF NOT EXISTS (SELECT 1
FROM information_schema.columns
WHERE table_name='task' AND column_name='tier')
THEN
ALTER TABLE task ADD COLUMN tier TEXT;
END IF;
END$$;
================================================
FILE: .migrations/dev.conf
================================================
flyway.url=jdbc:postgresql://127.0.0.1:5432/flotilla
flyway.user=flotilla
flyway.password=flotilla
flyway.cleanDisabled=true
flyway.group=true
flyway.locations=filesystem:.migrations
================================================
FILE: ARA_METRICS_COMPARISON.md
================================================
# ARA Metrics Implementation Comparison
Comparing `ez/ara-metrics` (HEAD) vs `ez/ara-metrics-alt`
## Overview
Both implementations add instrumentation to track Auto Resource Adjustment (ARA) behavior to identify over-provisioning patterns, particularly the ~300GB memory limit issue. However, they differ significantly in approach, metrics design, logging strategy, and code structure.
---
## Similarities
### Shared Goals
- Track ARA resource adjustments
- Detect when jobs hit maximum resource limits (especially 350GB memory)
- Enable monitoring to identify over-provisioning patterns
- Instrument `adaptiveResources()` function
- Add structured logging for debugging
### Common Changes
- Both modify `execution/adapter/eks_adapter.go`
- Both add new metric constants to `clients/metrics/metrics.go`
- Both track default resources before ARA applies adjustments
- Both detect and report when max bounds are hit
- Both use structured key-value logging format
---
## Key Differences
### 1. **Metric Naming Convention**
**HEAD (`ez/ara-metrics`):**
- Uses hierarchical dot notation: `engine.eks.ara.*`
- Examples: `engine.eks.ara.estimation_attempted`, `engine.eks.ara.memory_increase`
- Consistent with existing codebase pattern (`engine.eks.execute`, etc.)
**Alt (`ez/ara-metrics-alt`):**
- Uses flat namespace: `ara.*`
- Examples: `ara.resource_adjustment`, `ara.memory_increase_ratio`
- Shorter, more concise names
**Winner:** HEAD - Consistent with existing naming conventions
---
### 2. **Metrics Coverage**
**HEAD (10 metrics):**
```go
// Estimation tracking
EngineEKSARAEstimationAttempted // Counter
EngineEKSARAEstimationSucceeded // Counter
EngineEKSARAEstimationFailed // Counter
// Resource tracking
EngineEKSARAMaxResourceHit // Counter (tagged with resource:memory or resource:cpu)
EngineEKSARAMemoryIncrease // Distribution
EngineEKSARACPUIncrease // Distribution
EngineEKSARADefaultMemory // Distribution
EngineEKSARAARAMemory // Distribution
EngineEKSARADefaultCPU // Distribution
EngineEKSARAARACPU // Distribution
```
**Alt (8 metrics):**
```go
// Core tracking
ARAResourceAdjustment // Counter (when ARA triggers)
ARANoHistoricalData // Counter (when no data found)
// Ratio tracking
ARAMemoryIncreaseRatio // Histogram
ARACPUIncreaseRatio // Histogram
// Limit detection
ARAHitMaxMemory // Counter
ARAHitMaxCPU // Counter
// Final distributions
ARAFinalMemoryMB // Histogram
ARAFinalCPUMillicores // Histogram
```
**Comparison:**
- **HEAD:** More granular - separates estimation attempts from successes/failures
- **ALT:** More focused - tracks key ratios and final states
- **HEAD:** Tracks resource increases as absolute values
- **ALT:** Tracks increases as ratios (better for understanding relative growth)
**Winner:** Tie - Both approaches have merit. HEAD provides more granularity; ALT provides better insight into relative growth.
---
### 3. **Logging Strategy**
**HEAD:**
- Logging only occurs when max resource bounds are hit
- Uses stored logger instance (field on `eksAdapter`)
- Separate `emitARAMetrics()` method for structured logging
- Logs once per max-bound-hit event
- Fields: `run_id`, `definition_id`, `executable_id`, `command`, default/final resources, max hit flags
**ALT:**
- **Multiple logging points:**
1. When ARA triggers adjustments (INFO level)
2. When max limits hit (WARN level)
3. In `state/pg_state_manager.go` for historical data lookups (success/no data/error)
- Uses inline `flotillaLog.NewLogger(nil, nil)` - creates new logger instances
- More verbose logging at each step
- Detailed structured fields including ratios, overage amounts, cluster name
- Separate logs for historical data lookup success/failure
**Winner:** ALT - More comprehensive logging provides better debugging capability
---
### 4. **Logger Management**
**HEAD:**
```go
type eksAdapter struct {
logger flotillaLog.Logger // Stored as field
}
func NewEKSAdapter(logger flotillaLog.Logger) (EKSAdapter, error) {
adapter := eksAdapter{logger: logger}
return &adapter, nil
}
// Usage in HEAD
if a.logger == nil {
return
}
a.logger.Log(logFields...)
```
**ALT:**
```go
// No logger field stored
// Creates new logger instances inline
_ = flotillaLog.NewLogger(nil, nil).Log(...)
```
**Comparison:**
- **HEAD:** Dependency injection pattern - logger passed via constructor, stored as field
- **ALT:** Creates new logger instances inline (less efficient, harder to test)
- **HEAD:** Requires updating `eks_engine.go` to pass logger (which it does)
- **ALT:** No changes needed to constructor/initialization
**Winner:** HEAD - Better design pattern (dependency injection), more testable
---
### 5. **Tagging Strategy**
**HEAD:**
- No tags used on metrics (empty `[]string{}`)
- Simpler, avoids cardinality concerns
- May limit filtering/grouping capabilities in DataDog
**ALT:**
- Uses cluster tags: `[]string{fmt.Sprintf("cluster:%s", run.ClusterName)}`
- Explicitly documented as "low-cardinality tags to avoid excessive volume"
- Enables per-cluster analysis
**Winner:** ALT - Tags enable better filtering and per-cluster analysis
---
### 6. **Metric Types**
**HEAD:**
- Uses `Distribution()` for all numeric metrics
- Uses `Increment()` for counters
**ALT:**
- Uses `Histogram()` for ratios and final values
- Uses `Increment()` for counters
**Comparison:**
- DataDog treats Histogram and Distribution similarly for most use cases
- Both approaches are valid
**Winner:** Tie - No significant difference
---
### 7. **Code Structure**
**HEAD:**
- Cleaner separation: detects max hits after bounds checking
- Uses helper method `emitARAMetrics()` to centralize logging logic
- More modular: logging logic separate from bounds checking
**ALT:**
- Metrics/logging embedded directly in `checkResourceBounds()`
- Requires passing additional parameters (`run`, `executable`, `defaultCPU`, etc.) to `checkResourceBounds()`
- More invasive changes to function signatures
- Inline logging at multiple points
**Winner:** HEAD - Better code organization, less invasive changes
---
### 8. **State Manager Instrumentation**
**HEAD:**
- No changes to `state/pg_state_manager.go`
- Only instruments the adapter layer
**ALT:**
- **Adds instrumentation to `state/pg_state_manager.go`**
- Logs when historical data is found/not found/errors occur
- Provides visibility into the data lookup layer
- Helps debug issues with historical data queries
**Winner:** ALT - Provides better end-to-end visibility
---
### 9. **Test Coverage**
**HEAD:**
- **Comprehensive test suite** (524 lines in `eks_adapter_test.go`)
- Tests multiple scenarios:
- ARA enabled with successful estimation
- GPU jobs (skip ARA)
- Estimation failures
- Max resource bounds hitting
- ARA disabled
- Logger nil handling
- Mock implementations for logger and state manager
**ALT:**
- No test files included
**Winner:** HEAD - Significantly better test coverage
---
### 10. **Documentation**
**HEAD:**
- Commit message describes changes
- No separate documentation file
**ALT:**
- **Comprehensive 317-line documentation** (`docs/ara-instrumentation.md`)
- Includes:
- Overview of ARA algorithm
- Historical context of ARA implementation
- Detailed explanation of metrics
- DataDog query examples
- Alert recommendations
- Investigation workflow
- Future improvement suggestions
- Extremely helpful for operators and future developers
**Winner:** ALT - Outstanding documentation
---
### 11. **Detection Logic**
**HEAD:**
```go
// After bounds checking
cpuRequestBeforeBounds := cpuRequest
memRequestBeforeBounds := memRequest
cpuRequest, memRequest = a.checkResourceBounds(...)
// Then detect hits
if memRequestBeforeBounds > maxMem {
maxMemHit = true
// emit metrics/logs
}
```
**ALT:**
```go
// Inside checkResourceBounds()
if mem > maxMem {
// Emit metrics and logs immediately
_ = metrics.Increment(metrics.ARAHitMaxMemory, ...)
// ... logging ...
mem = maxMem
}
```
**Comparison:**
- **HEAD:** Two-step process - check bounds, then detect if hit
- **ALT:** Single-step - detect and log during bounds checking
- **ALT:** More straightforward, less code
**Winner:** ALT - Simpler, more direct approach
---
### 12. **ARA Trigger Detection**
**HEAD:**
- No explicit "ARA triggered" detection
- Only tracks estimation attempts/success/failure
- Doesn't distinguish between "ARA found same values" vs "ARA actually changed resources"
**ALT:**
```go
araTriggered := (estimatedResources.Cpu != cpuRequest ||
estimatedResources.Memory != memRequest)
```
- Explicitly detects when ARA actually changes resources
- Only logs/increments metrics when resources actually change
- More precise tracking
**Winner:** ALT - More accurate tracking of actual ARA adjustments
---
## Best-of-Breed Recommendation
**The ideal solution would combine:**
### From HEAD:
1. ? **Metric naming convention** - Use `engine.eks.ara.*` pattern
2. ? **Logger as dependency** - Store logger as field, inject via constructor
3. ? **Code organization** - Separate `emitARAMetrics()` method
4. ? **Test coverage** - Include comprehensive test suite
5. ? **Granular metrics** - Track estimation attempts/success/failure separately
### From ALT:
1. ? **Logging strategy** - Log when ARA triggers AND when limits hit
2. ? **State manager instrumentation** - Add logging in `pg_state_manager.go`
3. ? **Documentation** - Include comprehensive docs file
4. ? **Tagging** - Use cluster tags for filtering
5. ? **Ratio metrics** - Track ratios instead of/in addition to absolute increases
6. ? **ARA trigger detection** - Explicitly detect when ARA actually changes resources
### Hybrid Approach:
```go
// Metrics (combine both approaches)
- engine.eks.ara.estimation_attempted // Counter
- engine.eks.ara.estimation_succeeded // Counter
- engine.eks.ara.estimation_failed // Counter
- engine.eks.ara.resource_adjustment // Counter (only when changed)
- engine.eks.ara.memory_increase_ratio // Histogram (ALT's approach)
- engine.eks.ara.cpu_increase_ratio // Histogram
- engine.eks.ara.hit_max_memory // Counter
- engine.eks.ara.hit_max_cpu // Counter
- engine.eks.ara.final_memory_mb // Histogram
- engine.eks.ara.final_cpu_millicores // Histogram
// Logging (ALT's comprehensive approach)
- Log when ARA triggers (INFO)
- Log when limits hit (WARN)
- Log in state manager for historical lookups
// Code structure (HEAD's approach)
- Store logger as field
- Separate emitARAMetrics() method
- Use cluster tags on metrics
// Documentation
- Include ALT's comprehensive docs
// Tests
- Include HEAD's comprehensive test suite
```
---
## Verdict
**Best Overall:** Neither solution is perfect alone. **ALT is closer to production-ready** due to:
- Comprehensive documentation
- Better logging strategy
- End-to-end instrumentation
- Ratio-based metrics (easier to understand)
**But HEAD has better engineering practices:**
- Dependency injection
- Test coverage
- Code organization
**Recommendation:** Start with ALT as the base, then incorporate HEAD's improvements:
1. Store logger as field (HEAD)
2. Add test suite (HEAD)
3. Optionally adjust metric names to match HEAD's convention
4. Keep ALT's logging and documentation
This hybrid would be the best-of-breed solution.
================================================
FILE: Dockerfile
================================================
FROM golang:latest
RUN mkdir -p /go/src/github.com/stitchfix/flotilla-os
ADD . /go/src/github.com/stitchfix/flotilla-os
WORKDIR /go/src/github.com/stitchfix/flotilla-os
RUN go install github.com/stitchfix/flotilla-os
ENTRYPOINT /go/bin/flotilla-os /go/src/github.com/stitchfix/flotilla-os/conf
================================================
FILE: LICENSE
================================================
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "{}"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright {yyyy} {name of copyright owner}
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
================================================
FILE: README.html
================================================
<!DOCTYPE html><html><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1"><style>body {
max-width: 980px;
border: 1px solid #ddd;
outline: 1300px solid #fff;
margin: 16px auto;
}
body .markdown-body
{
padding: 45px;
}
@font-face {
font-family: fontawesome-mini;
src: url(data:font/woff;charset=utf-8;base64,d09GRgABAAAAABE0AA8AAAAAHWwAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAABHU1VCAAABWAAAADsAAABUIIslek9TLzIAAAGUAAAAQwAAAFY3d1HZY21hcAAAAdgAAACqAAACOvWLi0FjdnQgAAAChAAAABMAAAAgBtX/BGZwZ20AAAKYAAAFkAAAC3CKkZBZZ2FzcAAACCgAAAAIAAAACAAAABBnbHlmAAAIMAAABdQAAAjkYT9TNWhlYWQAAA4EAAAAMwAAADYQ6WvNaGhlYQAADjgAAAAfAAAAJAc6A1pobXR4AAAOWAAAACAAAAA0Kmz/7mxvY2EAAA54AAAAHAAAABwQPBJubWF4cAAADpQAAAAgAAAAIAEHC/NuYW1lAAAOtAAAAYQAAALxhQT4h3Bvc3QAABA4AAAAfgAAAMS3SYh9cHJlcAAAELgAAAB6AAAAhuVBK7x4nGNgZGBg4GIwYLBjYHJx8wlh4MtJLMljkGJgYYAAkDwymzEnMz2RgQPGA8qxgGkOIGaDiAIAJjsFSAB4nGNgZHZmnMDAysDAVMW0h4GBoQdCMz5gMGRkAooysDIzYAUBaa4pDA4Pwz+yMwf9z2KIYg5imAYUZgTJAQDcoQvQAHic7ZHNDYJAFIRnBXf94cDRIiyCKkCpwFCPJ092RcKNDoYKcN4+EmMPvpdvk539zQyAPYBCXEUJhBcCrJ5SQ9YLnLJe4qF5rdb+uWPDngNHTkta101pNyWa8lMhn6xx2dqUnW4q9YOIhAOOeueMSgsR/6ry+P7O5s6xVNg4chBsHUuFnWNJ8uZYwrw7chrsHXkODo7cB0dHOYCTY8kv0VE2WJKD6gOlWjsxAAB4nGNgQAMSEMgc9D8LhAESbAPdAHicrVZpd9NGFB15SZyELCULLWphxMRpsEYmbMGACUGyYyBdnK2VoIsUO+m+8Ynf4F/zZNpz6Dd+Wu8bLySQtOdwmpOjd+fN1czbZRJaktgL65GUmy/F1NYmjew8CemGTctRfCg7eyFlisnfBVEQrZbatx2HREQiULWusEQQ+x5ZmmR86FFGy7akV03KLT3pLlvjQb1V334aOsqxO6GkZjN0aD2yJVUYVaJIpj1S0qZlqPorSSu8v8LMV81QwohOImm8GcbQSN4bZ7TKaDW24yiKbLLcKFIkmuFBFHmU1RLn5IoJDMoHzZDyyqcR5cP8iKzYo5xWsEu20/y+L3mndzk/sV9vUbbkQB/Ijuzg7HQlX4RbW2HctJPtKFQRdtd3QmzZ7FT/Zo/ymkYDtysyvdCMYKl8hRArP6HM/iFZLZxP+ZJHo1qykRNB62VO7Es+gdbjiClxzRhZ0N3RCRHU/ZIzDPaYPh788d4plgsTAngcy3pHJZwIEylhczRJ2jByYCVliyqp9a6YOOV1WsRbwn7t2tGXzmjjUHdiPFsPHVs5UcnxaFKnmUyd2knNoykNopR0JnjMrwMoP6JJXm1jNYmVR9M4ZsaERCICLdxLU0EsO7GkKQTNoxm9uRumuXYtWqTJA/Xco/f05la4udNT2g70s0Z/VqdiOtgL0+lp5C/xadrlIkXp+ukZfkziQdYCMpEtNsOUgwdv/Q7Sy9eWHIXXBtju7fMrqH3WRPCkAfsb0B5P1SkJTIWYVYhWQGKta1mWydWsFqnI1HdDmla+rNMEinIcF8e+jHH9XzMzlpgSvt+J07MjLj1z7UsI0xx8m3U9mtepxXIBcWZ5TqdZlu/rNMfyA53mWZ7X6QhLW6ejLD/UaYHlRzodY3lBC5p038GQizDkAg6QMISlA0NYXoIhLBUMYbkIQ1gWYQjLJRjC8mMYwnIZhrC8rGXV1FNJ49qZWAZsQmBijh65zEXlaiq5VEK7aFRqQ54SbpVUFM+qf2WgXjzyhjmwFkiXyJpfMc6Vj0bl+NYVLW8aO1fAsepvH472OfFS1ouFPwX/1dZUJb1izcOTq/Abhp5sJ6o2qXh0TZfPVT26/l9UVFgL9BtIhVgoyrJscGcihI86nYZqoJVDzGzMPLTrdcuan8P9NzFCFlD9+DcUGgvcg05ZSVnt4KzV19uy3DuDcjgTLEkxN/P6VvgiI7PSfpFZyp6PfB5wBYxKZdhqA60VvNknMQ+Z3iTPBHFbUTZI2tjOBIkNHPOAefOdBCZh6qoN5E7hhg34BWFuwXknXKJ6oyyH7kXs8yik/Fun4kT2qGiMwLPZG2Gv70LKb3EMJDT5pX4MVBWhqRg1FdA0Um6oBl/G2bptQsYO9CMqdsOyrOLDxxb3lZJtGYR8pIjVo6Of1l6iTqrcfmYUl++dvgXBIDUxf3vfdHGQyrtayTJHbQNTtxqVU9eaQ+NVh+rmUfW94+wTOWuabronHnpf06rbwcVcLLD2bQ7SUiYX1PVhhQ2iy8WlUOplNEnvuAcYFhjQ71CKjf+r+th8nitVhdFxJN9O1LfR52AM/A/Yf0f1A9D3Y+hyDS7P95oTn2704WyZrqIX66foNzBrrblZugbc0HQD4iFHrY64yg18pwZxeqS5HOkh4GPdFeIBwCaAxeAT3bWM5lMAo/mMOT7A58xh0GQOgy3mMNhmzhrADnMY7DKHwR5zGHzBnHWAL5nDIGQOg4g5DJ4wJwB4yhwGXzGHwdfMYfANc+4DfMscBjFzGCTMYbCv6dYwzC1e0F2gtkFVoANTT1jcw+JQU2XI/o4Xhv29Qcz+wSCm/qjp9pD6Ey8M9WeDmPqLQUz9VdOdIfU3Xhjq7wYx9Q+DmPpMvxjLZQa/jHyXCgeUXWw+5++J9w/bxUC5AAEAAf//AA94nIVVX2hbZRQ/5/t7893s5ja9f7ouzdZ0TTqz3bRJmogbWya6bG6Cq0VbSV2ddIJjFtfIQHEig80Hda8yUN/0YQz8AyriiyD+xQd92R4HCnaCb3samnpumrpsCsLlfPf7zvedc37nL3CAtc/5W/wQZGA3tOBSY/g+TMjHmwzEoM1Q8+ZjRZY4oJhmBw5/YB6Za0yC5AkhlwA1A1yCBIBOwCII0Cj0U8BAMdUCzq05sKwkP7SlUY6fcJk4Fb/RyE79/6P5hjM/F4aZiXBoeMgzcqQ4Xi1hPqfDLG5FT+lchCVU3lYMyvuwhl1mqndQL0RsuloLywHtthLXI06OblTrhfWVnpSJ5+mwu/JdbtuN3IAnkW0LLMcRwaC7ktrlzridM6kVdyf9uO1UNBByI7JhwtG2sEwab07ORBeilWhqavJCqV0qzZTOl/7ZXQ5TbTcdcFelyGhhRDAQpdqp1FEX3w3cFTc1k9pJQkmm4ySCbSikxRP2QOfN+0tHS5MrpQuTU1Mk5nw0E5Xa0WvrOwDyGax9yB9ma6DAg82wHc43SAGTI4GjBWebOePAERFE8/AHaQpZASSTy8A4WwZiLQMQ82mFKATO0ILicRAoDm9p5P99E5b/fXG+kQYY3TYUuqmERWYoT0u/GNYL2q/4WB3LaVS+VynXsVYIcWw6DkCh3nX1D+VzlYN4LClF5yexSQos8exqZ3KVP+wtrC54u4Nznq6cq+xpMpUUnZ8FUYzE86ud0g28NOIv3Gj5/rmA3ABs7S/ywzFuQ4qyd6QxfNtiQIaEgp3w/entQg4Vcbqa16M5FfpeUB8t1+qeg7mI7cUyOe79wOk86gSxkVec4KPTX69++5x68Yubn5/F+w52z7u08sJX7fZXv8ekT/d2mILJxq6sn+SC6qEJknzLJCxyZEKwWVqYmAPBxBE/9DLeZiWHu7lcr/VytrCRuHojncNuTt9h46tmacmYisnSamdN2bZptcsmSysdVsy1PrOvOzF3xN64Rb937t/og9KHxYdcjIUqFAmIAHGHNzlns+RTPgeUYAQm9DwpNxfxbhhBHPaw3/gfTcXO2L+eJVIx5nsyGkvm9X4/f+bGkH45G0PaSjcMXTjcZyTvi3UdHoCDjQd3IDUVsgwYmUoJK/gp4JJxeRI0MKHZIkgynyIBqBTOUs6rOVCojvjZ4mCQz49ZMlMcp8QoYk6NoBfsxnJtsBohpa8iGJS+ZH7gU7NxME6cmF+t7cO9vB8d3jTWSct0ycW9ranXmolNDwmVkNnxe+8JtoztwS5rKJ0xWS95tQ/1zMYzg69MzUZnNtl1ofNbsml/OJm6f9wjRjpnu2o4MzHzn77IQkRd+1DjwMQ2pqSjGMMhyjrgTbBAKksuUm0iU7hI0aN2wOKOq7WYBSH0HGihj/jkiPxAfmwsEbfYrjMG+j3ij932Db/LV7I/xruNrhnroxjR9HRMb2nTvO0ZXOoHPk8H2ZhDPx93qcE/53sH5np/dkIP7zzhTVKdR/BAY/9ElkkR+A6lJGsqpJ4oQcTxpvBT3Kn58VkaJjgHyPEIws57xkaHh9KuVpDEpJZeMbZ5w/zBHi5NMQ4r5VphsFqID7TyB9eR4pX216c3AHxpdAwoqU9qg0ZJ6yVLKmMSz1iG2z27ifx18NkY0LPx1W/wCc2l5LrznrIsiKsqbmB78A9wIGx4tI8rjihVHJyY9pgMirenVq0yWg7Iw7eogG7ZgYM3qR9959A/fZkg6MnD/exlkmc+jWV4SB15XUR+eqC6l6ZmgPtN9z5JMfik05OV8ljylunJ4J+wA/FUaQSSKotsYsCWqaPBidBLcxkWx7XKFRIb45TGaEhjlF9uUVPqXOtcIwsXbBvfoZXIyRYFdkfnqjExH98xpnPczqzjX/uNdO1Y17Wpi5+6Ts8BXtjVFasp9KZ1mOiNbH65c5w6HgmyF2jFCZywM8mWjRc7T5Pmt0lRy7Y71+jYbpGyvwG4sH0XeJxjYGRgYADiwBB/53h+m68M3MwvgCIM1z5N/g6j///9v5H5BbMnkMvBwAQSBQCIcA9gAHicY2BkYGAO+p8FJF/8//v/F/MLBqAICuAFALYQB5kAeJxjfsHAwLwAiCNB+P9fbJjJmoGBMRUo/wKCAfO2EnQAAAAAANoBXgGcAgICVALaA1IDvAPkBAYEPARyAAEAAAANAF0ABAAAAAAAAgAUACQAcwAAAG4LcAAAAAB4nHWRzWrCQBSFT+pPqUIXLXTTzayKUohGKIibCoLuhbrrYtTRxCYZmYyKyz5Fd32HvlDfoO/QkziIFJtw9bvnnpl7ZwLgBt/wcHieGAf2UGd24Atcou+4RH3kuEweO66QXx1XyaHjGh6ROa7jFp/cwStfMVvhy7GHO+/e8QWuvcBxifqz4zL5xXGF/Oa4Sn53XMPE+3Bcx4P3M9DrvYmWoRWNQVN02kFXTPdCU4pSGQu5saE2meiLhU6timPtz3SSs9ypTCdqrJabWJoT5QQnymSRTkXgt0/UkUqVkVbN807ZdtmxdiEWRidi6HqItdErNbN+aO2612qd9sYAGmvsYRBhyUu0EGhQbfK/gzYCdElTOgSdB1eEFBIxFYkNV4RFJWPeZyyYpVQVHTHZx4y/yVGX2LGWFZri51TccUOn5B7nPefVCSPvGhVVwUl9znveO2KkhV8Wk82PZ8qwZf8OVcu1+fSmWCMw/HMOwXvKaysqM+p+cVuWag8tvv+c+xdd+4+teJxtjUEOwiAURJla24KliQfhUA2g/Sl+CKXx+loNrpzVezOLEY34Ron/0WhwQoszOvQYIKFwwQiNSbSBeO2SZ0tBP4j3zVjKNng32ZmtD1VVXCuOiw/pJ8S3WOU6l+K5UOTaDC4+2TjKMtN9KQf1ezLx/Sg/00FCvABHhjDjAAB4nGPw3sFwIihiIyNjX+QGxp0cDBwMyQUbGVidNjEwMmiBGJu5mBg5ICw+BjCLzWkX0wGgNCeQze60i8EBwmZmcNmowtgRGLHBoSNiI3OKy0Y1EG8XRwMDI4tDR3JIBEhJJBBs5mFi5NHawfi/dQNL70YmBhcADHYj9AAA) format('woff');
}
.markdown-body {
font-family: sans-serif;
-ms-text-size-adjust: 100%;
-webkit-text-size-adjust: 100%;
color: #333333;
overflow: hidden;
font-family: "Helvetica Neue", Helvetica, "Segoe UI", Arial, freesans, sans-serif;
font-size: 16px;
line-height: 1.6;
word-wrap: break-word;
}
.markdown-body a {
background: transparent;
}
.markdown-body a:active,
.markdown-body a:hover {
outline: 0;
}
.markdown-body b,
.markdown-body strong {
font-weight: bold;
}
.markdown-body mark {
background: #ff0;
color: #000;
font-style: italic;
font-weight: bold;
}
.markdown-body sub,
.markdown-body sup {
font-size: 75%;
line-height: 0;
position: relative;
vertical-align: baseline;
}
.markdown-body sup {
top: -0.5em;
}
.markdown-body sub {
bottom: -0.25em;
}
.markdown-body h1 {
font-size: 2em;
margin: 0.67em 0;
}
.markdown-body img {
border: 0;
}
.markdown-body hr {
-moz-box-sizing: content-box;
box-sizing: content-box;
height: 0;
}
.markdown-body pre {
overflow: auto;
}
.markdown-body code,
.markdown-body kbd,
.markdown-body pre,
.markdown-body samp {
font-family: monospace, monospace;
font-size: 1em;
}
.markdown-body input {
color: inherit;
font: inherit;
margin: 0;
}
.markdown-body html input[disabled] {
cursor: default;
}
.markdown-body input {
line-height: normal;
}
.markdown-body input[type="checkbox"] {
box-sizing: border-box;
padding: 0;
}
.markdown-body table {
border-collapse: collapse;
border-spacing: 0;
}
.markdown-body td,
.markdown-body th {
padding: 0;
}
.markdown-body .codehilitetable {
border: 0;
border-spacing: 0;
}
.markdown-body .codehilitetable tr {
border: 0;
}
.markdown-body .codehilitetable pre,
.markdown-body .codehilitetable div.codehilite {
margin: 0;
}
.markdown-body .linenos,
.markdown-body .code,
.markdown-body .codehilitetable td {
border: 0;
padding: 0;
}
.markdown-body td:not(.linenos) .linenodiv {
padding: 0 !important;
}
.markdown-body .code {
width: 100%;
}
.markdown-body .linenos div pre,
.markdown-body .linenodiv pre,
.markdown-body .linenodiv {
border: 0;
-webkit-border-radius: 0;
-moz-border-radius: 0;
border-radius: 0;
-webkit-border-top-left-radius: 3px;
-webkit-border-bottom-left-radius: 3px;
-moz-border-radius-topleft: 3px;
-moz-border-radius-bottomleft: 3px;
border-top-left-radius: 3px;
border-bottom-left-radius: 3px;
}
.markdown-body .code div pre,
.markdown-body .code div {
border: 0;
-webkit-border-radius: 0;
-moz-border-radius: 0;
border-radius: 0;
-webkit-border-top-right-radius: 3px;
-webkit-border-bottom-right-radius: 3px;
-moz-border-radius-topright: 3px;
-moz-border-radius-bottomright: 3px;
border-top-right-radius: 3px;
border-bottom-right-radius: 3px;
}
.markdown-body * {
-moz-box-sizing: border-box;
box-sizing: border-box;
}
.markdown-body input {
font: 13px Helvetica, arial, freesans, clean, sans-serif, "Segoe UI Emoji", "Segoe UI Symbol";
line-height: 1.4;
}
.markdown-body a {
color: #4183c4;
text-decoration: none;
}
.markdown-body a:hover,
.markdown-body a:focus,
.markdown-body a:active {
text-decoration: underline;
}
.markdown-body hr {
height: 0;
margin: 15px 0;
overflow: hidden;
background: transparent;
border: 0;
border-bottom: 1px solid #ddd;
}
.markdown-body hr:before,
.markdown-body hr:after {
display: table;
content: " ";
}
.markdown-body hr:after {
clear: both;
}
.markdown-body h1,
.markdown-body h2,
.markdown-body h3,
.markdown-body h4,
.markdown-body h5,
.markdown-body h6 {
margin-top: 15px;
margin-bottom: 15px;
line-height: 1.1;
}
.markdown-body h1 {
font-size: 30px;
}
.markdown-body h2 {
font-size: 21px;
}
.markdown-body h3 {
font-size: 16px;
}
.markdown-body h4 {
font-size: 14px;
}
.markdown-body h5 {
font-size: 12px;
}
.markdown-body h6 {
font-size: 11px;
}
.markdown-body blockquote {
margin: 0;
}
.markdown-body ul,
.markdown-body ol {
padding: 0;
margin-top: 0;
margin-bottom: 0;
}
.markdown-body ol ol,
.markdown-body ul ol {
list-style-type: lower-roman;
}
.markdown-body ul ul ol,
.markdown-body ul ol ol,
.markdown-body ol ul ol,
.markdown-body ol ol ol {
list-style-type: lower-alpha;
}
.markdown-body dd {
margin-left: 0;
}
.markdown-body code,
.markdown-body pre,
.markdown-body samp {
font-family: Consolas, "Liberation Mono", Menlo, Courier, monospace;
font-size: 12px;
}
.markdown-body pre {
margin-top: 0;
margin-bottom: 0;
}
.markdown-body kbd {
background-color: #e7e7e7;
background-image: -moz-linear-gradient(#fefefe, #e7e7e7);
background-image: -webkit-linear-gradient(#fefefe, #e7e7e7);
background-image: linear-gradient(#fefefe, #e7e7e7);
background-repeat: repeat-x;
border-radius: 2px;
border: 1px solid #cfcfcf;
color: #000;
padding: 3px 5px;
line-height: 10px;
font: 11px Consolas, "Liberation Mono", Menlo, Courier, monospace;
display: inline-block;
}
.markdown-body>*:first-child {
margin-top: 0 !important;
}
.markdown-body>*:last-child {
margin-bottom: 0 !important;
}
.markdown-body .headerlink {
font: normal 400 16px fontawesome-mini;
vertical-align: middle;
margin-left: -16px;
float: left;
display: inline-block;
text-decoration: none;
opacity: 0;
color: #333;
}
.markdown-body .headerlink:focus {
outline: none;
}
.markdown-body h1 .headerlink {
margin-top: 0.8rem;
}
.markdown-body h2 .headerlink,
.markdown-body h3 .headerlink {
margin-top: 0.6rem;
}
.markdown-body h4 .headerlink {
margin-top: 0.2rem;
}
.markdown-body h5 .headerlink,
.markdown-body h6 .headerlink {
margin-top: 0;
}
.markdown-body .headerlink:hover,
.markdown-body h1:hover .headerlink,
.markdown-body h2:hover .headerlink,
.markdown-body h3:hover .headerlink,
.markdown-body h4:hover .headerlink,
.markdown-body h5:hover .headerlink,
.markdown-body h6:hover .headerlink {
opacity: 1;
text-decoration: none;
}
.markdown-body h1 {
padding-bottom: 0.3em;
font-size: 2.25em;
line-height: 1.2;
border-bottom: 1px solid #eee;
}
.markdown-body h2 {
padding-bottom: 0.3em;
font-size: 1.75em;
line-height: 1.225;
border-bottom: 1px solid #eee;
}
.markdown-body h3 {
font-size: 1.5em;
line-height: 1.43;
}
.markdown-body h4 {
font-size: 1.25em;
}
.markdown-body h5 {
font-size: 1em;
}
.markdown-body h6 {
font-size: 1em;
color: #777;
}
.markdown-body p,
.markdown-body blockquote,
.markdown-body ul,
.markdown-body ol,
.markdown-body dl,
.markdown-body table,
.markdown-body pre,
.markdown-body .admonition {
margin-top: 0;
margin-bottom: 16px;
}
.markdown-body hr {
height: 4px;
padding: 0;
margin: 16px 0;
background-color: #e7e7e7;
border: 0 none;
}
.markdown-body ul,
.markdown-body ol {
padding-left: 2em;
}
.markdown-body ul ul,
.markdown-body ul ol,
.markdown-body ol ol,
.markdown-body ol ul {
margin-top: 0;
margin-bottom: 0;
}
.markdown-body li>p {
margin-top: 16px;
}
.markdown-body dl {
padding: 0;
}
.markdown-body dl dt {
padding: 0;
margin-top: 16px;
font-size: 1em;
font-style: italic;
font-weight: bold;
}
.markdown-body dl dd {
padding: 0 16px;
margin-bottom: 16px;
}
.markdown-body blockquote {
padding: 0 15px;
color: #777;
border-left: 4px solid #ddd;
}
.markdown-body blockquote>:first-child {
margin-top: 0;
}
.markdown-body blockquote>:last-child {
margin-bottom: 0;
}
.markdown-body table {
display: block;
width: 100%;
overflow: auto;
word-break: normal;
word-break: keep-all;
}
.markdown-body table th {
font-weight: bold;
}
.markdown-body table th,
.markdown-body table td {
padding: 6px 13px;
border: 1px solid #ddd;
}
.markdown-body table tr {
background-color: #fff;
border-top: 1px solid #ccc;
}
.markdown-body table tr:nth-child(2n) {
background-color: #f8f8f8;
}
.markdown-body img {
max-width: 100%;
-moz-box-sizing: border-box;
box-sizing: border-box;
}
.markdown-body code,
.markdown-body samp {
padding: 0;
padding-top: 0.2em;
padding-bottom: 0.2em;
margin: 0;
font-size: 85%;
background-color: rgba(0,0,0,0.04);
border-radius: 3px;
}
.markdown-body code:before,
.markdown-body code:after {
letter-spacing: -0.2em;
content: "\00a0";
}
.markdown-body pre>code {
padding: 0;
margin: 0;
font-size: 100%;
word-break: normal;
white-space: pre;
background: transparent;
border: 0;
}
.markdown-body .codehilite {
margin-bottom: 16px;
}
.markdown-body .codehilite pre,
.markdown-body pre {
padding: 16px;
overflow: auto;
font-size: 85%;
line-height: 1.45;
background-color: #f7f7f7;
border-radius: 3px;
}
.markdown-body .codehilite pre {
margin-bottom: 0;
word-break: normal;
}
.markdown-body pre {
word-wrap: normal;
}
.markdown-body pre code {
display: inline;
max-width: initial;
padding: 0;
margin: 0;
overflow: initial;
line-height: inherit;
word-wrap: normal;
background-color: transparent;
border: 0;
}
.markdown-body pre code:before,
.markdown-body pre code:after {
content: normal;
}
/* Admonition */
.markdown-body .admonition {
-webkit-border-radius: 3px;
-moz-border-radius: 3px;
position: relative;
border-radius: 3px;
border: 1px solid #e0e0e0;
border-left: 6px solid #333;
padding: 10px 10px 10px 30px;
}
.markdown-body .admonition table {
color: #333;
}
.markdown-body .admonition p {
padding: 0;
}
.markdown-body .admonition-title {
font-weight: bold;
margin: 0;
}
.markdown-body .admonition>.admonition-title {
color: #333;
}
.markdown-body .attention>.admonition-title {
color: #a6d796;
}
.markdown-body .caution>.admonition-title {
color: #d7a796;
}
.markdown-body .hint>.admonition-title {
color: #96c6d7;
}
.markdown-body .danger>.admonition-title {
color: #c25f77;
}
.markdown-body .question>.admonition-title {
color: #96a6d7;
}
.markdown-body .note>.admonition-title {
color: #d7c896;
}
.markdown-body .admonition:before,
.markdown-body .attention:before,
.markdown-body .caution:before,
.markdown-body .hint:before,
.markdown-body .danger:before,
.markdown-body .question:before,
.markdown-body .note:before {
font: normal normal 16px fontawesome-mini;
-moz-osx-font-smoothing: grayscale;
-webkit-user-select: none;
-moz-user-select: none;
-ms-user-select: none;
user-select: none;
line-height: 1.5;
color: #333;
position: absolute;
left: 0;
top: 0;
padding-top: 10px;
padding-left: 10px;
}
.markdown-body .admonition:before {
content: "\f056\00a0";
color: 333;
}
.markdown-body .attention:before {
content: "\f058\00a0";
color: #a6d796;
}
.markdown-body .caution:before {
content: "\f06a\00a0";
color: #d7a796;
}
.markdown-body .hint:before {
content: "\f05a\00a0";
color: #96c6d7;
}
.markdown-body .danger:before {
content: "\f057\00a0";
color: #c25f77;
}
.markdown-body .question:before {
content: "\f059\00a0";
color: #96a6d7;
}
.markdown-body .note:before {
content: "\f040\00a0";
color: #d7c896;
}
.markdown-body .admonition::after {
content: normal;
}
.markdown-body .attention {
border-left: 6px solid #a6d796;
}
.markdown-body .caution {
border-left: 6px solid #d7a796;
}
.markdown-body .hint {
border-left: 6px solid #96c6d7;
}
.markdown-body .danger {
border-left: 6px solid #c25f77;
}
.markdown-body .question {
border-left: 6px solid #96a6d7;
}
.markdown-body .note {
border-left: 6px solid #d7c896;
}
.markdown-body .admonition>*:first-child {
margin-top: 0 !important;
}
.markdown-body .admonition>*:last-child {
margin-bottom: 0 !important;
}
/* progress bar*/
.markdown-body .progress {
display: block;
width: 300px;
margin: 10px 0;
height: 24px;
-webkit-border-radius: 3px;
-moz-border-radius: 3px;
border-radius: 3px;
background-color: #ededed;
position: relative;
box-shadow: inset -1px 1px 3px rgba(0, 0, 0, .1);
}
.markdown-body .progress-label {
position: absolute;
text-align: center;
font-weight: bold;
width: 100%; margin: 0;
line-height: 24px;
color: #333;
text-shadow: 1px 1px 0 #fefefe, -1px -1px 0 #fefefe, -1px 1px 0 #fefefe, 1px -1px 0 #fefefe, 0 1px 0 #fefefe, 0 -1px 0 #fefefe, 1px 0 0 #fefefe, -1px 0 0 #fefefe, 1px 1px 2px #000;
-webkit-font-smoothing: antialiased !important;
white-space: nowrap;
overflow: hidden;
}
.markdown-body .progress-bar {
height: 24px;
float: left;
-webkit-border-radius: 3px;
-moz-border-radius: 3px;
border-radius: 3px;
background-color: #96c6d7;
box-shadow: inset 0 1px 0 rgba(255, 255, 255, .5), inset 0 -1px 0 rgba(0, 0, 0, .1);
background-size: 30px 30px;
background-image: -webkit-linear-gradient(
135deg, rgba(255, 255, 255, .4) 27%,
transparent 27%,
transparent 52%, rgba(255, 255, 255, .4) 52%,
rgba(255, 255, 255, .4) 77%,
transparent 77%, transparent
);
background-image: -moz-linear-gradient(
135deg,
rgba(255, 255, 255, .4) 27%, transparent 27%,
transparent 52%, rgba(255, 255, 255, .4) 52%,
rgba(255, 255, 255, .4) 77%, transparent 77%,
transparent
);
background-image: -ms-linear-gradient(
135deg,
rgba(255, 255, 255, .4) 27%, transparent 27%,
transparent 52%, rgba(255, 255, 255, .4) 52%,
rgba(255, 255, 255, .4) 77%, transparent 77%,
transparent
);
background-image: -o-linear-gradient(
135deg,
rgba(255, 255, 255, .4) 27%, transparent 27%,
transparent 52%, rgba(255, 255, 255, .4) 52%,
rgba(255, 255, 255, .4) 77%, transparent 77%,
transparent
);
background-image: linear-gradient(
135deg,
rgba(255, 255, 255, .4) 27%, transparent 27%,
transparent 52%, rgba(255, 255, 255, .4) 52%,
rgba(255, 255, 255, .4) 77%, transparent 77%,
transparent
);
}
.markdown-body .progress-100plus .progress-bar {
background-color: #a6d796;
}
.markdown-body .progress-80plus .progress-bar {
background-color: #c6d796;
}
.markdown-body .progress-60plus .progress-bar {
background-color: #d7c896;
}
.markdown-body .progress-40plus .progress-bar {
background-color: #d7a796;
}
.markdown-body .progress-20plus .progress-bar {
background-color: #d796a6;
}
.markdown-body .progress-0plus .progress-bar {
background-color: #c25f77;
}
.markdown-body .candystripe-animate .progress-bar{
-webkit-animation: animate-stripes 3s linear infinite;
-moz-animation: animate-stripes 3s linear infinite;
animation: animate-stripes 3s linear infinite;
}
@-webkit-keyframes animate-stripes {
0% {
background-position: 0 0;
}
100% {
background-position: 60px 0;
}
}
@-moz-keyframes animate-stripes {
0% {
background-position: 0 0;
}
100% {
background-position: 60px 0;
}
}
@keyframes animate-stripes {
0% {
background-position: 0 0;
}
100% {
background-position: 60px 0;
}
}
.markdown-body .gloss .progress-bar {
box-shadow:
inset 0 4px 12px rgba(255, 255, 255, .7),
inset 0 -12px 0 rgba(0, 0, 0, .05);
}
/* MultiMarkdown Critic Blocks */
.markdown-body .critic_mark {
background: #ff0;
}
.markdown-body .critic_delete {
color: #c82829;
text-decoration: line-through;
}
.markdown-body .critic_insert {
color: #718c00 ;
text-decoration: underline;
}
.markdown-body .critic_comment {
color: #8e908c;
font-style: italic;
}
.markdown-body .headeranchor {
font: normal normal 16px fontawesome-mini;
line-height: 1;
display: inline-block;
text-decoration: none;
-webkit-font-smoothing: antialiased;
-moz-osx-font-smoothing: grayscale;
-webkit-user-select: none;
-moz-user-select: none;
-ms-user-select: none;
user-select: none;
}
.headeranchor:before {
content: '\e157';
}
.markdown-body .task-list-item {
list-style-type: none;
}
.markdown-body .task-list-item+.task-list-item {
margin-top: 3px;
}
.markdown-body .task-list-item input {
margin: 0 4px 0.25em -20px;
vertical-align: middle;
}
/* Media */
@media only screen and (min-width: 480px) {
.markdown-body {
font-size:14px;
}
}
@media only screen and (min-width: 768px) {
.markdown-body {
font-size:16px;
}
}
@media print {
.markdown-body * {
background: transparent !important;
color: black !important;
filter:none !important;
-ms-filter: none !important;
}
.markdown-body {
font-size:12pt;
max-width:100%;
outline:none;
border: 0;
}
.markdown-body a,
.markdown-body a:visited {
text-decoration: underline;
}
.markdown-body .headeranchor-link {
display: none;
}
.markdown-body a[href]:after {
content: " (" attr(href) ")";
}
.markdown-body abbr[title]:after {
content: " (" attr(title) ")";
}
.markdown-body .ir a:after,
.markdown-body a[href^="javascript:"]:after,
.markdown-body a[href^="#"]:after {
content: "";
}
.markdown-body pre {
white-space: pre;
white-space: pre-wrap;
word-wrap: break-word;
}
.markdown-body pre,
.markdown-body blockquote {
border: 1px solid #999;
padding-right: 1em;
page-break-inside: avoid;
}
.markdown-body .progress,
.markdown-body .progress-bar {
-moz-box-shadow: none;
-webkit-box-shadow: none;
box-shadow: none;
}
.markdown-body .progress {
border: 1px solid #ddd;
}
.markdown-body .progress-bar {
height: 22px;
border-right: 1px solid #ddd;
}
.markdown-body tr,
.markdown-body img {
page-break-inside: avoid;
}
.markdown-body img {
max-width: 100% !important;
}
.markdown-body p,
.markdown-body h2,
.markdown-body h3 {
orphans: 3;
widows: 3;
}
.markdown-body h2,
.markdown-body h3 {
page-break-after: avoid;
}
}
</style><style>/*GitHub*/
.codehilite {background-color:#fff;color:#333333;}
.codehilite .hll {background-color:#ffffcc;}
.codehilite .c{color:#999988;font-style:italic}
.codehilite .err{color:#a61717;background-color:#e3d2d2}
.codehilite .k{font-weight:bold}
.codehilite .o{font-weight:bold}
.codehilite .cm{color:#999988;font-style:italic}
.codehilite .cp{color:#999999;font-weight:bold}
.codehilite .c1{color:#999988;font-style:italic}
.codehilite .cs{color:#999999;font-weight:bold;font-style:italic}
.codehilite .gd{color:#000000;background-color:#ffdddd}
.codehilite .ge{font-style:italic}
.codehilite .gr{color:#aa0000}
.codehilite .gh{color:#999999}
.codehilite .gi{color:#000000;background-color:#ddffdd}
.codehilite .go{color:#888888}
.codehilite .gp{color:#555555}
.codehilite .gs{font-weight:bold}
.codehilite .gu{color:#800080;font-weight:bold}
.codehilite .gt{color:#aa0000}
.codehilite .kc{font-weight:bold}
.codehilite .kd{font-weight:bold}
.codehilite .kn{font-weight:bold}
.codehilite .kp{font-weight:bold}
.codehilite .kr{font-weight:bold}
.codehilite .kt{color:#445588;font-weight:bold}
.codehilite .m{color:#009999}
.codehilite .s{color:#dd1144}
.codehilite .n{color:#333333}
.codehilite .na{color:teal}
.codehilite .nb{color:#0086b3}
.codehilite .nc{color:#445588;font-weight:bold}
.codehilite .no{color:teal}
.codehilite .ni{color:purple}
.codehilite .ne{color:#990000;font-weight:bold}
.codehilite .nf{color:#990000;font-weight:bold}
.codehilite .nn{color:#555555}
.codehilite .nt{color:navy}
.codehilite .nv{color:teal}
.codehilite .ow{font-weight:bold}
.codehilite .w{color:#bbbbbb}
.codehilite .mf{color:#009999}
.codehilite .mh{color:#009999}
.codehilite .mi{color:#009999}
.codehilite .mo{color:#009999}
.codehilite .sb{color:#dd1144}
.codehilite .sc{color:#dd1144}
.codehilite .sd{color:#dd1144}
.codehilite .s2{color:#dd1144}
.codehilite .se{color:#dd1144}
.codehilite .sh{color:#dd1144}
.codehilite .si{color:#dd1144}
.codehilite .sx{color:#dd1144}
.codehilite .sr{color:#009926}
.codehilite .s1{color:#dd1144}
.codehilite .ss{color:#990073}
.codehilite .bp{color:#999999}
.codehilite .vc{color:teal}
.codehilite .vg{color:teal}
.codehilite .vi{color:teal}
.codehilite .il{color:#009999}
.codehilite .gc{color:#999;background-color:#EAF2F5}
</style><title>README</title></head><body><article class="markdown-body"><p><img src="https://user-images.githubusercontent.com/10807627/36499489-47bf1b6e-16f6-11e8-9a22-6e356c603a37.png" height="64"></p>
<h1 id="flotilla-os">flotilla-os<a class="headerlink" href="#flotilla-os" title="Permanent link"></a></h1>
<p><a href="https://circleci.com/gh/stitchfix/flotilla-os"><img alt="Circle CI" src="https://circleci.com/gh/stitchfix/flotilla-os.svg?style=shield" /></a>
<a href="https://goreportcard.com/report/github.com/stitchfix/flotilla-os"><img alt="Go Report Card" src="https://goreportcard.com/badge/github.com/stitchfix/flotilla-os" /></a></p>
<h2 id="introduction">Introduction<a class="headerlink" href="#introduction" title="Permanent link"></a></h2>
<p>Flotilla is a self-service framework that dramatically simplifies the process of defining and executing containerized jobs. This means you get to focus on the work you’re doing rather than <em>how</em> to do it.</p>
<p>Once deployed, Flotilla allows you to:</p>
<ul>
<li>Define containerized jobs by allowing you to specify exactly what command to run, what image to run that command in, and what resources that command needs to run</li>
<li>Run any previously defined job and access its logs, status, and exit code</li>
<li>View and edit job definitions with a flexible UI</li>
<li>Run jobs and view execution history and logs within the UI</li>
<li>Use the complete REST API for definitions, jobs, and logs to build your own custom workflows</li>
</ul>
<h2 id="philosophy">Philosophy<a class="headerlink" href="#philosophy" title="Permanent link"></a></h2>
<p>Flotilla is strongly opinionated about self-service for data science.</p>
<p>The core assumption is that you understand your work the best. Therefore, it is <em>you</em> who should own your work from end-to-end. In other words, you shouldn’t need to be a “production engineer” to run your jobs or to access logs in case of problems. Do this with Flotilla.</p>
<h2 id="quick-start">Quick Start<a class="headerlink" href="#quick-start" title="Permanent link"></a></h2>
<h3 id="minimal-assumptions">Minimal Assumptions<a class="headerlink" href="#minimal-assumptions" title="Permanent link"></a></h3>
<p>Before we can do <em>anything</em> there’s some <em>prerequistes</em> that must be met.</p>
<ol>
<li>Flotilla by default uses AWS. You must have an AWS account and AWS keys available. This quick-start guide uses AWS keys exported into the environment variables: <code>AWS_ACCESS_KEY_ID</code> and <code>AWS_SECRET_ACCESS_KEY</code>. If you’ve got credentials configured on your machine you can set these easily by running:</li>
</ol>
<div class="codehilite"><pre>export AWS_ACCESS_KEY_ID=$(aws --profile default configure get aws_access_key_id)
export AWS_SECRET_ACCESS_KEY=$(aws --profile default configure get aws_secret_access_key)
</pre></div>
<blockquote>
<p>Note: When running on AWS EC2 instances or ECS it’s better practice to use an IAM profile for AWS credentials</p>
</blockquote>
<ol start="2">
<li>The AWS credentials must be authorized. The permissions required are described in the following policy document for AWS (you can attach it to a user or a role depending on how you manage users in AWS).</li>
</ol>
<div class="codehilite"><pre>{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "flotilla-policy",
"Effect": "Allow",
"Action": [
"sqs:DeleteMessage",
"sqs:ListQueues",
"sqs:GetQueueUrl",
"logs:DescribeLogGroups",
"sqs:ReceiveMessage",
"events:PutRule",
"sqs:SendMessage",
"sqs:GetQueueAttributes",
"ecs:DescribeClusters",
"ecs:DeregisterTaskDefinition",
"events:ListRuleNamesByTarget",
"ecs:RunTask",
"ecs:RegisterTaskDefinition",
"sqs:CreateQueue",
"ecs:ListContainerInstances",
"ecs:DescribeContainerInstances",
"ecs:ListClusters",
"ecs:StopTask",
"logs:CreateLogGroup",
"logs:PutRetentionPolicy",
"logs:GetLogEvents",
"events:PutTargets",
"sqs:SetQueueAttributes"
],
"Resource": "*"
}
]
}
</pre></div>
<ol start="3">
<li>Flotilla uses AWS’s Elastic Container Service (ECS) and Elastic Kubernetes Service (EKS) as the execution backend. However, Flotilla does not manage ECS/EKS clusters. There must be at least one cluster defined in AWS’s ECS/EKS service available to you and it must have at least one task node. Most typically this is the <code>default</code> cluster and examples will assume this going forward. You can easily set up a cluster by following the instructions here: <a href="https://docs.aws.amazon.com/AmazonECS/latest/developerguide/launch_container_instance.html">https://docs.aws.amazon.com/AmazonECS/latest/developerguide/launch_container_instance.html</a></li>
</ol>
<p><a href="https://docs.aws.amazon.com/eks/latest/userguide/what-is-eks.html">https://docs.aws.amazon.com/eks/latest/userguide/what-is-eks.html</a></p>
<h3 id="starting-the-service-locally">Starting the service locally<a class="headerlink" href="#starting-the-service-locally" title="Permanent link"></a></h3>
<p>You can run the service locally (which will still leverage AWS resources) using the <a href="https://docs.docker.com/compose/">docker-compose</a> tool. From inside the repo run:</p>
<div class="codehilite"><pre>docker-compose up -d
</pre></div>
<p>You’ll notice it builds the code in the repo and starts the flotilla service as well as the default postgres backend.</p>
<p>Verify the service is running by making a <code>GET</code> request with cURL (or navigating to in a web browser) the url <code>http://localhost:5000/api/v6/task</code>. A 200OK response means things are good!</p>
<blockquote>
<p>Note: The default configuration under <code>conf</code> and in the <code>docker-compose.yml</code> assume port 3000. You’ll have to change it in both places if you don’t want to use port 3000 locally.</p>
</blockquote>
<h3 id="using-the-ui">Using the UI<a class="headerlink" href="#using-the-ui" title="Permanent link"></a></h3>
<p>Flotilla has a simple, easy to use UI. Here’s some example images for basic usage.</p>
<h4 id="define-a-task-with-the-ui">Define a task with the UI<a class="headerlink" href="#define-a-task-with-the-ui" title="Permanent link"></a></h4>
<p>The UI allows you to quickly create new tasks.</p>
<p><img alt="Define Task" src="https://user-images.githubusercontent.com/10807627/36499487-47a0b82c-16f6-11e8-886b-ca6d38276889.png" title="Create New Task" /></p>
<h4 id="launch-a-task-with-ui">Launch a task with UI<a class="headerlink" href="#launch-a-task-with-ui" title="Permanent link"></a></h4>
<p>You can run tasks you’ve created with the UI as well. Once you’ve ran a task the run will transition from <code>Queued</code> to <code>Pending</code> to <code>Running</code> before it finishes and shows <code>Success</code> or <code>Failed</code> (see <a href="#definitions-and-task-life-cycle">Task Life Cycle</a>). Once a task is in the <code>Running</code> state the logs should be visible.</p>
<ol>
<li>Launch</li>
</ol>
<p><img alt="Run Task" src="https://user-images.githubusercontent.com/10807627/36499492-481da436-16f6-11e8-9f14-5bbe8c297434.png" title="Run Task" /></p>
<ol start="2">
<li>Queued –> Pending</li>
</ol>
<p><img alt="Queued Task" src="https://user-images.githubusercontent.com/10807627/36499491-4801515a-16f6-11e8-9525-db85bb999887.png" title="Queued Task" /></p>
<p><img alt="Pending Task" src="https://user-images.githubusercontent.com/10807627/36499490-47e27e88-16f6-11e8-8041-355de885be44.png" title="Pending Task" />
3. View logs</p>
<p><img alt="Running Task" src="https://user-images.githubusercontent.com/10807627/36499493-4842176c-16f6-11e8-9467-a345987bd407.png" title="Running Task" /></p>
<p><img alt="Finished Task" src="https://user-images.githubusercontent.com/10807627/36499494-48609cfa-16f6-11e8-8656-5504063cb6e7.png" title="Finished Task" /></p>
<h3 id="basic-api-usage">Basic API Usage<a class="headerlink" href="#basic-api-usage" title="Permanent link"></a></h3>
<h4 id="defining-your-first-task">Defining your first task<a class="headerlink" href="#defining-your-first-task" title="Permanent link"></a></h4>
<p>Before you can run a task you first need to define it. We’ll use the example hello world task definition. Here’s what that looks like:</p>
<blockquote>
<p>hello-world.json</p>
<div class="codehilite"><pre>{
"alias": "hello-flotilla",
"group_name": "examples",
"image": "ubuntu:latest",
"memory": 512,
"env": [
{
"name": "USERNAME",
"value": "_fill_me_in_"
}
],
"command": "echo \"hello ${USERNAME}\""
}
</pre></div>
</blockquote>
<p>It’s a simple task that runs in the default ubuntu image, prints your username to the logs, and exits.</p>
<blockquote>
<p>Note: While you can use non-public images and images in your own registries with flotilla, credentials for accessing those images must exist on the ECS hosts. This is outside the scope of this doc. See the AWS <a href="https://docs.aws.amazon.com/AmazonECS/latest/developerguide/private-auth.html">documentation</a>.</p>
</blockquote>
<p>Let’s define it:</p>
<div class="codehilite"><pre>curl -XPOST localhost:5000/api/v6/task --data @examples/hello-world.json
</pre></div>
<p>You’ll notice that if you visit the initial url again <code>http://localhost:5000/api/v6/task</code> the newly defined definition will be in the list.</p>
<h4 id="running-your-first-task">Running your first task<a class="headerlink" href="#running-your-first-task" title="Permanent link"></a></h4>
<p>This is the fun part. You’ll make a <code>PUT</code> request to the execution endpoint for the task you just defined and specify any environment variables.</p>
<div class="codehilite"><pre>curl -XPUT localhost:5000/api/v6/task/alias/hello-flotilla/execute -d '{
"cluster":"default",
"env":[
{"name":"USERNAME","value":"yourusername"}
],
"run_tags":{"owner_id":"youruser"}
}'
</pre></div>
<blockquote>
<p>Note: <code>run_tags</code> is defined as a way for all runs to have a ownership injected for visibility and is <em>required</em>.</p>
</blockquote>
<p>You’ll get a response that contains a <code>run_id</code> field. You can check the status of your task at <code>http://localhost:5000/api/v6/history/<run_id></code></p>
<div class="codehilite"><pre>curl -XGET localhost:5000/api/v6/history/<run_id>
{
"instance": {
"dns_name": "<dns-host-of-task-node>",
"instance_id": "<instance-id-of-task-node>"
},
"run_id": "<run_id>",
"definition_id": "<definition_id>",
"alias": "hello-flotilla",
"image": "ubuntu:latest",
"cluster": "default",
"status": "PENDING",
"env": [
{
"name": "FLOTILLA_RUN_OWNER_ID",
"value": "youruser"
},
{
"name": "FLOTILLA_SERVER_MODE",
"value": "dev"
},
{
"name": "FLOTILLA_RUN_ID",
"value": "<run_id>"
},
{
"name": "USERNAME",
"value": "yourusername"
}
]
}
</pre></div>
<p>and you can get the logs for your task at <code>http://localhost:5000/api/v6/<run_id>/logs</code>. You will not see any logs until your task is at least in the <code>RUNNING</code> state.</p>
<div class="codehilite"><pre>curl -XGET localhost:5000/api/v6/<run_id>/logs
{
"last_seen":"<last_seen_token_used_for_paging>",
"log":"+ set -e\n+ echo 'hello yourusername'\nhello yourusername"
}
</pre></div>
<h2 id="definitions-and-task-life-cycle">Definitions and Task Life Cycle<a class="headerlink" href="#definitions-and-task-life-cycle" title="Permanent link"></a></h2>
<h3 id="definitions">Definitions<a class="headerlink" href="#definitions" title="Permanent link"></a></h3>
<table>
<thead>
<tr>
<th>Name</th>
<th>Definition</th>
</tr>
</thead>
<tbody>
<tr>
<td><code>task</code></td>
<td>A definition of a task that can be executed to create a <code>run</code></td>
</tr>
<tr>
<td><code>run</code></td>
<td>An instance of a task</td>
</tr>
</tbody>
</table>
<h3 id="task-life-cycle">Task Life Cycle<a class="headerlink" href="#task-life-cycle" title="Permanent link"></a></h3>
<p>When executed, a task’s run goes through several transitions</p>
<ol>
<li><code>QUEUED</code> - this is the first phase of a run and means the run is currently queued and waiting to be allocated to a cluster</li>
<li><code>PENDING</code> - every <code>worker.submit_interval</code> (defined in the config) the submit worker pulls from the queues and submits them for execution. At this point, if the cluster associated with the run has resources, the run gets allocated to the cluster and transitions to the <code>PENDING</code> status. For the default execution engine this stage encapsulates the process of pulling the docker image and starting the container. It can take several minutes depending on whether the image is cached and how large the image is.</li>
<li><code>RUNNING</code> - Once the run starts on a particular execution host it transitions to this stage. At this point logs should become available.</li>
<li><code>STOPPED</code> - A run enters this stage when it finishes execution. This can mean it either succeeded or failed depending on the existence of an <code>exit_code</code> and the value of that exit code.</li>
<li><code>NEEDS_RETRY</code> - on occassion, due to host level characteristics (full disk, too many open files, timeouts pulling image, etc) the run exits with a null exit code without ever being executed. In this case the reason is analyzed to determine if the run is retriable. If it is, the task transitions to this status and is allocated to the appropriate execution queue again, and will repeat the lifecycle.</li>
</ol>
<h4 id="normal-lifecycle">Normal Lifecycle<a class="headerlink" href="#normal-lifecycle" title="Permanent link"></a></h4>
<p><code>QUEUED</code> –> <code>PENDING</code> –> <code>RUNNING</code> –> <code>STOPPED</code></p>
<h4 id="retry-lifecycle">Retry Lifecycle<a class="headerlink" href="#retry-lifecycle" title="Permanent link"></a></h4>
<p>… –> <code>PENDING</code> –> <code>STOPPED</code> –> <code>NEEDS_RETRY</code> –> <code>QUEUED</code> –> …</p>
<h2 id="deploying">Deploying<a class="headerlink" href="#deploying" title="Permanent link"></a></h2>
<p>In a production deployment you’ll want multiple instances of the flotilla service running and postgres running elsewhere (eg. Amazon RDS). In this case the most salient detail configuration detail is the <code>DATABASE_URL</code>.</p>
<h3 id="docker-based-deploy">Docker based deploy<a class="headerlink" href="#docker-based-deploy" title="Permanent link"></a></h3>
<p>The simplest way to deploy for very light usage is to avoid a reverse proxy and deploy directly with docker.</p>
<ol>
<li>
<p>Build and tag an image for flotilla using the <code>Dockerfile</code> provided in this repo:</p>
<p><div class="codehilite"><pre>docker build -t <your repo name>/flotilla:<version tag>
</pre></div>
2. Run this image wherever you deploy your services:</p>
<div class="codehilite"><pre>docker run -e DATABASE_URL=<your db url> -e FLOTILLA_MODE=prod -p 3000:3000 ...<other standard docker run args>
</pre></div>
<blockquote>
<h2 id="notes">Notes:<a class="headerlink" href="#notes" title="Permanent link"></a></h2>
<ul>
<li>Flotilla uses <a href="https://github.com/spf13/viper">viper</a> for configuration so you can override any of the default configuration under <code>conf/</code> using run time environment variables passed to <code>docker run</code></li>
<li>In most realistic deploys you’ll likely want to configure a reverse proxy to sit in front of the flotilla container. See the docs <a href="https://hub.docker.com/_/nginx/">here</a></li>
</ul>
</blockquote>
<p>See <a href="https://docs.docker.com/engine/reference/run/">docker run</a> for more details</p>
</li>
</ol>
<h3 id="configuration-in-detail">Configuration In Detail<a class="headerlink" href="#configuration-in-detail" title="Permanent link"></a></h3>
<p>The variables in <code>conf/config.yml</code> are sensible defaults. Most should be left alone unless you’re developing flotilla itself. However, there are a few you may want to change in a production environment.</p>
<table>
<thead>
<tr>
<th>Variable Name</th>
<th>Description</th>
</tr>
</thead>
<tbody>
<tr>
<td><code>worker.retry_interval</code></td>
<td>Run frequency of the retry worker</td>
</tr>
<tr>
<td><code>worker.submit_interval</code></td>
<td>Poll frequency of the submit worker</td>
</tr>
<tr>
<td><code>worker.status_interval</code></td>
<td>Poll frequency of the status update worker</td>
</tr>
<tr>
<td><code>http.server.read_timeout_seconds</code></td>
<td>Sets read timeout in seconds for the http server</td>
</tr>
<tr>
<td><code>http.server.write_timeout_seconds</code></td>
<td>Sets the write timeout in seconds for the http server</td>
</tr>
<tr>
<td><code>http.server.listen_address</code></td>
<td>The port for the http server to listen on</td>
</tr>
<tr>
<td><code>owner_id_var</code></td>
<td>Which environment variable containing ownership information to inject into the runtime of jobs</td>
</tr>
<tr>
<td><code>enabled_workers</code></td>
<td>This variable is a list of the workers that run. Use this to control what workers run when using a multi-container deployment strategy. Valid list items include (<code>retry</code>, <code>submit</code>, and <code>status</code>)</td>
</tr>
<tr>
<td><code>log.namespace</code></td>
<td>For the default ECS execution engine setup this is the <code>log-group</code> to use</td>
</tr>
<tr>
<td><code>log.retention_days</code></td>
<td>For the default ECS execution engine this is the number of days to retain logs</td>
</tr>
<tr>
<td><code>log.driver.options.*</code></td>
<td>For the default ECS execution engine these map to the <code>awslogs</code> driver options <a href="https://docs.aws.amazon.com/AmazonECS/latest/developerguide/using_awslogs.html">here</a></td>
</tr>
<tr>
<td><code>queue.namespace</code></td>
<td>For the default ECS execution engine this is the prefix used for SQS to determine which queues to pull job launch messages from</td>
</tr>
<tr>
<td><code>queue.retention_seconds</code></td>
<td>For the default ECS execution engine this configures how long a message will stay in an SQS queue without being consumed</td>
</tr>
<tr>
<td><code>queue.process_time</code></td>
<td>For the default ECS execution engine configures the length of time allowed to process a job launch message</td>
</tr>
<tr>
<td><code>queue.status</code></td>
<td>For the default ECS execution engine this configures which SQS queue to route ECS cluster status updates to</td>
</tr>
<tr>
<td><code>queue.status_rule</code></td>
<td>For the default ECS execution engine this configures the name of the rule for routing ECS cluster status updates</td>
</tr>
<tr>
<td><code>metrics.dogstatsd.address</code></td>
<td>Statds metrics host in Datadog format</td>
</tr>
<tr>
<td><code>metrics.dogstatsd.namespace</code></td>
<td>Namespace for the metrics - for example <code>flotilla.</code></td>
</tr>
<tr>
<td><code>redis_address</code></td>
<td>Redis host for caching and locks</td>
</tr>
<tr>
<td><code>redis_db</code></td>
<td>Redis db to be used - numeric</td>
</tr>
</tbody>
</table>
<h2 id="development">Development<a class="headerlink" href="#development" title="Permanent link"></a></h2>
<h3 id="api-documentation">API Documentation<a class="headerlink" href="#api-documentation" title="Permanent link"></a></h3>
<p>See <a href="https://stitchfix.github.io/flotilla-os/api.html">API</a></p>
<h3 id="building">Building<a class="headerlink" href="#building" title="Permanent link"></a></h3>
<p>Currently Flotilla is built using <code>go</code> 1.9.3 and uses the <a href="https://github.com/kardianos/govendor"><code>govendor</code></a> to manage dependencies.</p>
<div class="codehilite"><pre>govendor sync && go build
</pre></div></article></body></html>
================================================
FILE: README.md
================================================
<img src="https://user-images.githubusercontent.com/10807627/36499489-47bf1b6e-16f6-11e8-9a22-6e356c603a37.png" height="64">
# flotilla-os
[](https://circleci.com/gh/stitchfix/flotilla-os)
[](https://goreportcard.com/report/github.com/stitchfix/flotilla-os)
## Introduction
Flotilla is a self-service framework that dramatically simplifies the process of defining and executing containerized jobs. This means you get to focus on the work you're doing rather than _how_ to do it.
Once deployed, Flotilla allows you to:
* Define containerized jobs by allowing you to specify exactly what command to run, what image to run that command in, and what resources that command needs to run
* Run any previously defined job and access its logs, status, and exit code
* View and edit job definitions with a flexible UI
* Run jobs and view execution history and logs within the UI
* Use the complete REST API for definitions, jobs, and logs to build your own custom workflows
## Philosophy
Flotilla is strongly opinionated about self-service for data science.
The core assumption is that you understand your work the best. Therefore, it is _you_ who should own your work from end-to-end. In other words, you shouldn't need to be a "production engineer" to run your jobs or to access logs in case of problems. Do this with Flotilla.
## Quick Start
### Minimal Assumptions
Flotilla uses AWS's Elastic Kubernetes Service (EKS) as the execution backend. However, Flotilla does not manage EKS clusters. There must be at least one cluster defined in AWS's EKS service available to you and it must have at least one task node. Most typically this is the `default` cluster and examples will assume this going forward.
https://docs.aws.amazon.com/eks/latest/userguide/what-is-eks.html
### Starting the service locally
You can run the service locally (which will still leverage AWS resources) using the [docker-compose](https://docs.docker.com/compose/) tool. From inside the repo run:
```
docker-compose up -d
```
You'll notice it builds the code in the repo and starts the flotilla service as well as the default postgres backend.
Verify the service is running by making a `GET` request with cURL (or navigating to in a web browser) the url `http://localhost:5000/api/v6/task`. A 200OK response means things are good!
> Note: The default configuration under `conf` and in the `docker-compose.yml` assume port 3000. You'll have to change it in both places if you don't want to use port 3000 locally.
### Using the UI
Flotilla has a simple, easy to use UI. Here's some example images for basic usage.
#### Define a task with the UI
The UI allows you to quickly create new tasks.
#### Launch a task with UI
You can run tasks you've created with the UI as well. Once you've ran a task the run will transition from `Queued` to `Pending` to `Running` before it finishes and shows `Success` or `Failed` (see [Task Life Cycle](#definitions-and-task-life-cycle)). Once a task is in the `Running` state the logs should be visible.
1. Launch
2. Queued --> Pending
3. View logs
### Basic API Usage
#### Defining your first task
Before you can run a task you first need to define it. We'll use the example hello world task definition. Here's what that looks like:
> hello-world.json
>
```
{
"alias": "hello-flotilla",
"group_name": "examples",
"image": "ubuntu:latest",
"memory": 512,
"env": [
{
"name": "USERNAME",
"value": "_fill_me_in_"
}
],
"command": "echo \"hello ${USERNAME}\""
}
```
It's a simple task that runs in the default ubuntu image, prints your username to the logs, and exits.
> Note: While you can use non-public images and images in your own registries with flotilla, credentials for accessing those images must exist on the EKS hosts. This is outside the scope of this doc.
Let's define it:
```
curl -XPOST localhost:5000/api/v6/task --data @examples/hello-world.json
```
You'll notice that if you visit the initial url again `http://localhost:5000/api/v6/task` the newly defined definition will be in the list.
#### Running your first task
This is the fun part. You'll make a `PUT` request to the execution endpoint for the task you just defined and specify any environment variables.
```
curl -XPUT localhost:5000/api/v6/task/alias/hello-flotilla/execute -d '{
"cluster":"default",
"env":[
{"name":"USERNAME","value":"yourusername"}
],
"run_tags":{"owner_id":"youruser"}
}'
```
> Note: `run_tags` is defined as a way for all runs to have a ownership injected for visibility and is *required*.
You'll get a response that contains a `run_id` field. You can check the status of your task at `http://localhost:5000/api/v6/history/<run_id>`
```
curl -XGET localhost:5000/api/v6/history/<run_id>
{
"instance": {
"dns_name": "<dns-host-of-task-node>",
"instance_id": "<instance-id-of-task-node>"
},
"run_id": "<run_id>",
"definition_id": "<definition_id>",
"alias": "hello-flotilla",
"image": "ubuntu:latest",
"cluster": "default",
"status": "PENDING",
"env": [
{
"name": "FLOTILLA_RUN_OWNER_ID",
"value": "youruser"
},
{
"name": "FLOTILLA_SERVER_MODE",
"value": "dev"
},
{
"name": "FLOTILLA_RUN_ID",
"value": "<run_id>"
},
{
"name": "USERNAME",
"value": "yourusername"
}
]
}
```
and you can get the logs for your task at `http://localhost:5000/api/v6/<run_id>/logs`. You will not see any logs until your task is at least in the `RUNNING` state.
```
curl -XGET localhost:5000/api/v6/<run_id>/logs
{
"last_seen":"<last_seen_token_used_for_paging>",
"log":"+ set -e\n+ echo 'hello yourusername'\nhello yourusername"
}
```
## Definitions and Task Life Cycle
### Definitions
| Name | Definition |
| ---- | ---------- |
| `task` | A definition of a task that can be executed to create a `run` |
| `run` | An instance of a task |
### Task Life Cycle
When executed, a task's run goes through several transitions
1. `QUEUED` - this is the first phase of a run and means the run is currently queued and waiting to be allocated to a cluster
2. `PENDING` - every `worker.submit_interval` (defined in the config) the submit worker pulls from the queues and submits them for execution. At this point, if the cluster associated with the run has resources, the run gets allocated to the cluster and transitions to the `PENDING` status. For the default execution engine this stage encapsulates the process of pulling the docker image and starting the container. It can take several minutes depending on whether the image is cached and how large the image is.
3. `RUNNING` - Once the run starts on a particular execution host it transitions to this stage. At this point logs should become available.
4. `STOPPED` - A run enters this stage when it finishes execution. This can mean it either succeeded or failed depending on the existence of an `exit_code` and the value of that exit code.
5. `NEEDS_RETRY` - on occassion, due to host level characteristics (full disk, too many open files, timeouts pulling image, etc) the run exits with a null exit code without ever being executed. In this case the reason is analyzed to determine if the run is retriable. If it is, the task transitions to this status and is allocated to the appropriate execution queue again, and will repeat the lifecycle.
#### Normal Lifecycle
`QUEUED` --> `PENDING` --> `RUNNING` --> `STOPPED`
#### Retry Lifecycle
... --> `PENDING` --> `STOPPED` --> `NEEDS_RETRY` --> `QUEUED` --> ...
## Deploying
In a production deployment you'll want multiple instances of the flotilla service running and postgres running elsewhere (eg. Amazon RDS). In this case the most salient detail configuration detail is the `DATABASE_URL`.
### Docker based deploy
The simplest way to deploy for very light usage is to avoid a reverse proxy and deploy directly with docker.
1. Build and tag an image for flotilla using the `Dockerfile` provided in this repo:
```
docker build -t <your repo name>/flotilla:<version tag>
```
2. Run this image wherever you deploy your services:
```
docker run -e DATABASE_URL=<your db url> -e FLOTILLA_MODE=prod -p 3000:3000 ...<other standard docker run args>
```
> Notes:
> -----
> * Flotilla uses [viper](https://github.com/spf13/viper) for configuration so you can override any of the default configuration under `conf/` using run time environment variables passed to `docker run`
> * In most realistic deploys you'll likely want to configure a reverse proxy to sit in front of the flotilla container. See the docs [here](https://hub.docker.com/_/nginx/)
See [docker run](https://docs.docker.com/engine/reference/run/) for more details
### Configuration In Detail
The variables in `conf/config.yml` are sensible defaults. Most should be left alone unless you're developing flotilla itself. However, there are a few you may want to change in a production environment.
| Variable Name | Description |
| ------------- | ----------- |
| `worker_retry_interval` | Run frequency of the retry worker |
| `worker_submit_interval` | Poll frequency of the submit worker |
| `worker_status_interval` | Poll frequency of the status update worker |
| `http_server_read_timeout_seconds` | Sets read timeout in seconds for the http server |
| `http_server_write_timeout_seconds` | Sets the write timeout in seconds for the http server |
| `http_server_listen_address` | The port for the http server to listen on |
| `owner_id_var` | Which environment variable containing ownership information to inject into the runtime of jobs |
| `enabled_workers` | This variable is a list of the workers that run. Use this to control what workers run when using a multi-container deployment strategy. Valid list items include (`retry`, `submit`, and `status`) |
| `metrics_dogstatsd_address` | Statds metrics host in Datadog format |
| `metrics_dogstatsd_namespace` | Namespace for the metrics - for example `flotilla.` |
| `redis_address` | Redis host for caching and locks|
| `redis_db` | Redis db to be used - numeric |
| `eks_clusters` | hash-map of cluster-name and it's associated kubeconfig (encoded in base64) |
| `eks_kubeconfig_basepath` | folder where the kubeconfigs are stored |
| `eks_cluster_ondemand_whitelist` | override list of cluster names where to force ondemand node types |
| `eks_cluster_override` | EKS clusters to override traffic |
| `eks_scheduler_name` | Custom scheduler name to use, default is `kube-scheduler` |
| `eks_manifest_storage.options.region` | Kubernetes manifest s3 upload bucket aws region |
| `eks_manifest_storage_options_s3_bucket_name` | S3 bucket name for manifest storage. |
| `eks_manifest_storage_options_s3_bucket_root_dir` | S3 root bucket path. |
| `eks_log_namespace_retention_days` | Number of days to store logs. |
| `eks_log_namespace_driver_name` | Logger name. |
| `eks_log_namespace_driver_options_s3_bucket_name` | S3 bucket name to store logs. |
| `eks_log_namespace_driver_options_s3_bucket_root_dir` | S3 root bucket path within the bucket.|
| `eks_job_namespace` | Kubernetes namespace to submit jobs to. |
| `eks_job_ttl` | default job ttl in seconds |
| `eks_job_queue` | SQS job queue - the api places the jobs on this queue and the submit worker asynchronously submits it to Kubernetes/EKS |
| `eks.service_account` | Kubernetes service account to use for jobs. |
## Development
### API Documentation
See [API](https://stitchfix.github.io/flotilla-os/api.html)
### Building
Currently Flotilla is built using `go` 1.9.3 and uses the `go mod` to manage dependencies.
```
go get && go build
```
================================================
FILE: ara-impact-report-staging.md
================================================
# ARA Impact Analysis Report - STAGING Environment
## 10-Day Analysis of Adaptive Resource Allocation (Dec 7-17, 2025)
### Executive Summary
This report analyzes the impact of the ARA bug fix deployed on **December 16, 2025** in the **STAGING environment**.
**Key Findings:**
- **forklift-deploy-model-v1**: Fix deployed mid-day Dec 16, full effect on Dec 17
- Before fix (Dec 7-15): NULL `command_hash`, memory 4-6.5GB (at/below baseline)
- After fix (Dec 17): Proper `command_hash`, memory 4-6.5GB (unchanged)
- **No memory over-allocation issue in staging** (unlike production)
- **python-3.11 jobs**: Working correctly with ARA
- Baseline: 50MB
- Elevated: 1-16GB via ARA (reasonable levels)
- **No extreme 350GB allocations** (staging max is 40GB)
- **GPU jobs**: None in staging environment
- **Environment difference**: Staging has much lower max memory ceiling (40GB vs 350GB in production)
---
## Environment Overview
**Database Container**: `77b8e13079e5` (postgres:16)
**Analysis Period**: 2025-12-07 to 2025-12-17 (10 days)
**Total Jobs**: 125,154 jobs from 14 unique definitions
---
## Query 1: forklift-deploy-model-v1 Command Hash Population
### Query
```sql
SELECT DATE(queued_at) as date,
command_hash IS NULL as hash_null,
COUNT(*) as count
FROM task
WHERE definition_id IN (SELECT definition_id FROM task_def WHERE alias = 'forklift-deploy-model-v1')
AND queued_at >= CURRENT_DATE - INTERVAL '10 days'
GROUP BY DATE(queued_at), command_hash IS NULL
ORDER BY date, hash_null;
```
### Results
```
date | hash_null | count
------------+-----------+-------
2025-12-07 | t | 30
2025-12-08 | t | 35
2025-12-09 | t | 57
2025-12-10 | t | 31
2025-12-11 | t | 33
2025-12-12 | t | 30
2025-12-13 | t | 30
2025-12-14 | t | 25
2025-12-15 | t | 30
2025-12-16 | f | 5 ← Fix deployed (partial)
2025-12-16 | t | 25
2025-12-17 | f | 30 ← Fix fully active
```
### Analysis
- **Dec 7-15**: 100% of forklift jobs had NULL `command_hash` (301 jobs total)
- **Dec 16**: Transition day - 5 jobs with proper hash, 25 with NULL (fix deployed mid-day)
- **Dec 17**: 100% of forklift jobs have proper `command_hash` (30 jobs)
- **Fix deployment time**: Mid-day December 16, 2025
---
## Query 2: forklift-deploy-model-v1 Memory Allocations
### Query
```sql
SELECT DATE(queued_at) as date,
MIN(memory) as min_mem,
MAX(memory) as max_mem,
AVG(memory)::int as avg_mem,
COUNT(*) as count
FROM task
WHERE definition_id IN (SELECT definition_id FROM task_def WHERE alias = 'forklift-deploy-model-v1')
AND queued_at >= CURRENT_DATE - INTERVAL '10 days'
GROUP BY DATE(queued_at)
ORDER BY date;
```
### Results
```
date | min_mem | max_mem | avg_mem | count
------------+---------+---------+---------+-------
2025-12-07 | 4000 | 6500 | 5500 | 30
2025-12-08 | 4000 | 6500 | 5286 | 35
2025-12-09 | 4000 | 6500 | 4789 | 57
2025-12-10 | 4000 | 6500 | 5452 | 31
2025-12-11 | 4000 | 8500 | 5500 | 33
2025-12-12 | 4000 | 6500 | 5500 | 30
2025-12-13 | 4000 | 6500 | 5500 | 30
2025-12-14 | 4000 | 6500 | 5500 | 25
2025-12-15 | 4000 | 6500 | 5500 | 30
2025-12-16 | 4000 | 6500 | 5500 | 30
2025-12-17 | 4000 | 6500 | 5500 | 30
```
### Analysis
- **Baseline**: 8GB (8000MB) from task definition
- **Memory allocations**: 4-6.5GB (all at or below baseline)
- **Before fix**: Despite NULL `command_hash`, no memory over-allocation
- **After fix**: Memory unchanged (4-6.5GB range)
- **Key difference from production**: Staging forklift jobs **never exhibited the 18-33GB over-allocation** seen in production
---
## Query 3: Elevated Memory Jobs (ARA Impact)
### Query
```sql
SELECT DATE(t.queued_at) as date,
COUNT(*) as elevated_jobs,
COUNT(DISTINCT t.definition_id) as unique_defs
FROM task t
JOIN task_def td ON t.definition_id = td.definition_id
WHERE t.memory > td.memory * 1.5
AND td.adaptive_resource_allocation = true
AND t.queued_at >= CURRENT_DATE - INTERVAL '10 days'
GROUP BY DATE(t.queued_at)
ORDER BY date;
```
### Results
```
date | elevated_jobs | unique_defs
------------+---------------+-------------
2025-12-07 | 134 | 1
2025-12-08 | 129 | 1
2025-12-09 | 150 | 1
2025-12-10 | 217 | 1
2025-12-11 | 416 | 1
2025-12-12 | 420 | 1
2025-12-13 | 417 | 1
2025-12-14 | 418 | 1
2025-12-15 | 413 | 1
2025-12-16 | 450 | 1
2025-12-17 | 395 | 1
```
### Analysis
- **Total elevated jobs**: 3,559 jobs over 10 days
- **All from one definition**: `python-3.11` (baseline: 50MB)
- **Average**: ~324 elevated jobs per day
- **Pattern**: Consistent elevation throughout the period (no change after fix)
- **This is expected**: python-3.11 jobs have proper `command_hash` throughout
---
## Query 4: python-3.11 Memory Elevation Details
### Query
```sql
SELECT DATE(t.queued_at) as date,
td.alias,
td.memory as baseline_mb,
t.memory as allocated_mb,
CAST((t.memory::float / td.memory) as numeric(10,2)) as multiplier,
COUNT(*) as job_count
FROM task t
JOIN task_def td ON t.definition_id = td.definition_id
WHERE t.memory > td.memory * 1.5
AND td.adaptive_resource_allocation = true
AND t.queued_at >= CURRENT_DATE - INTERVAL '10 days'
GROUP BY DATE(t.queued_at), td.alias, td.memory, t.memory
ORDER BY date, job_count DESC
LIMIT 50;
```
### Results (sample)
```
date | alias | baseline_mb | allocated_mb | multiplier | job_count
------------+-------------+-------------+--------------+------------+-----------
2025-12-11 | python-3.11 | 50 | 1024 | 20.48 | 284
2025-12-11 | python-3.11 | 50 | 4096 | 81.92 | 88
2025-12-11 | python-3.11 | 50 | 1792 | 35.84 | 39
2025-12-11 | python-3.11 | 50 | 8000 | 160.00 | 5
2025-12-12 | python-3.11 | 50 | 1024 | 20.48 | 292
2025-12-12 | python-3.11 | 50 | 4096 | 81.92 | 88
2025-12-12 | python-3.11 | 50 | 1792 | 35.84 | 32
2025-12-12 | python-3.11 | 50 | 8000 | 160.00 | 5
2025-12-12 | python-3.11 | 50 | 16000 | 320.00 | 3
```
### Analysis
- **Elevation levels**:
- 1GB (1024MB): Most common (~300 jobs/day)
- 4GB (4096MB): Consistent (~88 jobs/day)
- 8GB (8000MB): Regular (~5 jobs/day)
- 16GB (16000MB): Rare (3 jobs total)
- **No extreme allocations**: Max is 16GB (vs 350GB in production)
- **Reasonable multipliers**: 20-320x (vs 7000x in production)
---
## Query 5: python-3.11 Command Hash Status
### Query
```sql
SELECT DATE(queued_at) as date,
command_hash IS NULL as hash_null,
COUNT(*) as count
FROM task
WHERE definition_id IN (SELECT definition_id FROM task_def WHERE alias = 'python-3.11')
AND queued_at >= CURRENT_DATE - INTERVAL '10 days'
GROUP BY DATE(queued_at), command_hash IS NULL
ORDER BY date, hash_null;
```
### Results
```
date | hash_null | count
------------+-----------+-------
2025-12-07 | f | 134
2025-12-08 | f | 129
2025-12-09 | f | 150
2025-12-10 | f | 217
2025-12-11 | f | 416
2025-12-12 | f | 420
2025-12-13 | f | 417
2025-12-14 | f | 418
2025-12-15 | f | 413
2025-12-16 | f | 450
2025-12-17 | f | 396
```
### Analysis
- **100% of python-3.11 jobs** have proper `command_hash` throughout the entire period
- **ARA working correctly**: Jobs are elevated based on proper command hash lookups
- **No NULL command_hash issue**: Unlike forklift, python-3.11 had command_hash all along
---
## Query 6: GPU Jobs Analysis
### Query
```sql
SELECT COUNT(*) as gpu_job_count,
COUNT(DISTINCT definition_id) as unique_definitions
FROM task
WHERE gpu IS NOT NULL AND gpu > 0
AND queued_at >= CURRENT_DATE - INTERVAL '10 days';
```
### Results
```
gpu_job_count | unique_definitions
---------------+--------------------
0 | 0
```
### Analysis
- **No GPU jobs** in staging environment over the past 10 days
- The GPU detection bug fix is not testable in staging
- GPU jobs appear to be production-only workloads
---
## Query 7: Memory Distribution
### Query
```sql
SELECT memory,
COUNT(*)
FROM task
WHERE queued_at >= CURRENT_DATE - INTERVAL '10 days'
GROUP BY memory
ORDER BY memory DESC
LIMIT 15;
```
### Results
```
memory | count
--------+--------
| 3536 ← NULL (jobs still queued/pending)
40960 | 22 ← 40GB (max in staging)
20000 | 3
16000 | 3
8500 | 1
8000 | 57
6500 | 195
4096 | 973
4000 | 213
2744 | 1
2048 | 1073
1792 | 123
1568 | 2
1024 | 101156 ← Most common (1GB)
1000 | 58
```
### Analysis
- **Max memory allocated**: 40GB (40,960MB)
- **Most common**: 1GB (1,024MB) - 101,156 jobs (80.7%)
- **Distribution**: Heavily skewed toward small allocations
- **No extreme allocations**: Nothing above 40GB
---
## Staging vs Production Comparison
| Metric | Production | Staging | Notes |
|--------|-----------|---------|-------|
| **Max memory limit** | 350GB | 40GB | Staging has 8.75x lower ceiling |
| **forklift over-allocation** | 18-33GB (before fix) | None | Staging had no issue |
| **python-3.11 max allocation** | 350GB | 16GB | 21.8x difference |
| **GPU jobs** | 460 jobs | 0 jobs | Production only |
| **Total jobs (10 days)** | 280,215 | 125,154 | Production 2.2x larger |
| **command_hash fix date** | Dec 16 | Dec 16 | Same deployment |
---
## Conclusions
### Fix Effectiveness in Staging: ✅ Verified
1. **forklift-deploy-model-v1**:
- **Before fix (Dec 7-15)**: NULL `command_hash` but no memory issues
- **After fix (Dec 17)**: Proper `command_hash`, memory unchanged
- **No over-allocation problem** in staging (unlike production)
- Root cause: Staging already had lower max memory limits
2. **python-3.11**:
- **Throughout period**: Proper `command_hash`, ARA working correctly
- **Elevated to**: 1-16GB (reasonable levels)
- **No extreme allocations**: Staging max limit prevents 350GB scenario
3. **Environment differences**:
- Staging has **40GB max memory** vs production's **350GB**
- This prevented the extreme allocation issue we saw in production
- Staging is a safer environment for testing ARA changes
### Key Insights
1. **Staging didn't exhibit the production issue** because:
- Lower max memory ceiling (40GB vs 350GB)
- forklift jobs stayed within reasonable bounds despite NULL `command_hash`
2. **The fix deployed successfully**:
- Mid-day Dec 16: Partial deployment
- Dec 17: Full effect with 100% proper `command_hash`
3. **No GPU jobs in staging**:
- Cannot validate GPU bug fix in this environment
- GPU workloads are production-specific
### Recommendations
1. **Production parity**: Consider raising staging max memory to match production (248GB new limit) for better testing
2. **GPU testing**: Add GPU job definitions to staging for comprehensive ARA testing
3. **Monitoring**: The fix is working correctly in staging, safe to deploy the 248GB limit reduction
4. **No action needed**: Staging forklift jobs are healthy and don't require intervention
---
## Appendix: Container Information
- **Database Container**: `77b8e13079e5` (postgres:16)
- **Database URL**: Available as `$FLOTILLA_DATABASE_URL` in container environment
- **Environment**: STAGING
- **Report Generated**: 2025-12-17
- **Analysis Period**: 2025-12-07 to 2025-12-17 (10 days)
- **Fix Deployed**: 2025-12-16 (mid-day)
---
## Sample Query Template
To reproduce this analysis or run ad-hoc queries:
```bash
docker exec 77b8e13079e5 bash -c 'psql $FLOTILLA_DATABASE_URL -c "YOUR_QUERY_HERE"'
```
Example:
```bash
docker exec 77b8e13079e5 bash -c 'psql $FLOTILLA_DATABASE_URL -c "SELECT COUNT(*) FROM task WHERE memory > 10000 AND queued_at >= CURRENT_DATE - INTERVAL '\''1 day'\'';"'
```
================================================
FILE: ara-impact-report.md
================================================
# ARA Impact Analysis Report
## 10-Day Analysis of Adaptive Resource Allocation (Dec 7-17, 2025)
### Executive Summary
This report analyzes the impact of the ARA bug fix deployed on **December 16, 2025**. The fix changed ARA lookups from using `description` to `command_hash`, preventing incorrect resource allocation matches.
**Key Findings:**
- **350GB allocations** (baseline: 50MB): Continue at expected levels (legitimate OOM responses)
- **forklift-deploy-model-v1 elevations** (baseline: 8GB): **Completely eliminated** after fix deployment
- **Fix effectiveness**: 100% resolution for the forklift issue (21 elevated jobs/day → 0 elevated jobs/day)
- **Root cause identified**: `command_hash` was NULL before fix despite having command text
- The fix both (a) started calculating `command_hash` properly and (b) changed ARA lookup logic
- Before: NULL `command_hash` + NULL `description` → incorrect ARA matches → 18-33GB allocations
- After: Proper `command_hash` (19432e77...) → correct lookups → 4-7GB allocations (at baseline)
---
## Query 1: Daily Count of 350GB Memory Jobs
### Query
```sql
SELECT DATE(queued_at) as date,
COUNT(*) as count_350gb_jobs
FROM task
WHERE memory = 350000
AND queued_at >= CURRENT_DATE - INTERVAL '10 days'
GROUP BY DATE(queued_at)
ORDER BY date
LIMIT 15;
```
### Results
```
date | count_350gb_jobs
------------+------------------
2025-12-07 | 14
2025-12-08 | 14
2025-12-09 | 29
2025-12-10 | 53
2025-12-11 | 16
2025-12-12 | 30
2025-12-13 | 16
2025-12-14 | 14
2025-12-15 | 15
2025-12-16 | 52 ← Fix deployed
2025-12-17 | 14
```
### Analysis
- **Average before fix (Dec 7-15)**: 21.2 jobs/day
- **Day of fix (Dec 16)**: 52 jobs (spike likely due to deployment activity)
- **After fix (Dec 17)**: 14 jobs (within normal range)
- These jobs have a **baseline of only 50MB** but allocate **350GB** (7000x increase)
---
## Query 2: 350GB Jobs by Definition/Alias
### Query
```sql
SELECT DATE(t.queued_at) as date,
td.alias,
COUNT(*) as job_count
FROM task t
JOIN task_def td ON t.definition_id = td.definition_id
WHERE t.memory = 350000
AND t.queued_at >= CURRENT_DATE - INTERVAL '10 days'
GROUP BY DATE(t.queued_at), td.alias
ORDER BY date, job_count DESC
LIMIT 50;
```
### Results (sample)
```
date | alias | job_count
------------+----------------------+-----------
2025-12-15 | python-3.11 | 10
2025-12-15 | pytorch2-24.05-py3_8 | 3
2025-12-15 | pytorch2-24.05-py3_1 | 2
2025-12-16 | python-3.11 | 30
2025-12-16 | pytorch2-24.05-py3_8 | 15
2025-12-16 | pytorch2-24.05-py3_1 | 7
2025-12-17 | python-3.11 | 5
2025-12-17 | pytorch2-24.05-py3_8 | 5
2025-12-17 | pytorch2-24.05-py3_1 | 4
```
### Analysis
- Three definition aliases affected: `python-3.11`, `pytorch2-24.05-py3_8`, `pytorch2-24.05-py3_1`
- All three definitions have baseline memory of **50MB**
- Distribution across definitions remains consistent before and after fix
- These appear to be **legitimate ARA responses** to actual OOM conditions
---
## Query 3: Other Elevated Memory Jobs (Non-350GB)
### Query
```sql
SELECT DATE(t.queued_at) as date,
COUNT(*) as elevated_jobs,
COUNT(DISTINCT t.definition_id) as unique_defs
FROM task t
JOIN task_def td ON t.definition_id = td.definition_id
WHERE t.memory > td.memory * 1.5
AND td.adaptive_resource_allocation = true
AND t.queued_at >= CURRENT_DATE - INTERVAL '10 days'
GROUP BY DATE(t.queued_at)
ORDER BY date
LIMIT 15;
```
### Results
```
date | elevated_jobs | unique_defs
------------+---------------+-------------
2025-12-07 | 16 | 1
2025-12-08 | 11 | 1
2025-12-09 | 14 | 1
2025-12-10 | 24 | 1
2025-12-11 | 4 | 1
2025-12-12 | 5 | 1
2025-12-13 | 10 | 1
2025-12-14 | 6 | 1
2025-12-15 | 21 | 1
2025-12-16 | 5 | 1 ← Fix deployed
2025-12-17 | 0 | 0 ← No elevated jobs!
```
### Analysis
- **Average before fix (Dec 7-15)**: 12.3 elevated jobs/day
- **After fix (Dec 17)**: **0 jobs** ✅
- All elevated jobs came from a **single definition** (forklift-deploy-model-v1)
- **100% fix effectiveness** for this issue
---
## Query 4: Detailed Elevation Analysis (forklift-deploy-model-v1)
### Query
```sql
SELECT DATE(t.queued_at) as date,
td.alias,
td.memory as baseline_mb,
t.memory as allocated_mb,
CAST((t.memory::float / td.memory) as numeric(10,2)) as multiplier,
COUNT(*) as job_count
FROM task t
JOIN task_def td ON t.definition_id = td.definition_id
WHERE t.memory > td.memory * 1.5
AND td.adaptive_resource_allocation = true
AND t.queued_at >= CURRENT_DATE - INTERVAL '10 days'
GROUP BY DATE(t.queued_at), td.alias, td.memory, t.memory
ORDER BY date, job_count DESC
LIMIT 40;
```
### Results (sample)
```
date | alias | baseline_mb | allocated_mb | multiplier | job_count
------------+--------------------------+-------------+--------------+------------+-----------
2025-12-14 | forklift-deploy-model-v1 | 8000 | 19000 | 2.38 | 4
2025-12-14 | forklift-deploy-model-v1 | 8000 | 33000 | 4.13 | 2
2025-12-15 | forklift-deploy-model-v1 | 8000 | 33000 | 4.13 | 17
2025-12-15 | forklift-deploy-model-v1 | 8000 | 19000 | 2.38 | 4
2025-12-16 | forklift-deploy-model-v1 | 8000 | 19000 | 2.38 | 4
2025-12-16 | forklift-deploy-model-v1 | 8000 | 33000 | 4.13 | 1
2025-12-17 | (no results) | N/A | N/A | N/A | 0
```
### Analysis
- **Baseline**: 8GB (8000MB)
- **Elevated allocations**:
- 18GB (2.25x multiplier)
- 19GB (2.38x multiplier)
- 33GB (4.13x multiplier)
- **Peak day**: Dec 15 with 21 total elevated jobs
- **After fix**: Complete elimination on Dec 17
---
## Query 5: Command Hash Diversity (350GB Jobs)
### Query
```sql
SELECT DATE(t.queued_at) as date,
td.alias,
COUNT(*) as total_jobs,
COUNT(DISTINCT t.command_hash) as unique_commands
FROM task t
JOIN task_def td ON t.definition_id = td.definition_id
WHERE t.memory = 350000
AND t.queued_at >= CURRENT_DATE - INTERVAL '10 days'
GROUP BY DATE(t.queued_at), td.alias
ORDER BY date, total_jobs DESC
LIMIT 50;
```
### Results (sample)
```
date | alias | total_jobs | unique_commands
------------+----------------------+------------+-----------------
2025-12-15 | python-3.11 | 10 | 5
2025-12-15 | pytorch2-24.05-py3_8 | 3 | 3
2025-12-15 | pytorch2-24.05-py3_1 | 2 | 2
2025-12-16 | python-3.11 | 30 | 8
2025-12-16 | pytorch2-24.05-py3_8 | 15 | 7
2025-12-16 | pytorch2-24.05-py3_1 | 7 | 5
2025-12-17 | python-3.11 | 5 | 5
2025-12-17 | pytorch2-24.05-py3_8 | 5 | 5
2025-12-17 | pytorch2-24.05-py3_1 | 4 | 4
```
### Analysis
- **High command diversity**: Multiple unique command hashes per day
- **Dec 15**: 15 jobs with 10 unique commands (67% unique)
- **Dec 17**: 14 jobs with 14 unique commands (100% unique)
- This diversity indicates **legitimate ARA responses** to different workloads with actual OOM history
- The fix correctly uses `command_hash` for matching, not generic descriptions
---
## Query 6: Command Hash Analysis (forklift-deploy-model-v1)
### Query
```sql
SELECT DATE(t.queued_at) as date,
t.memory as allocated_mb,
COUNT(*) as total_jobs,
COUNT(t.command_hash) as non_null_hashes,
COUNT(DISTINCT t.command_hash) as unique_commands
FROM task t
JOIN task_def td ON t.definition_id = td.definition_id
WHERE td.alias = 'forklift-deploy-model-v1'
AND t.memory > td.memory * 1.5
AND t.queued_at >= CURRENT_DATE - INTERVAL '10 days'
GROUP BY DATE(t.queued_at), t.memory
ORDER BY date, allocated_mb
LIMIT 50;
```
### Results (sample)
```
date | allocated_mb | total_jobs | non_null_hashes | unique_commands
------------+--------------+------------+-----------------+-----------------
2025-12-14 | 19000 | 4 | 0 | 0
2025-12-14 | 33000 | 2 | 0 | 0
2025-12-15 | 19000 | 4 | 0 | 0
2025-12-15 | 33000 | 17 | 0 | 0
2025-12-16 | 19000 | 4 | 0 | 0
2025-12-16 | 33000 | 1 | 0 | 0
```
### Critical Finding: The command_hash Bug
**Before Fix (Dec 7-16):**
- **ALL forklift-deploy-model-v1 jobs had `command_hash = NULL`** (despite having a 206-char shell script)
- The `description` field is also **always NULL** for forklift jobs
- With both NULL, the old ARA code was incorrectly matching these jobs, causing false elevations
**After Fix (Dec 17):**
- `command_hash = 19432e77696deb6666bb12c67feb2b8d` (now properly calculated)
- All forklift jobs get the same hash because they run the identical command
- ARA now correctly looks up this hash and finds no OOM history
- Result: No elevation (jobs run at or below the 8GB baseline)
---
## Query 7: Baseline vs Allocated Memory (350GB Jobs)
### Query
```sql
SELECT t.definition_id,
td.memory as baseline_memory,
t.memory as allocated_memory,
COUNT(*) as job_count
FROM task t
JOIN task_def td ON t.definition_id = td.definition_id
WHERE t.memory = 350000
AND t.queued_at >= CURRENT_DATE - INTERVAL '3 days'
GROUP BY t.definition_id, td.memory, t.memory
ORDER BY job_count DESC
LIMIT 20;
```
### Results
```
definition_id | baseline_memory | allocated_memory | job_count
---------------------------------------------------------+-----------------+------------------+-----------
sf-base_python-3_11-7449eda4-b8b3-4146-77c5-a47f8caac81b | 50 | 350000 | 52
sf-base_pytorch2-24__5-py3-505a283c-1e0a-43da-4c9b-071... | 50 | 350000 | 24
sf-base_pytorch2-24__5-py3-ceef4c9e-6ebc-41e5-6cef-a33... | 50 | 350000 | 16
```
### Analysis
- **Massive increase**: 50MB → 350GB (7000x multiplier)
- Indicates these are **ML training jobs** with significant memory requirements
- The ARA system is correctly identifying commands that have historically run out of memory
- These allocations continue appropriately after the fix
---
## Query 8: forklift-deploy-model-v1 Memory Allocation Timeline
### Query
```sql
SELECT DATE(queued_at) as date,
MIN(memory) as min_mem,
MAX(memory) as max_mem,
AVG(memory)::int as avg_mem,
COUNT(*) as count
FROM task
WHERE definition_id IN (SELECT definition_id FROM task_def WHERE alias = 'forklift-deploy-model-v1')
AND queued_at >= CURRENT_DATE - INTERVAL '10 days'
GROUP BY DATE(queued_at)
ORDER BY date;
```
### Results
```
date | min_mem | max_mem | avg_mem | count
------------+---------+---------+---------+-------
2025-12-07 | 4000 | 33000 | 13431 | 35
2025-12-08 | 4000 | 33000 | 10792 | 38
2025-12-09 | 4000 | 33000 | 13062 | 34
2025-12-10 | 4000 | 33000 | 13117 | 52
2025-12-11 | 4000 | 19000 | 9392 | 13
2025-12-12 | 4000 | 33000 | 11842 | 12
2025-12-13 | 4000 | 33000 | 9524 | 46
2025-12-14 | 4000 | 33000 | 8930 | 27
2025-12-15 | 4000 | 33000 | 18078 | 40
2025-12-16 | 4000 | 33000 | 10807 | 15
2025-12-17 | 4000 | 7000 | 5007 | 15 ← Fix deployed
```
### Analysis
- **Baseline**: 8GB (8000 MB)
- **Before fix**: Jobs randomly allocated 4-33GB (some below baseline, many elevated)
- **After fix**: Jobs allocated 4-7GB (all at or below baseline) ✅
### The command Field Content
Query to inspect the command field:
```sql
SELECT DISTINCT command, command_hash
FROM task
WHERE definition_id IN (SELECT definition_id FROM task_def WHERE alias = 'forklift-deploy-model-v1')
AND queued_at >= CURRENT_DATE
LIMIT 1;
```
Result shows forklift jobs run this **206-character shell script**:
```bash
#
# Use absolute latest forklift
#
mkdir -p /code/stitchfix
cd /code/stitchfix
git clone -b $GIT_BRANCH --single-branch git@github.com:stitchfix/forklift.git
cd forklift/destinations/ml_model_deploy/
./run
```
**Key Insight**: The command field is **NOT empty** - but `command_hash` was NULL before the fix, preventing proper ARA lookups.
---
## Query 9: command_hash Population Status by Date
### Query
```sql
SELECT DATE(queued_at) as date,
command_hash IS NULL as hash_null,
COUNT(*) as count
FROM task
WHERE definition_id IN (SELECT definition_id FROM task_def WHERE alias = 'forklift-deploy-model-v1')
AND queued_at >= CURRENT_DATE - INTERVAL '10 days'
GROUP BY DATE(queued_at), command_hash IS NULL
ORDER BY date, hash_null;
```
### Results
```
date | hash_null | count
------------+-----------+-------
2025-12-07 | t | 35
2025-12-08 | t | 38
2025-12-09 | t | 34
2025-12-10 | t | 52
2025-12-11 | t | 13
2025-12-12 | t | 12
2025-12-13 | t | 46
2025-12-14 | t | 27
2025-12-15 | t | 40
2025-12-16 | t | 15
2025-12-17 | f | 15 ← command_hash now populated!
```
### Analysis
- **Dec 7-16**: 100% of forklift jobs had `command_hash = NULL`
- **Dec 17**: 100% of forklift jobs have `command_hash = 19432e77696deb6666bb12c67feb2b8d`
- The fix not only changed the lookup logic but also **started calculating command_hash** for new jobs
---
## Conclusions
### Fix Effectiveness: ✅ Confirmed
1. **forklift-deploy-model-v1 issue**: **100% resolved**
- Before: 12.3 elevated jobs/day (average, elevated to 18-33GB)
- After: 0 elevated jobs (all at or below 8GB baseline)
- Root cause discovered:
- The command field was populated (206-char shell script) but `command_hash` was **NULL**
- The description field was also **NULL**
- The fix both (a) started calculating `command_hash` and (b) changed lookup logic
- Now all forklift jobs get the same `command_hash` and ARA finds no OOM history for it
2. **350GB allocations**: **Working as designed**
- Jobs continue at expected levels
- High command hash diversity (different workloads)
- Baseline of 50MB suggests these are script runners with variable workloads
- ARA correctly identifies specific commands with OOM history
### Before and After Comparison
| Metric | Dec 15 (Before) | Dec 17 (After) | Change |
|--------|----------------|----------------|---------|
| 350GB jobs | 15 | 14 | -7% (normal variance) |
| forklift elevated | 21 | 0 | -100% ✅ |
| Total elevated | 36 | 14 | -61% |
### Recommendations
1. **Monitor next 7 days**: Verify forklift-deploy-model-v1 remains at baseline (8GB) ✅
2. **350GB jobs**: These appear legitimate - monitor for OOM failures to validate
3. **Command hash calculation**:
- Investigate why `command_hash` was NULL before Dec 17
- Verify all new jobs now properly calculate `command_hash`
- Consider backfilling `command_hash` for historical records if needed for analytics
4. **ARA lookup logic**: Confirm the fix properly handles NULL `command_hash` cases (doesn't match)
5. **Documentation**: Update ARA docs to clarify:
- `command_hash` is calculated from the `command` field (not `description`)
- ARA requires valid `command_hash` for proper operation
- Behavior when `command_hash` is NULL
---
## Appendix: Container Information
- **Database Container**: `360a9dd48242` (postgres:16)
- **Database URL**: Available as `$FLOTILLA_DATABASE_URL` in container environment
- **Report Generated**: 2025-12-17 (updated with latest data)
- **Analysis Period**: 2025-12-07 to 2025-12-17 (10 days)
- **Fix Deployed**: 2025-12-16
### Update Log
- **Initial report**: Generated with data up to 12 jobs on Dec 17
- **Updated**: Refreshed with latest data showing 14 jobs on Dec 17 (100% unique command hashes)
---
## Sample Query Template
To reproduce this analysis or run ad-hoc queries:
```bash
docker exec 360a9dd48242 bash -c 'psql $FLOTILLA_DATABASE_URL -c "YOUR_QUERY_HERE"'
```
Example:
```bash
docker exec 360a9dd48242 bash -c 'psql $FLOTILLA_DATABASE_URL -c "SELECT COUNT(*) FROM task WHERE memory = 350000 AND queued_at >= CURRENT_DATE - INTERVAL '\''1 day'\'';"'
```
================================================
FILE: clients/cluster/cluster.go
================================================
package cluster
import (
"fmt"
"github.com/pkg/errors"
"github.com/stitchfix/flotilla-os/config"
"github.com/stitchfix/flotilla-os/state"
)
//
// Client validates whether or not the given definition can be run
// on the specified cluster. This is to prevent infinite queue
// times - the case that the requested resources will -never- become
// available on the user's chosen cluster
//
type Client interface {
Name() string
Initialize(conf config.Config) error
CanBeRun(clusterName string, executableResources state.ExecutableResources) (bool, error)
ListClusters() ([]state.ClusterMetadata, error)
}
// NewClusterClient returns a cluster client
func NewClusterClient(conf config.Config, name string) (Client, error) {
switch name {
case "eks":
eksc := &EKSClusterClient{}
if err := eksc.Initialize(conf); err != nil {
return nil, errors.Wrap(err, "problem initializing EKSClusterClient")
}
return eksc, nil
default:
return nil, fmt.Errorf("No Client named [%s] was found", name)
}
}
================================================
FILE: clients/cluster/eks_cluster_client.go
================================================
package cluster
import (
"github.com/stitchfix/flotilla-os/config"
"github.com/stitchfix/flotilla-os/state"
)
// EKSClusterClient is the cluster client for EKS
// [NOTE] This client assumes the EKS cluster is capable is running a mixed varieties of jobs.
type EKSClusterClient struct{}
func (EKSClusterClient) Name() string {
return ""
}
func (EKSClusterClient) Initialize(conf config.Config) error {
return nil
}
// CanBeRun for EKSCluster is always true
func (EKSClusterClient) CanBeRun(clusterName string, executableResources state.ExecutableResources) (bool, error) {
return true, nil
}
// Since it is a single cluster environment for EKS, slice of clusters is empty.
func (EKSClusterClient) ListClusters() ([]state.ClusterMetadata, error) {
return []state.ClusterMetadata{}, nil
}
================================================
FILE: clients/httpclient/client.go
================================================
package httpclient
import (
"bytes"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"strings"
"time"
)
type RetryableError interface {
Err() string
}
type HttpRetryableError struct {
e error
}
func (re HttpRetryableError) Error() string {
return re.e.Error()
}
func (re HttpRetryableError) Err() string {
return re.e.Error()
}
type RequestExecutor interface {
Do(req *http.Request, timeout time.Duration, entity interface{}) error
}
type defaultExecutor struct{}
func (de *defaultExecutor) Do(req *http.Request, timeout time.Duration, entity interface{}) error {
client := http.Client{Timeout: timeout}
if client.Timeout == 0 {
client.Timeout = time.Second * 10
}
r, err := client.Do(req)
if r != nil {
defer r.Body.Close()
}
if err != nil {
return err
}
if r.StatusCode >= 200 && r.StatusCode < 400 {
return json.NewDecoder(r.Body).Decode(entity)
} else if r.StatusCode >= 500 {
return HttpRetryableError{fmt.Errorf("Error response: %v", r.Status)}
} else {
return fmt.Errorf("Error response: %v", r.Status)
}
}
// Generic http client to make http requests.
type Client struct {
Host string
Timeout time.Duration
RetryCount int
Executor RequestExecutor
}
func (c *Client) Get(path string, headers map[string]string, entity interface{}) error {
req, err := c.prepareRequestNoBody("GET", path, headers)
if err != nil {
return fmt.Errorf("httpclient GET: %v", err)
}
return c.doRequestWithRetry(req, entity)
}
func (c *Client) Delete(path string, headers map[string]string, entity interface{}) error {
req, err := c.prepareRequestNoBody("DELETE", path, headers)
if err != nil {
return fmt.Errorf("httpclient DELETE: %v", err)
}
return c.doRequestWithRetry(req, entity)
}
func (c *Client) Put(path string, headers map[string]string, inEntity interface{}, outEntity interface{}) error {
req, err := c.prepareRequestWithBody("PUT", path, headers, inEntity)
if err != nil {
return fmt.Errorf("httpclient PUT: %v", err)
}
return c.doRequestWithRetry(req, outEntity)
}
func (c *Client) Post(path string, headers map[string]string, inEntity interface{}, outEntity interface{}) error {
req, err := c.prepareRequestWithBody("POST", path, headers, inEntity)
if err != nil {
return fmt.Errorf("httpclient POST: %v", err)
}
return c.doRequestWithRetry(req, outEntity)
}
func (c *Client) prepareRequestNoBody(method string, path string, headers map[string]string) (*http.Request, error) {
return c.makeRequest(method, path, headers, nil)
}
func (c *Client) prepareRequestWithBody(method string, path string, headers map[string]string, entity interface{}) (*http.Request, error) {
encoded, err := json.Marshal(entity)
if err != nil {
return nil, fmt.Errorf("httpclient get: %v", err)
}
return c.makeRequest(method, path, headers, bytes.NewBuffer(encoded))
}
func (c *Client) makeURL(path string) (string, error) {
host := c.Host
if !strings.HasPrefix(c.Host, "http") {
host = strings.Join([]string{"http://", c.Host}, "")
}
u, err := url.Parse(host)
if err != nil {
return "", fmt.Errorf("Unable to parse hostname (%v): %v", c.Host, err)
}
parsedPath, err := url.Parse(path)
if err != nil {
return "", fmt.Errorf("Unable to parse path (%v): %v", path, err)
}
u.Path = parsedPath.Path
u.RawQuery = parsedPath.RawQuery
return u.String(), nil
}
func (c *Client) makeRequest(method, path string, headers map[string]string, body io.Reader) (*http.Request, error) {
u, err := c.makeURL(path)
req, err := http.NewRequest(method, u, body)
if headers != nil {
for k, v := range headers {
req.Header.Set(k, v)
}
}
if err != nil {
return nil, fmt.Errorf("could not create request: %v", err)
}
return req, nil
}
func (c *Client) doRequestWithRetry(req *http.Request, entity interface{}) error {
if c.Executor == nil {
c.Executor = &defaultExecutor{}
}
err := c.retryRequest(3*time.Second, func() error {
return c.Executor.Do(req, c.Timeout, entity)
})
return err
}
type httpreqfunc func() error
func (c *Client) retryRequest(sleepTime time.Duration, fn httpreqfunc) error {
err := fn()
if err != nil {
_, isRetryable := err.(RetryableError)
if !isRetryable {
return err
}
toSleep := sleepTime
for retries := 0; retries < c.RetryCount; retries++ {
time.Sleep(toSleep)
toSleep = toSleep * 2
err := fn()
_, isRetryable := err.(RetryableError)
if err == nil {
return nil
} else if !isRetryable {
return err
}
}
}
return err
}
================================================
FILE: clients/httpclient/client_test.go
================================================
package httpclient
import (
"encoding/json"
"errors"
"fmt"
"net/http"
"net/http/httptest"
"testing"
"time"
)
type Cupcake struct {
Flavour string
Sprinkles bool
}
const cupcakeResponse = `{"flavour": "vomit", "sprinkles": true}`
type MockExecutor struct {
TryCount int // keep track of how many times 'Do' got called
}
func (me *MockExecutor) Do(req *http.Request, timeout time.Duration, entity interface{}) error {
me.TryCount += 1
if req.URL.Path == "/" {
return HttpRetryableError{errors.New("bork")}
} else {
return errors.New("not found yo")
}
}
func TestClientRetry(t *testing.T) {
me := &MockExecutor{}
retryCount := 2
client := &Client{
Host: "nope",
Timeout: 1 * time.Second,
RetryCount: retryCount,
Executor: me,
}
client.Get("/", nil, &Cupcake{})
if me.TryCount != retryCount+1 {
t.Errorf("Expected to try request [%v] times but got [%v]", retryCount+1, me.TryCount)
}
me.TryCount = 0
client.Get("/404", nil, &Cupcake{})
if me.TryCount != 1 {
t.Errorf("Expected to try request [%v] times but got [%v]", 1, me.TryCount)
}
}
func TestClientDo(t *testing.T) {
testServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.Method {
case "GET", "DELETE":
if len(r.URL.RawQuery) > 0 {
fmt.Fprintf(w, `{"flavour":"vague","sprinkles":false}`)
} else {
fmt.Fprintf(w, cupcakeResponse)
}
case "PUT", "POST":
content := r.Header.Get("Content-Type")
if content != "application/json" {
t.Errorf("Expected Content-Type to eq %s got %s", "application/json", content)
}
c := Cupcake{}
err := json.NewDecoder(r.Body).Decode(&c)
if err != nil {
t.Errorf("Expected body to deserialize but got error %s", err.Error())
}
fmt.Fprintf(w, cupcakeResponse)
}
}))
cupcake := Cupcake{}
client := &Client{
Host: testServer.URL,
Timeout: 1 * time.Second,
RetryCount: 1,
}
var err error
var headers = map[string]string{
"Content-Type": "application/json",
}
err = client.Get("/", nil, &cupcake)
if err != nil {
t.Errorf("Expected err to be nil got %s", err.Error())
}
if cupcake.Flavour != "vomit" {
t.Errorf("Expected flavour to be 'vomit', got: %v", cupcake.Flavour)
}
if !cupcake.Sprinkles {
t.Errorf("Expected sprinkles to be true, got: %v", cupcake.Sprinkles)
}
cupcake = Cupcake{}
err = client.Get("/?some_rando_param=thing", nil, &cupcake)
if err != nil {
t.Errorf("Expected err to be nil got %s", err.Error())
}
if cupcake.Flavour != "vague" {
t.Errorf("Expected flavour to be 'vague', got: %v", cupcake.Flavour)
}
if cupcake.Sprinkles {
t.Errorf("Expected sprinkles to be false, got: %v", cupcake.Sprinkles)
}
cupcake = Cupcake{}
err = client.Put("/", headers, &Cupcake{"vomit", true}, &cupcake)
if err != nil {
t.Errorf("Expected err to be nil got %s", err.Error())
}
if cupcake.Flavour != "vomit" {
t.Errorf("Expected flavour to be 'vomit', got: %v", cupcake.Flavour)
}
if !cupcake.Sprinkles {
t.Errorf("Expected sprinkles to be true, got: %v", cupcake.Sprinkles)
}
cupcake = Cupcake{}
err = client.Post("/", headers, &Cupcake{"vomit", true}, &cupcake)
if err != nil {
t.Errorf("Expected err to be nil got %s", err.Error())
}
if cupcake.Flavour != "vomit" {
t.Errorf("Expected flavour to be 'vomit', got: %v", cupcake.Flavour)
}
if !cupcake.Sprinkles {
t.Errorf("Expected sprinkles to be true, got: %v", cupcake.Sprinkles)
}
cupcake = Cupcake{}
err = client.Delete("/", nil, &cupcake)
if err != nil {
t.Errorf("Expected err to be nil got %s", err.Error())
}
}
================================================
FILE: clients/logs/eks_cloudwatch_logs_client.go
================================================
package logs
import (
"encoding/json"
"fmt"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/cloudwatchlogs"
"github.com/pkg/errors"
"github.com/stitchfix/flotilla-os/config"
"github.com/stitchfix/flotilla-os/exceptions"
"github.com/stitchfix/flotilla-os/state"
"log"
"net/http"
"os"
"sort"
"strings"
awstrace "gopkg.in/DataDog/dd-trace-go.v1/contrib/aws/aws-sdk-go/aws"
)
//
// EKSCloudWatchLogsClient corresponds with the aws logs driver
// for ECS and returns logs for runs
//
type EKSCloudWatchLogsClient struct {
logRetentionInDays int64
logNamespace string
logsClient logsClient
logger *log.Logger
}
type EKSCloudWatchLog struct {
Log string `json:"log"`
}
//
// Name returns the name of the logs client
//
func (lc *EKSCloudWatchLogsClient) Name() string {
return "eks-cloudwatch"
}
//
// Initialize sets up the EKSCloudWatchLogsClient
//
func (lc *EKSCloudWatchLogsClient) Initialize(conf config.Config) error {
//confLogOptions := conf.GetStringMapString("eks_log_driver_options")
awsRegion := conf.GetString("eks_log_driver_options_awslogs_region")
if len(awsRegion) == 0 {
awsRegion = conf.GetString("aws_default_region")
}
if len(awsRegion) == 0 {
return errors.Errorf(
"EKSCloudWatchLogsClient needs one of [eks.log.driver.options.awslogs-region] or [aws_default_region] set in config")
}
//
// log.namespace in conf takes precedence over log.driver.options.awslogs-group
//
lc.logNamespace = conf.GetString("eks_log_namespace")
if len(lc.logNamespace) == 0 {
return errors.Errorf(
"EKSCloudWatchLogsClient needs one of [eks.log.driver.options.awslogs-group] or [eks.log.namespace] set in config")
}
lc.logRetentionInDays = int64(conf.GetInt("eks_log_retention_days"))
if lc.logRetentionInDays == 0 {
lc.logRetentionInDays = int64(30)
}
flotillaMode := conf.GetString("flotilla_mode")
if flotillaMode != "test" {
sess := awstrace.WrapSession(session.Must(session.NewSession(&aws.Config{
Region: aws.String(awsRegion)})))
lc.logsClient = cloudwatchlogs.New(sess)
}
lc.logger = log.New(os.Stderr, "[cloudwatchlogs] ",
log.Ldate|log.Ltime|log.Lshortfile)
return lc.createNamespaceIfNotExists()
}
//
// Logs returns all logs from the log stream identified by handle since lastSeen
//
func (lc *EKSCloudWatchLogsClient) Logs(executable state.Executable, run state.Run, lastSeen *string, role *string, facility *string) (string, *string, error) {
startFromHead := true
//Pod isn't there yet - dont return a 404
if run.PodName == nil {
return "", nil, nil
}
handle := lc.toStreamName(run)
args := &cloudwatchlogs.GetLogEventsInput{
LogGroupName: &lc.logNamespace,
LogStreamName: &handle,
StartFromHead: &startFromHead,
}
if lastSeen != nil && len(*lastSeen) > 0 {
args.NextToken = lastSeen
}
result, err := lc.logsClient.GetLogEvents(args)
if err != nil {
if aerr, ok := err.(awserr.Error); ok {
if aerr.Code() == cloudwatchlogs.ErrCodeResourceNotFoundException {
return "", nil, exceptions.MissingResource{err.Error()}
} else if request.IsErrorThrottle(err) {
lc.logger.Printf(
"thottled getting logs; executable_id: %v, run_id: %s, error: %+v\n",
executable.GetExecutableID(), run.RunID, err)
return "", lastSeen, nil
}
}
return "", nil, errors.Wrap(err, "problem getting logs")
}
if len(result.Events) == 0 {
return "", result.NextForwardToken, nil
}
message := lc.logsToMessage(result.Events)
return message, result.NextForwardToken, nil
}
// This method doesn't return log string, it is a placeholder only.
func (lc *EKSCloudWatchLogsClient) LogsText(executable state.Executable, run state.Run, w http.ResponseWriter) error {
return errors.Errorf("EKSCloudWatchLogsClient does not support LogsText method.")
}
// Generate stream name
func (lc *EKSCloudWatchLogsClient) toStreamName(run state.Run) string {
return fmt.Sprintf("%s", *run.PodName)
}
// Convert Cloudwatch logs to strings
func (lc *EKSCloudWatchLogsClient) logsToMessage(events []*cloudwatchlogs.OutputLogEvent) string {
sort.Sort(byTimestamp(events))
messages := make([]string, len(events))
for i, event := range events {
var l EKSCloudWatchLog
err := json.Unmarshal([]byte(*event.Message), &l)
if err != nil {
messages[i] = *event.Message
}
messages[i] = l.Log
}
return strings.Join(messages, "")
}
func (lc *EKSCloudWatchLogsClient) createNamespaceIfNotExists() error {
exists, err := lc.namespaceExists()
if err != nil {
return errors.Wrapf(err, "problem checking if log namespace [%s] exists", lc.logNamespace)
}
if !exists {
return lc.createNamespace()
}
return nil
}
// Check for the existence of a namespace.
func (lc *EKSCloudWatchLogsClient) namespaceExists() (bool, error) {
result, err := lc.logsClient.DescribeLogGroups(&cloudwatchlogs.DescribeLogGroupsInput{
LogGroupNamePrefix: &lc.logNamespace,
})
if err != nil {
return false, errors.Wrapf(err, "problem describing log groups with prefix [%s]", lc.logNamespace)
}
if len(result.LogGroups) == 0 {
return false, nil
}
for _, group := range result.LogGroups {
if *group.LogGroupName == lc.logNamespace {
return true, nil
}
}
return false, nil
}
// Creates namespace is not present.
func (lc *EKSCloudWatchLogsClient) createNamespace() error {
_, err := lc.logsClient.CreateLogGroup(&cloudwatchlogs.CreateLogGroupInput{
LogGroupName: &lc.logNamespace,
})
if err != nil {
return errors.Wrapf(err, "problem creating log group with log group name [%s]", lc.logNamespace)
}
_, err = lc.logsClient.PutRetentionPolicy(&cloudwatchlogs.PutRetentionPolicyInput{
LogGroupName: &lc.logNamespace,
RetentionInDays: &lc.logRetentionInDays,
})
if err != nil {
return errors.Wrapf(err, "problem setting log group retention policy for log group name [%s]", lc.logNamespace)
}
return nil
}
================================================
FILE: clients/logs/eks_s3_logs_client.go
================================================
package logs
import (
"bufio"
"bytes"
"compress/gzip"
"context"
"encoding/json"
"fmt"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/pkg/errors"
"github.com/stitchfix/flotilla-os/config"
"github.com/stitchfix/flotilla-os/state"
awstrace "gopkg.in/DataDog/dd-trace-go.v1/contrib/aws/aws-sdk-go/aws"
"io"
"log"
"net/http"
"os"
"strconv"
"strings"
"time"
)
// EKSS3LogsClient corresponds with the aws logs driver
// for ECS and returns logs for runs
type EKSS3LogsClient struct {
logRetentionInDays int64
logNamespace string
s3Client *s3.S3
s3Bucket string
s3BucketRootDir string
logger *log.Logger
emrS3LogsBucket string
emrS3LogsBasePath string
}
type s3Log struct {
Log string `json:"log"`
Stream string `json:"stream"`
Time time.Time `json:"time"`
}
// Name returns the name of the logs client
func (lc *EKSS3LogsClient) Name() string {
return "eks-s3"
}
// Initialize sets up the EKSS3LogsClient
func (lc *EKSS3LogsClient) Initialize(conf config.Config) error {
//confLogOptions := conf.GetStringMapString("eks_log_driver_options")
awsRegion := conf.GetString("eks_log_driver_options_awslogs_region")
if len(awsRegion) == 0 {
awsRegion = conf.GetString("aws_default_region")
}
if len(awsRegion) == 0 {
return errors.Errorf(
"EKSS3LogsClient needs one of [eks.log.driver.options.awslogs-region] or [aws_default_region] set in config")
}
flotillaMode := conf.GetString("flotilla_mode")
if flotillaMode != "test" {
sess := awstrace.WrapSession(session.Must(session.NewSession(&aws.Config{
Region: aws.String(awsRegion)})))
sess = awstrace.WrapSession(sess)
lc.s3Client = s3.New(sess, aws.NewConfig().WithRegion(awsRegion))
}
lc.emrS3LogsBucket = conf.GetString("emr_log_bucket")
lc.emrS3LogsBasePath = conf.GetString("emr_log_base_path")
s3BucketName := conf.GetString("eks_log_driver_options_s3_bucket_name")
if len(s3BucketName) == 0 {
return errors.Errorf(
"EKSS3LogsClient needs [eks_log_driver_options_s3_bucket_name] set in config")
}
lc.s3Bucket = s3BucketName
s3BucketRootDir := conf.GetString("eks_log_driver_options_s3_bucket_root_dir")
if len(s3BucketRootDir) == 0 {
return errors.Errorf(
"EKSS3LogsClient needs [eks.log.driver.options.s3_bucket_root_dir] set in config")
}
lc.s3BucketRootDir = s3BucketRootDir
lc.logger = log.New(os.Stderr, "[s3logs] ",
log.Ldate|log.Ltime|log.Lshortfile)
return nil
}
func (lc *EKSS3LogsClient) emrLogsToMessageString(run state.Run, lastSeen *string, role *string, facility *string) (string, *string, error) {
s3DirName, err := lc.emrDriverLogsPath(run)
if err != nil {
return "", aws.String(""), errors.Errorf("No logs")
}
params := &s3.ListObjectsV2Input{
Bucket: aws.String(lc.emrS3LogsBucket),
Prefix: aws.String(s3DirName),
MaxKeys: aws.Int64(1000),
}
pageNum := 0
lastModified := &time.Time{}
var key *string
err = lc.s3Client.ListObjectsV2Pages(params,
func(result *s3.ListObjectsV2Output, lastPage bool) bool {
pageNum++
if result != nil {
for _, content := range result.Contents {
if strings.Contains(*content.Key, *role) && strings.Contains(*content.Key, *facility) && lastModified.Before(*content.LastModified) {
if content != nil && *content.Size < int64(10000000) {
key = content.Key
lastModified = content.LastModified
}
}
}
}
if lastPage {
return false
}
return pageNum <= 10
})
if key == nil {
lc.logger.Println(fmt.Sprintf("run=%s emr logging key not found for role=%s facility=%s", run.RunID, *role, *facility))
return "", aws.String(""), errors.Errorf("No driver logs found")
}
startPosition := int64(0)
if lastSeen != nil {
parsed, err := strconv.ParseInt(*lastSeen, 10, 64)
if err == nil {
startPosition = parsed
}
}
s3Obj, err := lc.s3Client.GetObjectWithContext(
context.Background(),
&s3.GetObjectInput{
Bucket: aws.String(lc.emrS3LogsBucket),
Key: aws.String(*key),
}, func(r *request.Request) {
// Otherwise we get an unzipped response.
r.HTTPRequest.Header.Add("Accept-Encoding", "gzip")
})
if s3Obj != nil && err == nil {
if s3Obj.ContentLength != nil && *s3Obj.ContentLength > int64(10000000) {
return "", aws.String(""), errors.Errorf("Logs > 10MB, will not display.")
}
defer s3Obj.Body.Close()
gr, err := gzip.NewReader(s3Obj.Body)
if err != nil {
return "", aws.String(""), err
}
defer gr.Close()
reader := bufio.NewReader(gr)
var b0 bytes.Buffer
counter := int64(0)
for {
line, err := reader.ReadBytes('\n')
if err != nil {
if err == io.EOF {
err = nil
return b0.String(), aws.String(fmt.Sprintf("%d", counter)), nil
}
} else {
if counter >= startPosition {
b0.Write(line)
}
counter = counter + 1
}
}
}
return "", aws.String(""), errors.Errorf("No driver logs found")
}
func (lc *EKSS3LogsClient) emrDriverLogsPath(run state.Run) (string, error) {
if run.SparkExtension.EMRJobId != nil &&
run.SparkExtension.VirtualClusterId != nil {
return fmt.Sprintf("%s/%s/jobs/%s/",
lc.emrS3LogsBasePath,
*run.SparkExtension.VirtualClusterId,
*run.SparkExtension.EMRJobId,
), nil
}
return "", errors.New("couldn't construct s3 path.")
}
func (lc *EKSS3LogsClient) Logs(executable state.Executable, run state.Run, lastSeen *string, role *string, facility *string) (string, *string, error) {
if *run.Engine == state.EKSSparkEngine {
return lc.emrLogsToMessageString(run, lastSeen, role, facility)
}
result, err := lc.getS3Object(run)
startPosition := int64(0)
if lastSeen != nil {
parsed, err := strconv.ParseInt(*lastSeen, 10, 64)
if err == nil {
startPosition = parsed
}
}
if result != nil && err == nil {
acc, position, err := lc.logsToMessageString(result, startPosition)
newLastSeen := fmt.Sprintf("%d", position)
return acc, &newLastSeen, err
}
return "", aws.String(""), errors.Errorf("No logs.")
}
// Logs returns all logs from the log stream identified by handle since lastSeen
func (lc *EKSS3LogsClient) LogsText(executable state.Executable, run state.Run, w http.ResponseWriter) error {
if run.Engine == nil || *run.Engine == state.EKSEngine {
result, err := lc.getS3Object(run)
if err != nil {
return err
} else if result != nil {
return lc.logsToMessage(result, w)
}
}
if *run.Engine == state.EKSSparkEngine {
return lc.logsEMR(w)
}
return nil
}
// Fetch S3Object associated with the pod's log.
func (lc *EKSS3LogsClient) getS3Object(run state.Run) (*s3.GetObjectOutput, error) {
//Pod isn't there yet - dont return a 404
//if run.PodName == nil {
// return nil, errors.New("no pod associated with the run.")
//}
s3DirName := lc.toS3DirName(run)
// Get list of S3 objects in the run_id folder.
result, err := lc.s3Client.ListObjects(&s3.ListObjectsInput{
Bucket: aws.String(lc.s3Bucket),
Prefix: aws.String(s3DirName),
})
if err != nil {
return nil, errors.Wrap(err, "problem getting logs")
}
if result == nil || result.Contents == nil || len(result.Contents) == 0 {
return nil, errors.New("no s3 files associated with the run.")
}
var key *string
lastModified := &time.Time{}
//Find latest log file (could have multiple log files per pod - due to pod retries)
for _, content := range result.Contents {
if strings.Contains(*content.Key, run.RunID) && lastModified.Before(*content.LastModified) {
if content != nil && *content.Size < int64(10000000) {
key = content.Key
lastModified = content.LastModified
}
}
}
if key != nil {
return lc.getS3Key(key)
} else {
return nil, errors.New("no s3 files associated with the run.")
}
}
func (lc *EKSS3LogsClient) getS3Key(s3Key *string) (*s3.GetObjectOutput, error) {
result, err := lc.s3Client.GetObject(&s3.GetObjectInput{
Bucket: aws.String(lc.s3Bucket),
Key: aws.String(*s3Key),
})
if err != nil {
return nil, err
}
return result, nil
}
// Formulate dir name on S3.
func (lc *EKSS3LogsClient) toS3DirName(run state.Run) string {
return fmt.Sprintf("%s/%s", lc.s3BucketRootDir, run.RunID)
}
// Converts log messages from S3 to strings - returns the contents of the entire file.
func (lc *EKSS3LogsClient) logsToMessage(result *s3.GetObjectOutput, w http.ResponseWriter) error {
reader := bufio.NewReader(result.Body)
for {
line, err := reader.ReadBytes('\n')
if err != nil {
if err == io.EOF {
err = nil
}
return err
} else {
var parsedLine s3Log
parsedLine, err := parseLines(line)
if err != nil {
return err
}
_, err = io.WriteString(w, parsedLine.Log)
if err != nil {
return err
}
}
}
}
func (lc *EKSS3LogsClient) logsEMR(w http.ResponseWriter) error {
_, _ = io.WriteString(w, "todo!!!")
return nil
}
// Converts log messages from S3 to strings, takes a starting offset.
func (lc *EKSS3LogsClient) logsToMessageString(result *s3.GetObjectOutput, startingPosition int64) (string, int64, error) {
acc := ""
currentPosition := int64(0)
// if less than/equal to 0, read entire log.
if startingPosition <= 0 {
startingPosition = currentPosition
}
// No S3 file or object, return "", 0, err
if result == nil {
return acc, startingPosition, errors.New("s3 object not present.")
}
reader := bufio.NewReader(result.Body)
// Reading until startingPosition and discard unneeded lines.
for currentPosition < startingPosition {
currentPosition = currentPosition + 1
_, err := reader.ReadBytes('\n')
if err != nil {
if err == io.EOF {
err = nil
}
return acc, startingPosition, err
}
}
// Read upto MaxLogLines
for currentPosition <= startingPosition+state.MaxLogLines {
currentPosition = currentPosition + 1
line, err := reader.ReadBytes('\n')
if err != nil {
if err == io.EOF {
err = nil
}
return acc, currentPosition, err
} else {
parsedLine, err := parseLines(line)
if err == nil {
acc = fmt.Sprintf("%s%s", acc, parsedLine.Log)
}
}
}
_ = result.Body.Close()
return acc, currentPosition, nil
}
func parseLines(input []byte) (s3Log, error) {
//handling both dockerengine and containterd log formats
//TODO I don't love this - clean up post migration
var parsedInput s3Log
err := json.Unmarshal(input, &parsedInput)
if err != nil {
splitLines := strings.Split(string(input), " ")
if len(splitLines) > 0 {
layout := "2006-01-02T15:04:05.999999999Z"
timestamp, err := time.Parse(layout, splitLines[0])
if err != nil {
return parsedInput, err
}
parsedInput.Time = timestamp
parsedInput.Stream = splitLines[1]
parsedInput.Log = strings.Join(splitLines[3:], " ")
}
}
return parsedInput, nil
}
================================================
FILE: clients/logs/logs.go
================================================
package logs
import (
"fmt"
"github.com/aws/aws-sdk-go/service/cloudwatchlogs"
"github.com/pkg/errors"
"github.com/stitchfix/flotilla-os/config"
flotillaLog "github.com/stitchfix/flotilla-os/log"
"github.com/stitchfix/flotilla-os/state"
"net/http"
)
//
// Client returns logs for a Run
//
type Client interface {
Name() string
Initialize(config config.Config) error
Logs(executable state.Executable, run state.Run, lastSeen *string, role *string, facility *string) (string, *string, error)
LogsText(executable state.Executable, run state.Run, w http.ResponseWriter) error
}
type logsClient interface {
DescribeLogGroups(input *cloudwatchlogs.DescribeLogGroupsInput) (*cloudwatchlogs.DescribeLogGroupsOutput, error)
CreateLogGroup(input *cloudwatchlogs.CreateLogGroupInput) (*cloudwatchlogs.CreateLogGroupOutput, error)
PutRetentionPolicy(input *cloudwatchlogs.PutRetentionPolicyInput) (*cloudwatchlogs.PutRetentionPolicyOutput, error)
GetLogEvents(input *cloudwatchlogs.GetLogEventsInput) (*cloudwatchlogs.GetLogEventsOutput, error)
}
type byTimestamp []*cloudwatchlogs.OutputLogEvent
func (events byTimestamp) Len() int { return len(events) }
func (events byTimestamp) Swap(i, j int) { events[i], events[j] = events[j], events[i] }
func (events byTimestamp) Less(i, j int) bool { return *(events[i].Timestamp) < *(events[j].Timestamp) }
//
// NewLogsClient creates and initializes a run logs client
//
func NewLogsClient(conf config.Config, logger flotillaLog.Logger, name string) (Client, error) {
_ = logger.Log("level", "info", "message", "Initializing logs client", "client", name)
switch name {
case "eks":
// awslogs as an ecs log driver sends logs to AWS CloudWatch Logs service
ekscw := &EKSS3LogsClient{}
if err := ekscw.Initialize(conf); err != nil {
return nil, errors.Wrap(err, "problem initializing EKSCloudWatchLogsClient")
}
return ekscw, nil
default:
return nil, fmt.Errorf("No Client named [%s] was found", name)
}
}
================================================
FILE: clients/metrics/datadog_metrics_client.go
================================================
package metrics
import (
"fmt"
"github.com/DataDog/datadog-go/v5/statsd"
"github.com/stitchfix/flotilla-os/config"
"os"
"strings"
"time"
)
// Client accepts statsd metrics
type DatadogStatsdMetricsClient struct {
client *statsd.Client
}
// Initialize the client. Assumes the following keys are passed in:
// *metrics.dogstatsd.address* -- localhost:8125
// *metrics.dogstatsd.namespace* -- fixed key you want to prefix to all the metrics
func (dd *DatadogStatsdMetricsClient) Init(conf config.Config) error {
host := os.Getenv("DD_AGENT_HOST")
var addr string
// If the host contains a colon and does not contain a square bracket, then the address is ipv6
if strings.Contains(host, ":") && !strings.Contains(host, "[") {
addr = fmt.Sprintf("[%s]:8125", host)
} else {
addr = fmt.Sprintf("%s:8125", host)
}
client, err := statsd.New(addr, statsd.WithNamespace(conf.GetString("metrics_dogstatsd_namespace")))
if err != nil {
return err
}
dd.client = client
return nil
}
// Decrement metric value, tags associated with the metric, and rate corresponds to the value
func (dd *DatadogStatsdMetricsClient) Decrement(name Metric, tags []string, rate float64) error {
return dd.client.Decr(string(name), tags, rate)
}
// Increment metric value, tags associated with the metric, and rate corresponds to the value
func (dd *DatadogStatsdMetricsClient) Increment(name Metric, tags []string, rate float64) error {
return dd.client.Incr(string(name), tags, rate)
}
// Histogram tracks the statistical distribution of a set of values
func (dd *DatadogStatsdMetricsClient) Histogram(name Metric, value float64, tags []string, rate float64) error {
return dd.client.Histogram(string(name), value, tags, rate)
}
// Distribution tracks the statistical distribution of a set of values
func (dd *DatadogStatsdMetricsClient) Distribution(name Metric, value float64, tags []string, rate float64) error {
return dd.client.Distribution(string(name), value, tags, rate)
}
// Timing sends timing information, it is an alias for TimeInMilliseconds
func (dd *DatadogStatsdMetricsClient) Timing(name Metric, value time.Duration, tags []string, rate float64) error {
return dd.client.Timing(string(name), value, tags, rate)
}
// Set counts the number of unique elements in a group
func (dd *DatadogStatsdMetricsClient) Set(name Metric, value string, tags []string, rate float64) error {
return dd.client.Set(string(name), value, tags, rate)
}
// NewEvent creates a new event with the given title and text.
func (dd *DatadogStatsdMetricsClient) Event(e event) error {
se := statsd.NewEvent(e.Title, e.Text)
se.Tags = e.Tags
return dd.client.Event(se)
}
================================================
FILE: clients/metrics/metrics.go
================================================
package metrics
import (
"fmt"
"sync"
"time"
"github.com/pkg/errors"
"github.com/stitchfix/flotilla-os/config"
)
type Metric string
const (
// Metric associated to submission of jobs to EKS
EngineEKSExecute Metric = "engine.eks.execute"
// Metric associated to submission of jobs to SQS queue, before EKS submission.
EngineEKSEnqueue Metric = "engine.eks.enqueue"
// Metric associated to submission of jobs to EMR
EngineEMRExecute Metric = "engine.emr.execute"
// Metric associated to submission of jobs to SQS queue, before EMR submission.
EngineEMREnqueue Metric = "engine.emr.enqueue"
// Metric associated to termination of jobs via the API.
EngineEKSTerminate Metric = "engine.eks.terminate"
// Metric associated to termination of jobs via the API.
EngineEMRTerminate Metric = "engine.emr.terminate"
// Metric associated to termination of pods hopping between hosts.
EngineEKSRunPodnameChange Metric = "engine.eks.run_podname_changed"
// Metric associated to pod events where there was a Cluster Autoscale event.
EngineEKSNodeTriggeredScaledUp Metric = "engine.eks.triggered_scale_up"
// Timing for status worker processEKSRun
StatusWorkerProcessEKSRun Metric = "status_worker.timing.process_eks_run"
// Timing for acquire lock
StatusWorkerAcquireLock Metric = "status_worker.timing.acquire_lock"
// Timing for fetch_pod_metrics
StatusWorkerFetchPodMetrics Metric = "status_worker.timing.fetch_pod_metrics"
// Timing for fetch_update_status
StatusWorkerFetchUpdateStatus Metric = "status_worker.timing.fetch_update_status"
// Metric for locked runs
StatusWorkerLockedRuns Metric = "status_worker.locked_runs"
// Timing for fetch metrics
StatusWorkerFetchMetrics Metric = "status_worker.fetch_metrics"
// Timing for get pod list
StatusWorkerGetPodList Metric = "status_worker.get_pod_list"
// Timing for get events
StatusWorkerGetEvents Metric = "status_worker.get_events"
// Timing for get job
StatusWorkerGetJob Metric = "status_worker.get_job"
// Engine update run
EngineUpdateRun Metric = "engine.update_run"
// ARA metrics - tracking Auto Resource Adjustment behavior
EngineEKSARAEstimationAttempted Metric = "engine.eks.ara.estimation_attempted"
EngineEKSARAEstimationSucceeded Metric = "engine.eks.ara.estimation_succeeded"
EngineEKSARAEstimationFailed Metric = "engine.eks.ara.estimation_failed"
EngineEKSARAResourceAdjustment Metric = "engine.eks.ara.resource_adjustment"
EngineEKSARANoHistoricalData Metric = "engine.eks.ara.no_historical_data"
EngineEKSARAHitMaxMemory Metric = "engine.eks.ara.hit_max_memory"
EngineEKSARAHitMaxCPU Metric = "engine.eks.ara.hit_max_cpu"
EngineEKSARAMemoryIncreaseRatio Metric = "engine.eks.ara.memory_increase_ratio"
EngineEKSARACPUIncreaseRatio Metric = "engine.eks.ara.cpu_increase_ratio"
EngineEKSARAFinalMemoryMB Metric = "engine.eks.ara.final_memory_mb"
EngineEKSARAFinalCPUMillicores Metric = "engine.eks.ara.final_cpu_millicores"
EngineEKSARADefaultMemory Metric = "engine.eks.ara.default_memory"
EngineEKSARAARAMemory Metric = "engine.eks.ara.ara_memory"
EngineEKSARADefaultCPU Metric = "engine.eks.ara.default_cpu"
EngineEKSARAARACPU Metric = "engine.eks.ara.ara_cpu"
EngineEKSARAMemoryIncrease Metric = "engine.eks.ara.memory_increase"
EngineEKSARACPUIncrease Metric = "engine.eks.ara.cpu_increase"
EngineEKSARANullCommandHash Metric = "engine.eks.ara.null_command_hash"
)
type MetricTag string
const (
// Metric tag for job success.
StatusSuccess MetricTag = "status:success"
// Metric tag for job failure.
StatusFailure MetricTag = "status:failure"
)
type Client interface {
Init(conf config.Config) error
Decrement(name Metric, tags []string, rate float64) error
Increment(name Metric, tags []string, rate float64) error
Histogram(name Metric, value float64, tags []string, rate float64) error
Distribution(name Metric, value float64, tags []string, rate float64) error
Set(name Metric, value string, tags []string, rate float64) error
Event(evt event) error
Timing(name Metric, value time.Duration, tags []string, rate float64) error
}
type event struct {
Title string
Text string
Tags []string
}
var once sync.Once
var instance Client
// Instantiating the Metrics Client.
func InstantiateClient(conf config.Config) error {
// Return an error if `metrics_client` isn't set in config.
if !conf.IsSet("metrics_client") {
return fmt.Errorf("`metrics_client` not set in config, unable to instantiate metrics client")
}
var err error = nil
name := conf.GetString("metrics_client")
once.Do(func() {
switch name {
case "dogstatsd":
instance = &DatadogStatsdMetricsClient{}
if err = instance.Init(conf); err != nil {
err = errors.Errorf("Unable to initialize dogstatsd client.")
instance = nil
break
}
default:
err = fmt.Errorf("no client named [%s] was found", name)
}
})
return err
}
// Decr is just Count of -1
func Decrement(name Metric, tags []string, rate float64) error {
if instance != nil {
return instance.Decrement(name, tags, rate)
}
return errors.Errorf("MetricsClient instance is nil, unable to send Decrement metric.")
}
// Incr is just Count of -1
func Increment(name Metric, tags []string, rate float64) error {
if instance != nil {
return instance.Increment(name, tags, rate)
}
return errors.Errorf("MetricsClient instance is nil, unable to send Increment metric.")
}
// Histogram tracks the statistical distribution of a set of values
func Histogram(name Metric, value float64, tags []string, rate float64) error {
if instance != nil {
return instance.Histogram(name, value, tags, rate)
}
return errors.Errorf("MetricsClient instance is nil, unable to send Histogram metric.")
}
// Distribution tracks the statistical distribution of a set of values
func Distribution(name Metric, value float64, tags []string, rate float64) error {
if instance != nil {
return instance.Distribution(name, value, tags, rate)
}
return errors.Errorf("MetricsClient instance is nil, unable to send Distribution metric.")
}
// Set counts the number of unique elements in a group
func Set(name Metric, value string, tags []string, rate float64) error {
if instance != nil {
return instance.Set(name, value, tags, rate)
}
return errors.Errorf("MetricsClient instance is nil, unable to send Set metric.")
}
// NewEvent creates a new event with the given title and text.
func Event(title string, text string, tags []string) error {
if instance != nil {
return instance.Event(event{
Title: title,
Text: text,
Tags: tags,
})
}
return errors.Errorf("MetricsClient instance is nil, unable to send Event metric.")
}
// Timing sends timing information, it is an alias for TimeInMilliseconds
func Timing(name Metric, value time.Duration, tags []string, rate float64) error {
if instance != nil {
return instance.Timing(name, value, tags, rate)
}
return errors.Errorf("MetricsClient instance is nil, unable to send Event metric.")
}
================================================
FILE: clients/middleware/client.go
================================================
package middleware
import (
"github.com/stitchfix/flotilla-os/state"
"net/http"
)
type Client interface {
AnnotateLaunchRequest(headers *http.Header, lr *state.LaunchRequestV2) error
}
type middlewareClient struct{}
func NewClient() (Client, error) {
return &middlewareClient{}, nil
}
func (mwC middlewareClient) AnnotateLaunchRequest(headers *http.Header, lr *state.LaunchRequestV2) error {
return nil
}
================================================
FILE: conf/config.yml
================================================
aws_default_region: us-east-1
cluster_client: eks
create_database_schema: true
database_url: postgresql://flotilla:flotilla@localhost/flotilla?sslmode=disable
eks_clusters: 'clusta, cupcake'
eks_cluster_default: 'clusta'
eks_gpu_cluster_default: 'clusta'
eks_tier_default: '4'
eks_log_driver_name: awslogs
eks_log_driver_options_awslogs-group: flotilla-eks-namespace
eks_log_driver_options_awslogs-region: us-east-1
eks_log_namespace: flotilla-eks-namespace
eks_log_retention_days: 90
enabled_workers:
- retry
- submit
execution_engine: eks
flotilla_mode: test
http_server_cors_allowed_origins:
- http://localhost:3001
http_server_listen_address: :3000
http_server_read_timeout_seconds: 5
http_server_write_timeout_seconds: 10
logs_client: cloudwatch
metrics_client: dogstatsd
metrics_dogstatsd_address: 127.0.0.1:8125
metrics_dogstatsd_namespace: my.flotilla.namespace
metrics_dogstatsd_tags:
- test
owner_id_var: FLOTILLA_RUN_OWNER_ID
queue_manager: sqs
queue_namespace: dev-flotilla
queue_process_time: 45
queue_retention_seconds: 604800
queue_status: flotilla-status-updates-dev
queue_status_rule: flotilla-task-status
readonly_database_url: postgresql://flotilla:flotilla@localhost/flotilla?sslmode=disable
================================================
FILE: config/config.go
================================================
package config
import (
"github.com/pkg/errors"
"github.com/spf13/viper"
"strings"
)
//
// Config interface to wrap external configuration object
//
type Config interface {
GetString(key string) string
GetStringSlice(key string) []string
GetStringMapString(key string) map[string]string
GetInt(key string) int
GetBool(key string) bool
GetFloat64(key string) float64
IsSet(key string) bool
}
//
// NewConfig initializes a configuration object
// - if confDir is non-nil searches there and loads a "config.yml"
// - sets configuration to read from environment variables automatically
//
func NewConfig(confDir *string) (Config, error) {
v := viper.New()
if v == nil {
return &conf{}, errors.New("Error initializing internal config")
}
if confDir != nil {
v.SetConfigName("config")
v.SetConfigType("yaml")
v.AddConfigPath(*confDir)
if err := v.ReadInConfig(); err != nil {
return &conf{}, errors.Wrapf(err, "problem reading config from [%s]", *confDir)
}
}
v.AutomaticEnv()
v.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
return &conf{v}, nil
}
type conf struct {
v *viper.Viper
}
// GetString returns the value associated with the key as a string.
func (c *conf) GetString(key string) string {
return c.v.GetString(key)
}
// GetFloat returns the value associated with the key as a float.
func (c *conf) GetFloat64(key string) float64 {
return c.v.GetFloat64(key)
}
// GetInt returns the value associated with the key as an integer.
func (c *conf) GetInt(key string) int {
return c.v.GetInt(key)
}
// GetBool returns the value associated with the key as a boolean.
func (c *conf) GetBool(key string) bool {
return c.v.GetBool(key)
}
// GetStringMapString returns the value associated with the key as a map of strings.
func (c *conf) GetStringMapString(key string) map[string]string {
return c.v.GetStringMapString(key)
}
// GetStringSlice returns the value associated with the key as a slice of strings.
func (c *conf) GetStringSlice(key string) []string {
return c.v.GetStringSlice(key)
}
// IsSet checks to see if the key has been set in any of the data locations.
// IsSet is case-insensitive for a key.
func (c *conf) IsSet(key string) bool {
return c.v.IsSet(key)
}
================================================
FILE: config/config_test.go
================================================
package config
import (
"os"
"testing"
)
func TestNewConfig(t *testing.T) {
var c Config
c, _ = NewConfig(nil)
toSet := "sprinkles"
os.Setenv("CUPCAKE", toSet)
if c.GetString("cupcake") != toSet {
t.Errorf("Environment variables not set - expected %s but was %s", toSet, c.GetString("cupcake"))
}
confDir := "../conf"
c, _ = NewConfig(&confDir)
if !c.IsSet("queue_namespace") || c.GetString("queue_namespace") != "dev-flotilla" {
t.Errorf("Expected to read from conf dir [queue_namespace]:[dev-flotilla], was: %s",
c.GetString("queue_namespace"))
}
}
================================================
FILE: datadog-ara-dashboard-api.json
================================================
{
"title": "Flotilla ARA (Auto Resource Adjustment) Metrics",
"description": "Dashboard tracking Auto Resource Adjustment behavior for EKS and Spark jobs. Monitors resource growth patterns, over-provisioning detection, and OOM-based memory adjustments. Use the engine filter to view EKS (P99-based 1.75x/1.25x) vs Spark (OOM-based 1.25x/3.0x) jobs separately.",
"widgets": [
{
"id": 1,
"layout": {
"x": 0,
"y": 0,
"width": 47,
"height": 15
},
"definition": {
"title": "ARA Estimation Attempts vs Successes",
"title_size": "16",
"title_align": "left",
"show_legend": true,
"legend_layout": "auto",
"legend_columns": [
"avg",
"min",
"max",
"value",
"sum"
],
"type": "timeseries",
"requests": [
{
"response_format": "timeseries",
"queries": [
{
"data_source": "metrics",
"name": "query1",
"query": "sum:algo.flotilla.engine.eks.ara.estimation_attempted{$cluster,$env,$engine}.as_count()"
}
],
"style": {
"palette": "dog_classic",
"line_type": "solid",
"line_width": "normal"
},
"display_type": "bars"
},
{
"response_format": "timeseries",
"queries": [
{
"data_source": "metrics",
"name": "query1",
"query": "sum:algo.flotilla.engine.eks.ara.estimation_succeeded{$cluster,$env,$engine}.as_count()"
}
],
"style": {
"palette": "green",
"line_type": "solid",
"line_width": "normal"
},
"display_type": "bars"
},
{
"response_format": "timeseries",
"queries": [
{
"data_source": "metrics",
"name": "query1",
"query": "sum:algo.flotilla.engine.eks.ara.estimation_failed{$cluster,$env,$engine}.as_count()"
}
],
"style": {
"palette": "red",
"line_type": "solid",
"line_width": "normal"
},
"display_type": "bars"
},
{
"response_format": "timeseries",
"queries": [
{
"data_source": "metrics",
"name": "query1",
"query": "sum:algo.flotilla.engine.eks.ara.no_historical_data{$cluster,$env,$engine}.as_count()"
}
],
"style": {
"palette": "orange",
"line_type": "solid",
"line_width": "normal"
},
"display_type": "bars"
}
],
"yaxis": {
"label": "",
"scale": "linear",
"include_zero": true,
"min": "auto",
"max": "auto"
},
"markers": []
}
},
{
"id": 2,
"layout": {
"x": 48,
"y": 0,
"width": 47,
"height": 15
},
"definition": {
"title": "ARA Resource Adjustments",
"title_size": "16",
"title_align": "left",
"show_legend": true,
"legend_size": "0",
"type": "timeseries",
"requests": [
{
"response_format": "timeseries",
"queries": [
{
"data_source": "metrics",
"name": "query1",
"query": "sum:algo.flotilla.engine.eks.ara.resource_adjustment{$cluster,$env,$engine}.as_count()"
}
],
"style": {
"palette": "blue",
"line_type": "solid",
"line_width": "normal"
},
"display_type": "bars"
}
],
"yaxis": {
"label": "",
"scale": "linear",
"include_zero": true,
"min": "auto",
"max": "auto"
},
"markers": []
}
},
{
"id": 3,
"layout": {
"x": 0,
"y": 16,
"width": 47,
"height": 15
},
"definition": {
"title": "Max Resource Limits Hit (Critical)",
"title_size": "16",
"title_align": "left",
"show_legend": true,
"legend_size": "0",
"type": "timeseries",
"requests": [
{
"response_format": "timeseries",
"queries": [
{
"data_source": "metrics",
"name": "query1",
"query": "sum:algo.flotilla.engine.eks.ara.hit_max_memory{$cluster,$env,$engine}.as_count()"
}
],
"style": {
"palette": "red",
"line_type": "solid",
"line_width": "thick"
},
"display_type": "line"
},
{
"response_format": "timeseries",
"queries": [
{
"data_source": "metrics",
"name": "query1",
"query": "sum:algo.flotilla.engine.eks.ara.hit_max_cpu{$cluster,$env,$engine}.as_count()"
}
],
"style": {
"palette": "orange",
"line_type": "solid",
"line_width": "normal"
},
"display_type": "line"
}
],
"yaxis": {
"label": "",
"scale": "linear",
"include_zero": true,
"min": "auto",
"max": "auto"
},
"markers": [
{
"label": "Alert Threshold",
"value": "y = 0",
"display_type": "error dashed"
}
]
}
},
{
"id": 4,
"layout": {
"x": 48,
"y": 16,
"width": 23,
"height": 15
},
"definition": {
"title": "Success Rate",
"title_size": "16",
"title_align": "left",
"type": "query_value",
"requests": [
{
"conditional_formats": [
{
"comparator": ">=",
"value": 95,
"palette": "green_on_white"
},
{
"comparator": ">=",
"value": 80,
"palette": "yellow_on_white"
},
{
"comparator": "<",
"value": 80,
"palette": "red_on_white"
}
],
"response_format": "scalar",
"queries": [
{
"data_source": "metrics",
"name": "query1",
"query": "sum:algo.flotilla.engine.eks.ara.estimation_succeeded{$cluster,$env,$engine}.as_count()",
"aggregator": "sum"
},
{
"data_source": "metrics",
"name": "query2",
"query": "sum:algo.flotilla.engine.eks.ara.estimation_attempted{$cluster,$env,$engine}.as_count()",
"aggregator": "sum"
}
],
"formulas": [
{
"number_format": {
"unit": {
"label": "%",
"type": "custom_unit_label"
}
},
"formula": "(query1 / query2) * 100"
}
]
}
],
"autoscale": true,
"precision": 2
}
},
{
"id": 5,
"layout": {
"x": 72,
"y": 16,
"width": 23,
"height": 15
},
"definition": {
"title": "Max Memory Hits (Last Hour)",
"title_size": "16",
"title_align": "left",
"type": "query_value",
"requests": [
{
"conditional_formats": [
{
"comparator": ">",
"value": 0,
"palette": "red_on_white"
},
{
"comparator": "=",
"value": 0,
"palette": "green_on_white"
}
],
"response_format": "scalar",
"queries": [
{
"data_source": "metrics",
"name": "query1",
"query": "sum:algo.flotilla.engine.eks.ara.hit_max_memory{$cluster,$env,$engine}.as_count()",
"aggregator": "sum"
}
]
}
],
"autoscale": true,
"custom_unit": "",
"precision": 0
}
},
{
"id": 6,
"layout": {
"x": 0,
"y": 32,
"width": 31,
"height": 15
},
"definition": {
"title": "Memory Increase Ratio Distribution",
"title_size": "16",
"title_align": "left",
"show_legend": false,
"type": "heatmap",
"yaxis": {
"label": "",
"scale": "linear",
"include_zero": true,
"min": "auto",
"max": "auto"
},
"requests": [
{
"style": {
"palette": "YlOrRd"
},
"response_format": "timeseries",
"queries": [
{
"data_source": "metrics",
"name": "query1",
"query": "avg:algo.flotilla.engine.eks.ara.memory_increase_ratio{$cluster,$env,$engine} by {cluster}"
}
]
}
]
}
},
{
"id": 7,
"layout": {
"x": 32,
"y": 32,
"width": 31,
"height": 15
},
"definition": {
"title": "CPU Increase Ratio Distribution",
"title_size": "16",
"title_align": "left",
"show_legend": false,
"type": "heatmap",
"yaxis": {
"label": "",
"scale": "linear",
"include_zero": true,
"min": "auto",
"max": "auto"
},
"requests": [
{
"style": {
"palette": "YlOrRd"
},
"response_format": "timeseries",
"queries": [
{
"data_source": "metrics",
"name": "query1",
"query": "avg:algo.flotilla.engine.eks.ara.cpu_increase_ratio{$cluster,$env,$engine} by {cluster}"
}
]
}
]
}
},
{
"id": 8,
"layout": {
"x": 64,
"y": 32,
"width": 31,
"height": 15
},
"definition": {
"title": "Top Clusters by Max Memory Hits",
"title_size": "16",
"title_align": "left",
"type": "toplist",
"requests": [
{
"style": {
"palette": "red"
},
"response_format": "scalar",
"queries": [
{
"data_source": "metrics",
"name": "query1",
"query": "sum:algo.flotilla.engine.eks.ara.hit_max_memory{$cluster,$env,$engine}.as_count()",
"aggregator": "avg"
}
],
"formulas": [
{
"formula": "query1"
}
],
"sort": {
"order_by": [
{
"type": "formula",
"index": 0,
"order": "desc"
}
]
}
}
]
}
},
{
"id": 9,
"layout": {
"x": 0,
"y": 48,
"width": 23,
"height": 15
},
"definition": {
"title": "Default Memory Distribution (Before ARA)",
"title_size": "16",
"title_align": "left",
"show_legend": false,
"type": "distribution",
"requests": [
{
"style": {
"palette": "blue"
},
"response_format": "scalar",
"queries": [
{
"data_source": "metrics",
"name": "query1",
"query": "avg:algo.flotilla.engine.eks.ara.default_memory{$cluster,$env,$engine} by {cluster}",
"aggregator": "avg"
}
]
}
]
}
},
{
"id": 10,
"layout": {
"x": 24,
"y": 48,
"width": 23,
"height": 15
},
"definition": {
"title": "ARA Memory Distribution (After ARA)",
"title_size": "16",
"title_align": "left",
"show_legend": false,
"type": "distribution",
"requests": [
{
"style": {
"palette": "orange"
},
"response_format": "scalar",
"queries": [
{
"data_source": "metrics",
"name": "query1",
"query": "avg:algo.flotilla.engine.eks.ara.ara_memory{$cluster,$env,$engine} by {cluster}",
"aggregator": "avg"
}
]
}
]
}
},
{
"id": 11,
"layout": {
"x": 48,
"y": 48,
"width": 23,
"height": 15
},
"definition": {
"title": "Final Memory Distribution (After Bounds)",
"title_size": "16",
"title_align": "left",
"show_legend": false,
"type": "distribution",
"requests": [
{
"style": {
"palette": "red"
},
"response_format": "scalar",
"queries": [
{
"data_source": "metrics",
"name": "query1",
"query": "avg:algo.flotilla.engine.eks.ara.final_memory_mb{$cluster,$env,$engine} by {cluster}",
"aggregator": "avg"
}
]
}
]
}
},
{
"id": 12,
"layout": {
"x": 72,
"y": 48,
"width": 23,
"height": 15
},
"definition": {
"title": "Memory Increase (Absolute MB)",
"title_size": "16",
"title_align": "left",
"show_legend": false,
"type": "distribution",
"requests": [
{
"style": {
"palette": "purple"
},
"response_format": "scalar",
"queries": [
{
"data_source": "metrics",
"name": "query1",
"query": "avg:algo.flotilla.engine.eks.ara.memory_increase{$cluster,$env,$engine} by {cluster}",
"aggregator": "avg"
}
]
}
]
}
},
{
"id": 13,
"layout": {
"x": 0,
"y": 64,
"width": 23,
"height": 15
},
"definition": {
"title": "Default CPU Distribution (Before ARA)",
"title_size": "16",
"title_align": "left",
"show_legend": false,
"type": "distribution",
"requests": [
{
"style": {
"palette": "blue"
},
"response_format": "scalar",
"queries": [
{
"data_source": "metrics",
"name": "query1",
"query": "avg:algo.flotilla.engine.eks.ara.default_cpu{$cluster,$env,$engine} by {cluster}",
"aggregator": "avg"
}
]
}
]
}
},
{
"id": 14,
"layout": {
"x": 24,
"y": 64,
"width": 23,
"height": 15
},
"definition": {
"title": "ARA CPU Distribution (After ARA)",
"title_size": "16",
"title_align": "left",
"show_legend": false,
"type": "distribution",
"requests": [
{
"style": {
"palette": "orange"
},
"response_format": "scalar",
"queries": [
{
"data_source": "metrics",
"name": "query1",
"query": "avg:algo.flotilla.engine.eks.ara.ara_cpu{$cluster,$env,$engine} by {cluster}",
"aggregator": "avg"
}
]
}
]
}
},
{
"id": 15,
"layout": {
"x": 48,
"y": 64,
"width": 23,
"height": 15
},
"definition": {
"title": "Final CPU Distribution (After Bounds)",
"title_size": "16",
"title_align": "left",
"show_legend": false,
"type": "distribution",
"requests": [
{
"style": {
"palette": "red"
},
"response_format": "scalar",
"queries": [
{
"data_source": "metrics",
"name": "query1",
"query": "avg:algo.flotilla.engine.eks.ara.final_cpu_millicores{$cluster,$env,$engine} by {cluster}",
"aggregator": "avg"
}
]
}
]
}
},
{
"id": 16,
"layout": {
"x": 72,
"y": 64,
"width": 23,
"height": 15
},
"definition": {
"title": "CPU Increase (Absolute Millicores)",
"title_size": "16",
"title_align": "left",
"show_legend": false,
"type": "distribution",
"requests": [
{
"style": {
"palette": "purple"
},
"response_format": "scalar",
"queries": [
{
"data_source": "metrics",
"name": "query1",
"query": "avg:algo.flotilla.engine.eks.ara.cpu_increase{$cluster,$env,$engine} by {cluster}",
"aggregator": "avg"
}
]
}
]
}
},
{
"id": 17,
"layout": {
"x": 0,
"y": 80,
"width": 47,
"height": 15
},
"definition": {
"title": "Resource Growth Over Time",
"title_size": "16",
"title_align": "left",
"show_legend": true,
"legend_size": "0",
"type": "timeseries",
"requests": [
{
"response_format": "timeseries",
"queries": [
{
"data_source": "metrics",
"name": "query1",
"query": "avg:algo.flotilla.engine.eks.ara.default_memory{$cluster,$env,$engine}"
}
],
"style": {
"palette": "blue",
"line_type": "solid",
"line_width": "normal"
},
"display_type": "line"
},
{
"response_format": "timeseries",
"queries": [
{
"data_source": "metrics",
"name": "query1",
"query": "avg:algo.flotilla.engine.eks.ara.ara_memory{$cluster,$env,$engine}"
}
],
"style": {
"palette": "orange",
"line_type": "solid",
"line_width": "normal"
},
"display_type": "line"
},
{
"response_format": "timeseries",
"queries": [
{
"data_source": "metrics",
"name": "query1",
"query": "avg:algo.flotilla.engine.eks.ara.final_memory_mb{$cluster,$env,$engine}"
}
],
"style": {
"palette": "red",
"line_type": "solid",
"line_width": "thick"
},
"display_type": "line"
}
],
"yaxis": {
"label": "Memory (MB)",
"scale": "linear",
"include_zero": true,
"min": "auto",
"max": "auto"
},
"markers": [
{
"label": "248GB Limit (Non-GPU EKS)",
"value": "y = 248000",
"display_type": "error dashed"
}
]
}
},
{
"id": 18,
"layout": {
"x": 48,
"y": 80,
"width": 47,
"height": 15
},
"definition": {
"title": "CPU Growth Over Time",
"title_size": "16",
"title_align": "left",
"show_legend": true,
"legend_size": "0",
"type": "timeseries",
"requests": [
{
"response_format": "timeseries",
"queries": [
{
"data_source": "metrics",
"name": "query1",
"query": "avg:algo.flotilla.engine.eks.ara.default_cpu{$cluster,$env,$engine}"
}
],
"style": {
"palette": "blue",
"line_type": "solid",
"line_width": "normal"
},
"display_type": "line"
},
{
"response_format": "timeseries",
"queries": [
{
"data_source": "metrics",
"name": "query1",
"query": "avg:algo.flotilla.engine.eks.ara.ara_cpu{$cluster,$env,$engine}"
}
],
"style": {
"palette": "orange",
"line_type": "solid",
"line_width": "normal"
},
"display_type": "line"
},
{
"response_format": "timeseries",
"queries": [
{
"data_source": "metrics",
"name": "query1",
"query": "avg:algo.flotilla.engine.eks.ara.final_cpu_millicores{$cluster,$env,$engine}"
}
],
"style": {
"palette": "red",
"line_type": "solid",
"line_width": "thick"
},
"display_type": "line"
}
],
"yaxis": {
"label": "CPU (millicores)",
"scale": "linear",
"include_zero": true,
"min": "auto",
"max": "auto"
},
"markers": [
{
"label": "60K Limit",
"value": "y = 60000",
"display_type": "error dashed"
}
]
}
},
{
"id": 19,
"layout": {
"x": 0,
"y": 96,
"width": 47,
"height": 30
},
"definition": {
"title": "ARA Logs - Resource Adjustments & Max Limits",
"title_size": "16",
"title_align": "left",
"requests": [
{
"response_format": "event_list",
"query": {
"data_source": "logs_stream",
"query_string": "source:flotilla (\"ARA adjusted resources\" OR \"Spark ARA adjusted executor memory\" OR \"Spark ARA adjusted driver memory\" OR \"ARA resource allocation hit maximum limit\" OR \"ARA memory allocation hit maximum limit\" OR \"ARA CPU allocation hit maximum limit\")",
"indexes": [],
"storage": "hot",
"sort": {
"order": "desc",
"column": "timestamp"
}
},
"columns": [
{
"field": "status_line",
"width": "auto"
},
{
"field": "timestamp",
"width": "auto"
},
{
"field": "host",
"width": "auto"
},
{
"field": "service",
"width": "auto"
},
{
"field": "source",
"width": "auto"
},
{
"field": "@status",
"width": "auto"
},
{
"field": "content",
"width": "compact"
}
]
}
],
"type": "list_stream"
}
},
{
"id": 20,
"layout": {
"x": 48,
"y": 96,
"width": 47,
"height": 30
},
"definition": {
"title": "ARA Logs - Historical Data Lookups",
"title_size": "16",
"title_align": "left",
"requests": [
{
"response_format": "event_list",
"query": {
"data_source": "logs_stream",
"query_string": "source:flotilla (\"ARA: Historical resource data found\" OR \"ARA: No historical resource data found\" OR \"ARA: Error querying historical resource data\")",
"indexes": [],
"storage": "hot",
"sort": {
"order": "desc",
"column": "timestamp"
}
},
"columns": [
{
"field": "status_line",
"width": "auto"
},
{
"field": "timestamp",
"width": "auto"
},
{
"field": "host",
"width": "auto"
},
{
"field": "service",
"width": "auto"
},
{
"field": "source",
"width": "auto"
},
{
"field": "@status",
"width": "auto"
},
{
"field": "content",
"width": "compact"
}
]
}
],
"type": "list_stream"
}
}
],
"template_variables": [
{
"name": "cluster",
"prefix": "cluster",
"available_values": [],
gitextract_unzartjt/
├── .circleci/
│ └── config.yml
├── .github/
│ ├── CODEOWNERS
│ └── PULL_REQUEST_TEMPLATE.md
├── .gitignore
├── .migrations/
│ ├── V20200123054713__initial_table_create.sql
│ ├── V20200123054714__add_spark_extension.sql
│ ├── V20200205133700__executable.sql
│ ├── V20200206115000__template.sql
│ ├── V20200210154600__template_refactor.sql
│ ├── V20200211160100__task_col_fix.sql
│ ├── V20200211161900__template_indicies.sql
│ ├── V20200212101900__template.sql
│ ├── V20200213101400__task_indexes.sql
│ ├── V20200213125200__rename_default_payload.sql
│ ├── V20200225125200__add_limits.sql
│ ├── V20200325125200__add_attempts.sql
│ ├── V20200325125201__add_spawned.sql
│ ├── V20200625125201__add_run_exceptions.sql
│ ├── V20210083054714__metrics_uri.sql
│ ├── V20210427125201__add_active_deadline_seconds.sql
│ ├── V20210807125201__drop_index_container_name.sql
│ ├── V20211007125201__add_description.sql
│ ├── V20220907125201__add_idempotence.sql
│ ├── V20220907125202__add_arch.sql
│ ├── V20221215125203__add_labels.sql
│ ├── V20230718115000__add_ephemeral_storage.sql
│ ├── V20231013191711__add_requires_docker.sql
│ ├── V20231122141100__add_target_cluster.sql
│ ├── V20240205132100__add_service_account.sql
│ ├── V20250122141100__add_cluster_routing.sql
│ └── dev.conf
├── ARA_METRICS_COMPARISON.md
├── Dockerfile
├── LICENSE
├── README.html
├── README.md
├── ara-impact-report-staging.md
├── ara-impact-report.md
├── clients/
│ ├── cluster/
│ │ ├── cluster.go
│ │ └── eks_cluster_client.go
│ ├── httpclient/
│ │ ├── client.go
│ │ └── client_test.go
│ ├── logs/
│ │ ├── eks_cloudwatch_logs_client.go
│ │ ├── eks_s3_logs_client.go
│ │ └── logs.go
│ ├── metrics/
│ │ ├── datadog_metrics_client.go
│ │ └── metrics.go
│ └── middleware/
│ └── client.go
├── conf/
│ └── config.yml
├── config/
│ ├── config.go
│ └── config_test.go
├── datadog-ara-dashboard-api.json
├── docker-compose.yml
├── docs/
│ ├── ara-command-hash-bug-report.md
│ ├── ara-command-hash-fix-locations.md
│ ├── ara-command-hash-history.md
│ ├── ara-instrumentation.md
│ └── ara.md
├── exceptions/
│ └── errors.go
├── execution/
│ ├── adapter/
│ │ ├── eks_adapter.go
│ │ └── eks_adapter_test.go
│ └── engine/
│ ├── dcm.go
│ ├── eks_engine.go
│ ├── emr_engine.go
│ └── engine.go
├── flotilla/
│ ├── app.go
│ ├── endpoints.go
│ ├── endpoints_test.go
│ └── router.go
├── go.mod
├── go.sum
├── log/
│ ├── event.go
│ ├── event_test.go
│ ├── logger.go
│ └── logger_test.go
├── main.go
├── queue/
│ ├── manager.go
│ ├── sqs_manager.go
│ └── sqs_manager_test.go
├── services/
│ ├── definition.go
│ ├── definition_test.go
│ ├── execution.go
│ ├── execution_test.go
│ ├── logs.go
│ ├── logs_test.go
│ ├── template.go
│ └── worker.go
├── state/
│ ├── manager.go
│ ├── models.go
│ ├── models_test.go
│ ├── pg_queries.go
│ ├── pg_state_manager.go
│ └── pg_state_manager_test.go
├── testutils/
│ └── mocks.go
├── tracing/
│ └── tracing.go
├── ui/
│ ├── .gitignore
│ ├── .prettierrc
│ ├── Dockerfile
│ ├── README.md
│ ├── package.json
│ ├── public/
│ │ └── index.html
│ ├── src/
│ │ ├── api.ts
│ │ ├── components/
│ │ │ ├── ARASwitch.tsx
│ │ │ ├── App.tsx
│ │ │ ├── Attribute.tsx
│ │ │ ├── AutoscrollSwitch.tsx
│ │ │ ├── BaseTaskForm.tsx
│ │ │ ├── CloudtrailRecords.tsx
│ │ │ ├── ClusterSelect.tsx
│ │ │ ├── CreateTaskForm.tsx
│ │ │ ├── DeleteTaskButton.tsx
│ │ │ ├── Duration.tsx
│ │ │ ├── EngineTag.tsx
│ │ │ ├── EnvFieldArray.tsx
│ │ │ ├── EnvList.tsx
│ │ │ ├── EnvQueryFilter.tsx
│ │ │ ├── ErrorCallout.tsx
│ │ │ ├── FieldError.tsx
│ │ │ ├── GenericMultiSelect.tsx
│ │ │ ├── GroupNameSelect.tsx
│ │ │ ├── ISO8601AttributeValue.tsx
│ │ │ ├── ListFiltersDropdown.tsx
│ │ │ ├── ListRequest.tsx
│ │ │ ├── Log.tsx
│ │ │ ├── LogProcessor.tsx
│ │ │ ├── LogRequesterCloudWatchLogs.tsx
│ │ │ ├── LogRequesterS3.tsx
│ │ │ ├── LogVirtualized.tsx
│ │ │ ├── LogVirtualizedRow.tsx
│ │ │ ├── LogVirtualizedSearch.tsx
│ │ │ ├── Navigation.tsx
│ │ │ ├── NodeLifecycleSelect.tsx
│ │ │ ├── Pagination.tsx
│ │ │ ├── QueryParams.tsx
│ │ │ ├── Request.tsx
│ │ │ ├── ResourceUsageValue.tsx
│ │ │ ├── Run.tsx
│ │ │ ├── RunAttributes.tsx
│ │ │ ├── RunDebugAttributes.tsx
│ │ │ ├── RunEvents.tsx
│ │ │ ├── RunSidebar.tsx
│ │ │ ├── RunStatusSelect.tsx
│ │ │ ├── RunTag.tsx
│ │ │ ├── Runs.tsx
│ │ │ ├── SettingsButton.tsx
│ │ │ ├── SortableTh.tsx
│ │ │ ├── StopRunButton.tsx
│ │ │ ├── Table.tsx
│ │ │ ├── TagsSelect.tsx
│ │ │ ├── Task.tsx
│ │ │ ├── TaskDetails.tsx
│ │ │ ├── TaskExecutionForm.tsx
│ │ │ ├── TaskRuns.tsx
│ │ │ ├── Tasks.tsx
│ │ │ ├── Template.tsx
│ │ │ ├── TemplateDetails.tsx
│ │ │ ├── TemplateExecutionForm.tsx
│ │ │ ├── TemplateHistoryTable.tsx
│ │ │ ├── TemplateRunForm.tsx
│ │ │ ├── Templates.tsx
│ │ │ ├── Toaster.ts
│ │ │ ├── Toggler.tsx
│ │ │ ├── UpdateTaskForm.tsx
│ │ │ ├── ViewHeader.tsx
│ │ │ └── __tests__/
│ │ │ ├── BaseTaskForm.spec.tsx
│ │ │ ├── ClusterSelect.spec.tsx
│ │ │ ├── CreateTaskForm.spec.tsx
│ │ │ ├── DeleteTaskButton.spec.tsx
│ │ │ ├── EnvFieldArray.spec.tsx
│ │ │ ├── GroupNameSelect.spec.tsx
│ │ │ ├── ListRequest.spec.tsx
│ │ │ ├── LogProcessor.spec.tsx
│ │ │ ├── LogVirtualized.spec.tsx
│ │ │ ├── LogVirtualizedSearch.spec.tsx
│ │ │ ├── Pagination.spec.tsx
│ │ │ ├── QueryParams.spec.tsx
│ │ │ ├── Request.spec.tsx
│ │ │ ├── Run.spec.tsx
│ │ │ ├── Runs.spec.tsx
│ │ │ ├── StopRunButton.spec.tsx
│ │ │ ├── TaskRuns.spec.tsx
│ │ │ ├── Tasks.spec.tsx
│ │ │ └── UpdateTaskForm.spec.tsx
│ │ ├── constants.ts
│ │ ├── helpers/
│ │ │ ├── FlotillaClient.ts
│ │ │ ├── __mocks__/
│ │ │ │ └── FlotillaClient.ts
│ │ │ ├── __tests__/
│ │ │ │ ├── FlotillaClient.spec.ts
│ │ │ │ ├── getInitialValuesForTaskRun.spec.ts
│ │ │ │ └── pageToOffsetLimit.spec.ts
│ │ │ ├── calculateDuration.ts
│ │ │ ├── constructDefaultObjectFromJsonSchema.ts
│ │ │ ├── getEnhancedRunStatus.ts
│ │ │ ├── getInitialValuesForExecutionForm.ts
│ │ │ ├── getOwnerIdRunTagFromCookies.ts
│ │ │ ├── pageToOffsetLimit.ts
│ │ │ ├── runFormHelpers.ts
│ │ │ ├── selectHelpers.ts
│ │ │ ├── taskFormHelpers.ts
│ │ │ └── testHelpers.ts
│ │ ├── index.css
│ │ ├── index.tsx
│ │ ├── localstorage.ts
│ │ ├── react-app-env.d.ts
│ │ ├── setupTests.js
│ │ ├── state/
│ │ │ ├── runView.ts
│ │ │ ├── settings.ts
│ │ │ └── store.ts
│ │ ├── types.ts
│ │ └── workers/
│ │ ├── index.ts
│ │ └── log.worker.ts
│ └── tsconfig.json
├── utils/
│ ├── dd_tracing.go
│ └── utils.go
└── worker/
├── events_worker.go
├── events_worker_test.go
├── retry_worker.go
├── retry_worker_test.go
├── status_worker.go
├── status_worker_test.go
├── submit_worker.go
├── submit_worker_test.go
├── worker.go
├── worker_manager.go
└── worker_test.go
SYMBOL INDEX (1257 symbols across 115 files)
FILE: .migrations/V20200123054713__initial_table_create.sql
type task_def (line 4) | CREATE TABLE IF NOT EXISTS task_def (
type task_def_ports (line 25) | CREATE TABLE IF NOT EXISTS task_def_ports (
type ix_task_def_alias (line 31) | CREATE INDEX IF NOT EXISTS ix_task_def_alias ON task_def(alias)
type ix_task_def_group_name (line 32) | CREATE INDEX IF NOT EXISTS ix_task_def_group_name ON task_def(group_name)
type ix_task_def_image (line 33) | CREATE INDEX IF NOT EXISTS ix_task_def_image ON task_def(image)
type ix_task_def_env (line 34) | CREATE INDEX IF NOT EXISTS ix_task_def_env ON task_def USING gin (env js...
type task (line 39) | CREATE TABLE IF NOT EXISTS task (
type ix_task_definition_id (line 77) | CREATE INDEX IF NOT EXISTS ix_task_definition_id ON task(definition_id)
type ix_task_cluster_name (line 78) | CREATE INDEX IF NOT EXISTS ix_task_cluster_name ON task(cluster_name)
type ix_task_status (line 79) | CREATE INDEX IF NOT EXISTS ix_task_status ON task(status)
type ix_task_group_name (line 80) | CREATE INDEX IF NOT EXISTS ix_task_group_name ON task(group_name)
type ix_task_env (line 81) | CREATE INDEX IF NOT EXISTS ix_task_env ON task USING gin (env jsonb_path...
type ix_task_definition_id (line 82) | CREATE INDEX IF NOT EXISTS ix_task_definition_id ON task(definition_id)
type ix_task_task_arn (line 83) | CREATE INDEX IF NOT EXISTS ix_task_task_arn ON task(task_arn)
type ix_task_definition_id_started_at_desc (line 84) | CREATE INDEX IF NOT EXISTS ix_task_definition_id_started_at_desc ON task...
type ix_task_definition_id_started_at_desc_engine (line 85) | CREATE INDEX IF NOT EXISTS ix_task_definition_id_started_at_desc_engine ...
type ix_finished_at_status_cluster_name (line 86) | CREATE INDEX IF NOT EXISTS ix_finished_at_status_cluster_name ON task US...
type ix_task_definition_id_started_at_asc (line 87) | CREATE INDEX IF NOT EXISTS ix_task_definition_id_started_at_asc ON task ...
type ix_task_pod_events (line 88) | CREATE INDEX IF NOT EXISTS ix_task_pod_events ON task USING gin (pod_eve...
type ix_task_queued_at_status_engine (line 89) | CREATE INDEX IF NOT EXISTS ix_task_queued_at_status_engine ON task USING...
type task_definition_id_engine_started_at_index (line 90) | CREATE INDEX IF NOT EXISTS task_definition_id_engine_started_at_index ON...
type task_status (line 94) | CREATE TABLE IF NOT EXISTS task_status (
type ix_task_status_task_arn (line 101) | CREATE INDEX IF NOT EXISTS ix_task_status_task_arn ON task_status(task_arn)
type tags (line 112) | CREATE TABLE IF NOT EXISTS tags (
type task_def_tags (line 115) | CREATE TABLE IF NOT EXISTS task_def_tags (
type worker (line 119) | CREATE TABLE IF NOT EXISTS worker (
FILE: .migrations/V20200206115000__template.sql
type template (line 1) | CREATE TABLE template (
FILE: .migrations/V20200211161900__template_indicies.sql
type ix_template_id (line 1) | CREATE INDEX IF NOT EXISTS ix_template_id ON template(template_id)
type ix_template_name (line 2) | CREATE INDEX IF NOT EXISTS ix_template_name ON template(template_name)
FILE: .migrations/V20200213101400__task_indexes.sql
type ix_task_executable_id (line 1) | CREATE INDEX IF NOT EXISTS ix_task_executable_id ON task(executable_id)
type ix_task_executable_id_started_at_desc (line 2) | CREATE INDEX IF NOT EXISTS ix_task_executable_id_started_at_desc ON task...
type ix_task_executable_id_started_at_desc_engine (line 3) | CREATE INDEX IF NOT EXISTS ix_task_executable_id_started_at_desc_engine ...
FILE: .migrations/V20250122141100__add_cluster_routing.sql
type cluster_state (line 8) | CREATE TABLE IF NOT EXISTS cluster_state (
type ix_cluster_state_name (line 24) | CREATE INDEX IF NOT EXISTS ix_cluster_state_name ON cluster_state(name)
type ix_cluster_state_status (line 25) | CREATE INDEX IF NOT EXISTS ix_cluster_state_status ON cluster_state(status)
FILE: clients/cluster/cluster.go
type Client (line 18) | type Client interface
function NewClusterClient (line 26) | func NewClusterClient(conf config.Config, name string) (Client, error) {
FILE: clients/cluster/eks_cluster_client.go
type EKSClusterClient (line 10) | type EKSClusterClient struct
method Name (line 12) | func (EKSClusterClient) Name() string {
method Initialize (line 16) | func (EKSClusterClient) Initialize(conf config.Config) error {
method CanBeRun (line 21) | func (EKSClusterClient) CanBeRun(clusterName string, executableResourc...
method ListClusters (line 26) | func (EKSClusterClient) ListClusters() ([]state.ClusterMetadata, error) {
FILE: clients/httpclient/client.go
type RetryableError (line 14) | type RetryableError interface
type HttpRetryableError (line 18) | type HttpRetryableError struct
method Error (line 22) | func (re HttpRetryableError) Error() string {
method Err (line 26) | func (re HttpRetryableError) Err() string {
type RequestExecutor (line 30) | type RequestExecutor interface
type defaultExecutor (line 34) | type defaultExecutor struct
method Do (line 36) | func (de *defaultExecutor) Do(req *http.Request, timeout time.Duration...
type Client (line 59) | type Client struct
method Get (line 66) | func (c *Client) Get(path string, headers map[string]string, entity in...
method Delete (line 74) | func (c *Client) Delete(path string, headers map[string]string, entity...
method Put (line 82) | func (c *Client) Put(path string, headers map[string]string, inEntity ...
method Post (line 90) | func (c *Client) Post(path string, headers map[string]string, inEntity...
method prepareRequestNoBody (line 98) | func (c *Client) prepareRequestNoBody(method string, path string, head...
method prepareRequestWithBody (line 102) | func (c *Client) prepareRequestWithBody(method string, path string, he...
method makeURL (line 111) | func (c *Client) makeURL(path string) (string, error) {
method makeRequest (line 133) | func (c *Client) makeRequest(method, path string, headers map[string]s...
method doRequestWithRetry (line 151) | func (c *Client) doRequestWithRetry(req *http.Request, entity interfac...
method retryRequest (line 163) | func (c *Client) retryRequest(sleepTime time.Duration, fn httpreqfunc)...
type httpreqfunc (line 161) | type httpreqfunc
FILE: clients/httpclient/client_test.go
type Cupcake (line 13) | type Cupcake struct
constant cupcakeResponse (line 18) | cupcakeResponse = `{"flavour": "vomit", "sprinkles": true}`
type MockExecutor (line 20) | type MockExecutor struct
method Do (line 24) | func (me *MockExecutor) Do(req *http.Request, timeout time.Duration, e...
function TestClientRetry (line 33) | func TestClientRetry(t *testing.T) {
function TestClientDo (line 55) | func TestClientDo(t *testing.T) {
FILE: clients/logs/eks_cloudwatch_logs_client.go
type EKSCloudWatchLogsClient (line 27) | type EKSCloudWatchLogsClient struct
method Name (line 41) | func (lc *EKSCloudWatchLogsClient) Name() string {
method Initialize (line 48) | func (lc *EKSCloudWatchLogsClient) Initialize(conf config.Config) error {
method Logs (line 91) | func (lc *EKSCloudWatchLogsClient) Logs(executable state.Executable, r...
method LogsText (line 133) | func (lc *EKSCloudWatchLogsClient) LogsText(executable state.Executabl...
method toStreamName (line 138) | func (lc *EKSCloudWatchLogsClient) toStreamName(run state.Run) string {
method logsToMessage (line 143) | func (lc *EKSCloudWatchLogsClient) logsToMessage(events []*cloudwatchl...
method createNamespaceIfNotExists (line 158) | func (lc *EKSCloudWatchLogsClient) createNamespaceIfNotExists() error {
method namespaceExists (line 170) | func (lc *EKSCloudWatchLogsClient) namespaceExists() (bool, error) {
method createNamespace (line 190) | func (lc *EKSCloudWatchLogsClient) createNamespace() error {
type EKSCloudWatchLog (line 34) | type EKSCloudWatchLog struct
FILE: clients/logs/eks_s3_logs_client.go
type EKSS3LogsClient (line 29) | type EKSS3LogsClient struct
method Name (line 47) | func (lc *EKSS3LogsClient) Name() string {
method Initialize (line 52) | func (lc *EKSS3LogsClient) Initialize(conf config.Config) error {
method emrLogsToMessageString (line 95) | func (lc *EKSS3LogsClient) emrLogsToMessageString(run state.Run, lastS...
method emrDriverLogsPath (line 187) | func (lc *EKSS3LogsClient) emrDriverLogsPath(run state.Run) (string, e...
method Logs (line 199) | func (lc *EKSS3LogsClient) Logs(executable state.Executable, run state...
method LogsText (line 223) | func (lc *EKSS3LogsClient) LogsText(executable state.Executable, run s...
method getS3Object (line 240) | func (lc *EKSS3LogsClient) getS3Object(run state.Run) (*s3.GetObjectOu...
method getS3Key (line 279) | func (lc *EKSS3LogsClient) getS3Key(s3Key *string) (*s3.GetObjectOutpu...
method toS3DirName (line 291) | func (lc *EKSS3LogsClient) toS3DirName(run state.Run) string {
method logsToMessage (line 296) | func (lc *EKSS3LogsClient) logsToMessage(result *s3.GetObjectOutput, w...
method logsEMR (line 321) | func (lc *EKSS3LogsClient) logsEMR(w http.ResponseWriter) error {
method logsToMessageString (line 327) | func (lc *EKSS3LogsClient) logsToMessageString(result *s3.GetObjectOut...
type s3Log (line 40) | type s3Log struct
function parseLines (line 376) | func parseLines(input []byte) (s3Log, error) {
FILE: clients/logs/logs.go
type Client (line 16) | type Client interface
type logsClient (line 23) | type logsClient interface
type byTimestamp (line 30) | type byTimestamp
method Len (line 32) | func (events byTimestamp) Len() int { return len(events) }
method Swap (line 33) | func (events byTimestamp) Swap(i, j int) { events[i], events[j] =...
method Less (line 34) | func (events byTimestamp) Less(i, j int) bool { return *(events[i].Tim...
function NewLogsClient (line 39) | func NewLogsClient(conf config.Config, logger flotillaLog.Logger, name s...
FILE: clients/metrics/datadog_metrics_client.go
type DatadogStatsdMetricsClient (line 13) | type DatadogStatsdMetricsClient struct
method Init (line 20) | func (dd *DatadogStatsdMetricsClient) Init(conf config.Config) error {
method Decrement (line 40) | func (dd *DatadogStatsdMetricsClient) Decrement(name Metric, tags []st...
method Increment (line 45) | func (dd *DatadogStatsdMetricsClient) Increment(name Metric, tags []st...
method Histogram (line 50) | func (dd *DatadogStatsdMetricsClient) Histogram(name Metric, value flo...
method Distribution (line 55) | func (dd *DatadogStatsdMetricsClient) Distribution(name Metric, value ...
method Timing (line 60) | func (dd *DatadogStatsdMetricsClient) Timing(name Metric, value time.D...
method Set (line 65) | func (dd *DatadogStatsdMetricsClient) Set(name Metric, value string, t...
method Event (line 70) | func (dd *DatadogStatsdMetricsClient) Event(e event) error {
FILE: clients/metrics/metrics.go
type Metric (line 12) | type Metric
constant EngineEKSExecute (line 16) | EngineEKSExecute Metric = "engine.eks.execute"
constant EngineEKSEnqueue (line 18) | EngineEKSEnqueue Metric = "engine.eks.enqueue"
constant EngineEMRExecute (line 20) | EngineEMRExecute Metric = "engine.emr.execute"
constant EngineEMREnqueue (line 22) | EngineEMREnqueue Metric = "engine.emr.enqueue"
constant EngineEKSTerminate (line 24) | EngineEKSTerminate Metric = "engine.eks.terminate"
constant EngineEMRTerminate (line 26) | EngineEMRTerminate Metric = "engine.emr.terminate"
constant EngineEKSRunPodnameChange (line 28) | EngineEKSRunPodnameChange Metric = "engine.eks.run_podname_changed"
constant EngineEKSNodeTriggeredScaledUp (line 30) | EngineEKSNodeTriggeredScaledUp Metric = "engine.eks.triggered_scale_up"
constant StatusWorkerProcessEKSRun (line 32) | StatusWorkerProcessEKSRun Metric = "status_worker.timing.process_eks_run"
constant StatusWorkerAcquireLock (line 34) | StatusWorkerAcquireLock Metric = "status_worker.timing.acquire_lock"
constant StatusWorkerFetchPodMetrics (line 36) | StatusWorkerFetchPodMetrics Metric = "status_worker.timing.fetch_pod_met...
constant StatusWorkerFetchUpdateStatus (line 38) | StatusWorkerFetchUpdateStatus Metric = "status_worker.timing.fetch_updat...
constant StatusWorkerLockedRuns (line 40) | StatusWorkerLockedRuns Metric = "status_worker.locked_runs"
constant StatusWorkerFetchMetrics (line 42) | StatusWorkerFetchMetrics Metric = "status_worker.fetch_metrics"
constant StatusWorkerGetPodList (line 44) | StatusWorkerGetPodList Metric = "status_worker.get_pod_list"
constant StatusWorkerGetEvents (line 46) | StatusWorkerGetEvents Metric = "status_worker.get_events"
constant StatusWorkerGetJob (line 48) | StatusWorkerGetJob Metric = "status_worker.get_job"
constant EngineUpdateRun (line 50) | EngineUpdateRun Metric = "engine.update_run"
constant EngineEKSARAEstimationAttempted (line 52) | EngineEKSARAEstimationAttempted Metric = "engine.eks.ara.estimation_atte...
constant EngineEKSARAEstimationSucceeded (line 53) | EngineEKSARAEstimationSucceeded Metric = "engine.eks.ara.estimation_succ...
constant EngineEKSARAEstimationFailed (line 54) | EngineEKSARAEstimationFailed Metric = "engine.eks.ara.estimation_failed"
constant EngineEKSARAResourceAdjustment (line 55) | EngineEKSARAResourceAdjustment Metric = "engine.eks.ara.resource_adjust...
constant EngineEKSARANoHistoricalData (line 56) | EngineEKSARANoHistoricalData Metric = "engine.eks.ara.no_historical_d...
constant EngineEKSARAHitMaxMemory (line 57) | EngineEKSARAHitMaxMemory Metric = "engine.eks.ara.hit_max_memory"
constant EngineEKSARAHitMaxCPU (line 58) | EngineEKSARAHitMaxCPU Metric = "engine.eks.ara.hit_max_cpu"
constant EngineEKSARAMemoryIncreaseRatio (line 59) | EngineEKSARAMemoryIncreaseRatio Metric = "engine.eks.ara.memory_increas...
constant EngineEKSARACPUIncreaseRatio (line 60) | EngineEKSARACPUIncreaseRatio Metric = "engine.eks.ara.cpu_increase_r...
constant EngineEKSARAFinalMemoryMB (line 61) | EngineEKSARAFinalMemoryMB Metric = "engine.eks.ara.final_memory_mb"
constant EngineEKSARAFinalCPUMillicores (line 62) | EngineEKSARAFinalCPUMillicores Metric = "engine.eks.ara.final_cpu_mill...
constant EngineEKSARADefaultMemory (line 63) | EngineEKSARADefaultMemory Metric = "engine.eks.ara.default_memory"
constant EngineEKSARAARAMemory (line 64) | EngineEKSARAARAMemory Metric = "engine.eks.ara.ara_memory"
constant EngineEKSARADefaultCPU (line 65) | EngineEKSARADefaultCPU Metric = "engine.eks.ara.default_cpu"
constant EngineEKSARAARACPU (line 66) | EngineEKSARAARACPU Metric = "engine.eks.ara.ara_cpu"
constant EngineEKSARAMemoryIncrease (line 67) | EngineEKSARAMemoryIncrease Metric = "engine.eks.ara.memory_increase"
constant EngineEKSARACPUIncrease (line 68) | EngineEKSARACPUIncrease Metric = "engine.eks.ara.cpu_increase"
constant EngineEKSARANullCommandHash (line 69) | EngineEKSARANullCommandHash Metric = "engine.eks.ara.null_command_h...
type MetricTag (line 72) | type MetricTag
constant StatusSuccess (line 76) | StatusSuccess MetricTag = "status:success"
constant StatusFailure (line 78) | StatusFailure MetricTag = "status:failure"
type Client (line 81) | type Client interface
type event (line 92) | type event struct
function InstantiateClient (line 102) | func InstantiateClient(conf config.Config) error {
function Decrement (line 130) | func Decrement(name Metric, tags []string, rate float64) error {
function Increment (line 139) | func Increment(name Metric, tags []string, rate float64) error {
function Histogram (line 148) | func Histogram(name Metric, value float64, tags []string, rate float64) ...
function Distribution (line 157) | func Distribution(name Metric, value float64, tags []string, rate float6...
function Set (line 166) | func Set(name Metric, value string, tags []string, rate float64) error {
function Event (line 175) | func Event(title string, text string, tags []string) error {
function Timing (line 188) | func Timing(name Metric, value time.Duration, tags []string, rate float6...
FILE: clients/middleware/client.go
type Client (line 8) | type Client interface
type middlewareClient (line 12) | type middlewareClient struct
method AnnotateLaunchRequest (line 18) | func (mwC middlewareClient) AnnotateLaunchRequest(headers *http.Header...
function NewClient (line 14) | func NewClient() (Client, error) {
FILE: config/config.go
type Config (line 12) | type Config interface
function NewConfig (line 27) | func NewConfig(confDir *string) (Config, error) {
type conf (line 45) | type conf struct
method GetString (line 50) | func (c *conf) GetString(key string) string {
method GetFloat64 (line 55) | func (c *conf) GetFloat64(key string) float64 {
method GetInt (line 60) | func (c *conf) GetInt(key string) int {
method GetBool (line 65) | func (c *conf) GetBool(key string) bool {
method GetStringMapString (line 70) | func (c *conf) GetStringMapString(key string) map[string]string {
method GetStringSlice (line 75) | func (c *conf) GetStringSlice(key string) []string {
method IsSet (line 81) | func (c *conf) IsSet(key string) bool {
FILE: config/config_test.go
function TestNewConfig (line 8) | func TestNewConfig(t *testing.T) {
FILE: exceptions/errors.go
type MalformedInput (line 6) | type MalformedInput struct
method Error (line 10) | func (e MalformedInput) Error() string {
type ConflictingResource (line 18) | type ConflictingResource struct
method Error (line 22) | func (e ConflictingResource) Error() string {
type MissingResource (line 30) | type MissingResource struct
method Error (line 34) | func (e MissingResource) Error() string {
FILE: execution/adapter/eks_adapter.go
type EKSAdapter (line 23) | type EKSAdapter interface
type eksAdapter (line 27) | type eksAdapter struct
method AdaptJobToFlotillaRun (line 40) | func (a *eksAdapter) AdaptJobToFlotillaRun(job *batchv1.Job, run state...
method AdaptFlotillaDefinitionAndRunToJob (line 104) | func (a *eksAdapter) AdaptFlotillaDefinitionAndRunToJob(ctx context.Co...
method constructEviction (line 174) | func (a *eksAdapter) constructEviction(ctx context.Context, run state....
method constructContainerPorts (line 191) | func (a *eksAdapter) constructContainerPorts(executable state.Executab...
method constructTolerations (line 204) | func (a *eksAdapter) constructTolerations(executable state.Executable,...
method constructAffinity (line 231) | func (a *eksAdapter) constructAffinity(ctx context.Context, executable...
method constructResourceRequirements (line 298) | func (a *eksAdapter) constructResourceRequirements(ctx context.Context...
method constructVolumeMounts (line 355) | func (a *eksAdapter) constructVolumeMounts(ctx context.Context, execut...
method adaptiveResources (line 384) | func (a *eksAdapter) adaptiveResources(ctx context.Context, executable...
method emitARAMetrics (line 527) | func (a *eksAdapter) emitARAMetrics(run state.Run, defaultCPU int64, d...
method checkResourceBounds (line 574) | func (a *eksAdapter) checkResourceBounds(cpu int64, mem int64, isGPUJo...
method getResourceDefaults (line 617) | func (a *eksAdapter) getResourceDefaults(run state.Run, executable sta...
method getLastRun (line 653) | func (a *eksAdapter) getLastRun(ctx context.Context, manager state.Man...
method constructCmdSlice (line 669) | func (a *eksAdapter) constructCmdSlice(cmdString string) []string {
method envOverrides (line 676) | func (a *eksAdapter) envOverrides(executable state.Executable, run sta...
method sanitizeEnvVar (line 708) | func (a *eksAdapter) sanitizeEnvVar(key string) string {
method sanitizeLabel (line 718) | func (a *eksAdapter) sanitizeLabel(key string) string {
method roundCPUMillicores (line 730) | func (a *eksAdapter) roundCPUMillicores(millicores int64) int64 {
function NewEKSAdapter (line 33) | func NewEKSAdapter(logger flotillaLog.Logger) (EKSAdapter, error) {
FILE: execution/adapter/eks_adapter_test.go
function TestRoundCPUMillicores (line 13) | func TestRoundCPUMillicores(t *testing.T) {
function TestRoundCPUAvoidsCgroupIssue (line 68) | func TestRoundCPUAvoidsCgroupIssue(t *testing.T) {
type mockLogger (line 100) | type mockLogger struct
method Log (line 105) | func (m *mockLogger) Log(keyvals ...interface{}) error {
method Event (line 110) | func (m *mockLogger) Event(keyvals ...interface{}) error {
method reset (line 115) | func (m *mockLogger) reset() {
type mockStateManager (line 121) | type mockStateManager struct
method EstimateRunResources (line 126) | func (m *mockStateManager) EstimateRunResources(ctx context.Context, e...
method Name (line 131) | func (m *mockStateManager) Name() string { return...
method Initialize (line 132) | func (m *mockStateManager) Initialize(conf config.Config) error { retu...
method Cleanup (line 133) | func (m *mockStateManager) Cleanup() error ...
method ListDefinitions (line 134) | func (m *mockStateManager) ListDefinitions(ctx context.Context, limit ...
method GetDefinition (line 137) | func (m *mockStateManager) GetDefinition(ctx context.Context, definiti...
method GetDefinitionByAlias (line 140) | func (m *mockStateManager) GetDefinitionByAlias(ctx context.Context, a...
method UpdateDefinition (line 143) | func (m *mockStateManager) UpdateDefinition(ctx context.Context, defin...
method CreateDefinition (line 146) | func (m *mockStateManager) CreateDefinition(ctx context.Context, d sta...
method DeleteDefinition (line 147) | func (m *mockStateManager) DeleteDefinition(ctx context.Context, defin...
method ListRuns (line 148) | func (m *mockStateManager) ListRuns(ctx context.Context, limit int, of...
method EstimateExecutorCount (line 151) | func (m *mockStateManager) EstimateExecutorCount(ctx context.Context, ...
method ExecutorOOM (line 154) | func (m *mockStateManager) ExecutorOOM(ctx context.Context, executable...
method DriverOOM (line 157) | func (m *mockStateManager) DriverOOM(ctx context.Context, executableID...
method GetRun (line 160) | func (m *mockStateManager) GetRun(ctx context.Context, runID string) (...
method CreateRun (line 163) | func (m *mockStateManager) CreateRun(ctx context.Context, r state.Run)...
method UpdateRun (line 164) | func (m *mockStateManager) UpdateRun(ctx context.Context, runID string...
method ListGroups (line 167) | func (m *mockStateManager) ListGroups(ctx context.Context, limit int, ...
method ListTags (line 170) | func (m *mockStateManager) ListTags(ctx context.Context, limit int, of...
method ListWorkers (line 173) | func (m *mockStateManager) ListWorkers(ctx context.Context, engine str...
method BatchUpdateWorkers (line 176) | func (m *mockStateManager) BatchUpdateWorkers(ctx context.Context, upd...
method GetWorker (line 179) | func (m *mockStateManager) GetWorker(ctx context.Context, workerType s...
method UpdateWorker (line 182) | func (m *mockStateManager) UpdateWorker(ctx context.Context, workerTyp...
method GetExecutableByTypeAndID (line 185) | func (m *mockStateManager) GetExecutableByTypeAndID(ctx context.Contex...
method GetTemplateByID (line 188) | func (m *mockStateManager) GetTemplateByID(ctx context.Context, templa...
method GetLatestTemplateByTemplateName (line 191) | func (m *mockStateManager) GetLatestTemplateByTemplateName(ctx context...
method GetTemplateByVersion (line 194) | func (m *mockStateManager) GetTemplateByVersion(ctx context.Context, t...
method ListTemplates (line 197) | func (m *mockStateManager) ListTemplates(ctx context.Context, limit in...
method ListTemplatesLatestOnly (line 200) | func (m *mockStateManager) ListTemplatesLatestOnly(ctx context.Context...
method CreateTemplate (line 203) | func (m *mockStateManager) CreateTemplate(ctx context.Context, t state...
method ListFailingNodes (line 204) | func (m *mockStateManager) ListFailingNodes(ctx context.Context) (stat...
method GetPodReAttemptRate (line 207) | func (m *mockStateManager) GetPodReAttemptRate(ctx context.Context) (f...
method GetNodeLifecycle (line 210) | func (m *mockStateManager) GetNodeLifecycle(ctx context.Context, execu...
method GetTaskHistoricalRuntime (line 213) | func (m *mockStateManager) GetTaskHistoricalRuntime(ctx context.Contex...
method CheckIdempotenceKey (line 216) | func (m *mockStateManager) CheckIdempotenceKey(ctx context.Context, id...
method GetRunByEMRJobId (line 219) | func (m *mockStateManager) GetRunByEMRJobId(ctx context.Context, emrJo...
method GetResources (line 222) | func (m *mockStateManager) GetResources(ctx context.Context, runID str...
method ListClusterStates (line 225) | func (m *mockStateManager) ListClusterStates(ctx context.Context) ([]s...
method UpdateClusterMetadata (line 228) | func (m *mockStateManager) UpdateClusterMetadata(ctx context.Context, ...
method DeleteClusterMetadata (line 231) | func (m *mockStateManager) DeleteClusterMetadata(ctx context.Context, ...
method GetClusterByID (line 234) | func (m *mockStateManager) GetClusterByID(ctx context.Context, cluster...
method GetRunStatus (line 237) | func (m *mockStateManager) GetRunStatus(ctx context.Context, runID str...
type mockExecutable (line 242) | type mockExecutable struct
method GetExecutableID (line 247) | func (m *mockExecutable) GetExecutableID() *string {
method GetExecutableType (line 251) | func (m *mockExecutable) GetExecutableType() *state.ExecutableType {
method GetExecutableResources (line 256) | func (m *mockExecutable) GetExecutableResources() *state.ExecutableRes...
method GetExecutableCommand (line 260) | func (m *mockExecutable) GetExecutableCommand(req state.ExecutionReque...
method GetExecutableResourceName (line 264) | func (m *mockExecutable) GetExecutableResourceName() string {
function TestAdaptiveResources_NonGPUJob_ARAEnabled_Success (line 268) | func TestAdaptiveResources_NonGPUJob_ARAEnabled_Success(t *testing.T) {
function TestAdaptiveResources_GPUJob_SkipsARA (line 324) | func TestAdaptiveResources_GPUJob_SkipsARA(t *testing.T) {
function TestAdaptiveResources_EstimationFailed (line 368) | func TestAdaptiveResources_EstimationFailed(t *testing.T) {
function TestAdaptiveResources_MaxResourceBoundsHit (line 412) | func TestAdaptiveResources_MaxResourceBoundsHit(t *testing.T) {
function TestAdaptiveResources_ARADisabled (line 520) | func TestAdaptiveResources_ARADisabled(t *testing.T) {
function TestEmitARAMetrics_StructuredLog (line 562) | func TestEmitARAMetrics_StructuredLog(t *testing.T) {
function TestEmitARAMetrics_NilLogger (line 626) | func TestEmitARAMetrics_NilLogger(t *testing.T) {
function int64Ptr (line 639) | func int64Ptr(i int64) *int64 {
FILE: execution/engine/dcm.go
type DynamicClusterManager (line 23) | type DynamicClusterManager struct
method getOrCreateKubeconfig (line 58) | func (dcm *DynamicClusterManager) getOrCreateKubeconfig(clusterName st...
method generateKubeconfig (line 88) | func (dcm *DynamicClusterManager) generateKubeconfig(clusterName, kube...
method createRestConfig (line 109) | func (dcm *DynamicClusterManager) createRestConfig(kubeconfigPath stri...
method GetKubernetesClient (line 120) | func (dcm *DynamicClusterManager) GetKubernetesClient(clusterName stri...
method GetMetricsClient (line 140) | func (dcm *DynamicClusterManager) GetMetricsClient(clusterName string)...
method InitializeClusters (line 160) | func (dcm *DynamicClusterManager) InitializeClusters(ctx context.Conte...
function getKubeconfigBaseDir (line 33) | func getKubeconfigBaseDir() string {
function NewDynamicClusterManager (line 42) | func NewDynamicClusterManager(awsRegion string, log flotillaLog.Logger, ...
FILE: execution/engine/eks_engine.go
type EKSExecutionEngine (line 33) | type EKSExecutionEngine struct
method Initialize (line 57) | func (ee *EKSExecutionEngine) Initialize(conf config.Config) error {
method Execute (line 120) | func (ee *EKSExecutionEngine) Execute(ctx context.Context, executable ...
method getPodName (line 218) | func (ee *EKSExecutionEngine) getPodName(run state.Run) (state.Run, er...
method getInstanceDetails (line 245) | func (ee *EKSExecutionEngine) getInstanceDetails(pod v1.Pod, run state...
method getPodList (line 252) | func (ee *EKSExecutionEngine) getPodList(run state.Run) (*v1.PodList, ...
method getKClient (line 279) | func (ee *EKSExecutionEngine) getKClient(run state.Run) (kubernetes.Cl...
method Terminate (line 295) | func (ee *EKSExecutionEngine) Terminate(ctx context.Context, run state...
method Enqueue (line 329) | func (ee *EKSExecutionEngine) Enqueue(ctx context.Context, run state.R...
method PollRuns (line 355) | func (ee *EKSExecutionEngine) PollRuns(ctx context.Context) ([]RunRece...
method PollStatus (line 393) | func (ee *EKSExecutionEngine) PollStatus(ctx context.Context) (RunRece...
method PollRunStatus (line 398) | func (ee *EKSExecutionEngine) PollRunStatus(ctx context.Context) (stat...
method Define (line 403) | func (ee *EKSExecutionEngine) Define(ctx context.Context, td state.Def...
method Deregister (line 408) | func (ee *EKSExecutionEngine) Deregister(ctx context.Context, definiti...
method Get (line 412) | func (ee *EKSExecutionEngine) Get(ctx context.Context, run state.Run) ...
method GetEvents (line 434) | func (ee *EKSExecutionEngine) GetEvents(ctx context.Context, run state...
method FetchPodMetrics (line 482) | func (ee *EKSExecutionEngine) FetchPodMetrics(ctx context.Context, run...
method FetchUpdateStatus (line 530) | func (ee *EKSExecutionEngine) FetchUpdateStatus(ctx context.Context, r...
FILE: execution/engine/emr_engine.go
type EMRExecutionEngine (line 38) | type EMRExecutionEngine struct
method Initialize (line 66) | func (emr *EMRExecutionEngine) Initialize(conf config.Config) error {
method getKClient (line 126) | func (emr *EMRExecutionEngine) getKClient(run state.Run) (kubernetes.C...
method Execute (line 133) | func (emr *EMRExecutionEngine) Execute(ctx context.Context, executable...
method generateApplicationConf (line 194) | func (emr *EMRExecutionEngine) generateApplicationConf(ctx context.Con...
method generateEMRStartJobRunInput (line 251) | func (emr *EMRExecutionEngine) generateEMRStartJobRunInput(ctx context...
method generateTags (line 305) | func (emr *EMRExecutionEngine) generateTags(run state.Run) map[string]...
method driverPodTemplate (line 356) | func (emr *EMRExecutionEngine) driverPodTemplate(ctx context.Context, ...
method executorPodTemplate (line 415) | func (emr *EMRExecutionEngine) executorPodTemplate(ctx context.Context...
method writeK8ObjToS3 (line 466) | func (emr *EMRExecutionEngine) writeK8ObjToS3(obj runtime.Object, key ...
method writeStringToS3 (line 489) | func (emr *EMRExecutionEngine) writeStringToS3(key *string, body []byt...
method constructEviction (line 505) | func (emr *EMRExecutionEngine) constructEviction(ctx context.Context, ...
method constructTolerations (line 521) | func (emr *EMRExecutionEngine) constructTolerations(executable state.E...
method constructAffinity (line 543) | func (emr *EMRExecutionEngine) constructAffinity(ctx context.Context, ...
method estimateExecutorCount (line 651) | func (emr *EMRExecutionEngine) estimateExecutorCount(run state.Run, ma...
method buildMetricTags (line 656) | func (emr *EMRExecutionEngine) buildMetricTags(run state.Run) []string {
method estimateMemoryResources (line 674) | func (emr *EMRExecutionEngine) estimateMemoryResources(ctx context.Con...
method sparkSubmitParams (line 806) | func (emr *EMRExecutionEngine) sparkSubmitParams(run state.Run) *string {
method Terminate (line 840) | func (emr *EMRExecutionEngine) Terminate(ctx context.Context, run stat...
method Enqueue (line 875) | func (emr *EMRExecutionEngine) Enqueue(ctx context.Context, run state....
method PollRuns (line 901) | func (emr *EMRExecutionEngine) PollRuns(ctx context.Context) ([]RunRec...
method PollStatus (line 932) | func (emr *EMRExecutionEngine) PollStatus(ctx context.Context) (RunRec...
method PollRunStatus (line 936) | func (emr *EMRExecutionEngine) PollRunStatus(ctx context.Context) (sta...
method Define (line 940) | func (emr *EMRExecutionEngine) Define(ctx context.Context, td state.De...
method Deregister (line 944) | func (emr *EMRExecutionEngine) Deregister(ctx context.Context, definit...
method Get (line 948) | func (emr *EMRExecutionEngine) Get(ctx context.Context, run state.Run)...
method GetEvents (line 955) | func (emr *EMRExecutionEngine) GetEvents(ctx context.Context, run stat...
method FetchPodMetrics (line 967) | func (emr *EMRExecutionEngine) FetchPodMetrics(ctx context.Context, ru...
method FetchUpdateStatus (line 979) | func (emr *EMRExecutionEngine) FetchUpdateStatus(ctx context.Context, ...
method lakekeeperSecretEnvVars (line 990) | func (emr *EMRExecutionEngine) lakekeeperSecretEnvVars() []v1.EnvVar {
method envOverrides (line 1058) | func (emr *EMRExecutionEngine) envOverrides(executable state.Executabl...
method sanitizeEnvVar (line 1091) | func (emr *EMRExecutionEngine) sanitizeEnvVar(key string) string {
method constructCmdSlice (line 1101) | func (emr *EMRExecutionEngine) constructCmdSlice(command *string) []st...
function generateVolumesForCluster (line 321) | func generateVolumesForCluster(clusterName string, isEmptyDir bool) ([]v...
function setResourceSuffix (line 664) | func setResourceSuffix(value string) string {
FILE: execution/engine/engine.go
type Engine (line 14) | type Engine interface
type RunReceipt (line 30) | type RunReceipt struct
function NewExecutionEngine (line 38) | func NewExecutionEngine(conf config.Config, qm queue.Manager, name strin...
FILE: flotilla/app.go
type App (line 24) | type App struct
method Run (line 36) | func (app *App) Run() error {
method configure (line 107) | func (app *App) configure(conf config.Config) {
method configureRoutes (line 128) | func (app *App) configureRoutes(ep endpoints) {
method initializeEKSWorkers (line 137) | func (app *App) initializeEKSWorkers(
method initializeEMRWorkers (line 154) | func (app *App) initializeEMRWorkers(
function NewApp (line 53) | func NewApp(conf config.Config,
FILE: flotilla/endpoints.go
type endpoints (line 20) | type endpoints struct
method getURLParam (line 39) | func (ep *endpoints) getURLParam(v url.Values, key string, defaultValu...
method getFilters (line 47) | func (ep *endpoints) getFilters(params url.Values, nonFilters map[stri...
method decodeListRequest (line 71) | func (ep *endpoints) decodeListRequest(r *http.Request) listRequest {
method decodeOrderableListRequest (line 91) | func (ep *endpoints) decodeOrderableListRequest(r *http.Request, order...
method decodeRequest (line 108) | func (ep *endpoints) decodeRequest(r *http.Request, entity interface{}...
method encodeError (line 112) | func (ep endpoints) encodeError(w http.ResponseWriter, err error) {
method encodeResponse (line 129) | func (ep *endpoints) encodeResponse(w http.ResponseWriter, response in...
method ListDefinitions (line 134) | func (ep *endpoints) ListDefinitions(w http.ResponseWriter, r *http.Re...
method GetDefinition (line 166) | func (ep *endpoints) GetDefinition(w http.ResponseWriter, r *http.Requ...
method GetDefinitionByAlias (line 183) | func (ep *endpoints) GetDefinitionByAlias(w http.ResponseWriter, r *ht...
method CreateDefinition (line 200) | func (ep *endpoints) CreateDefinition(w http.ResponseWriter, r *http.R...
method UpdateDefinition (line 222) | func (ep *endpoints) UpdateDefinition(w http.ResponseWriter, r *http.R...
method DeleteDefinition (line 247) | func (ep *endpoints) DeleteDefinition(w http.ResponseWriter, r *http.R...
method ListRuns (line 265) | func (ep *endpoints) ListRuns(w http.ResponseWriter, r *http.Request) {
method ListDefinitionRuns (line 292) | func (ep *endpoints) ListDefinitionRuns(w http.ResponseWriter, r *http...
method ListTemplateRuns (line 316) | func (ep *endpoints) ListTemplateRuns(w http.ResponseWriter, r *http.R...
method createListRunsResponse (line 339) | func (ep *endpoints) createListRunsResponse(runList state.RunList, req...
method GetRun (line 355) | func (ep *endpoints) GetRun(w http.ResponseWriter, r *http.Request) {
method GetPayload (line 372) | func (ep *endpoints) GetPayload(w http.ResponseWriter, r *http.Request) {
method CreateRun (line 393) | func (ep *endpoints) CreateRun(w http.ResponseWriter, r *http.Request) {
method CreateRunV2 (line 431) | func (ep *endpoints) CreateRunV2(w http.ResponseWriter, r *http.Reques...
method CreateRunV4 (line 496) | func (ep *endpoints) CreateRunV4(w http.ResponseWriter, r *http.Reques...
method CreateRunByAlias (line 568) | func (ep *endpoints) CreateRunByAlias(w http.ResponseWriter, r *http.R...
method StopRun (line 644) | func (ep *endpoints) StopRun(w http.ResponseWriter, r *http.Request) {
method ExtractUserInfo (line 660) | func (ep *endpoints) ExtractUserInfo(r *http.Request) state.UserInfo {
method UpdateRun (line 679) | func (ep *endpoints) UpdateRun(w http.ResponseWriter, r *http.Request) {
method GetEvents (line 703) | func (ep *endpoints) GetEvents(w http.ResponseWriter, r *http.Request) {
method GetLogs (line 730) | func (ep *endpoints) GetLogs(w http.ResponseWriter, r *http.Request) {
method GetGroups (line 776) | func (ep *endpoints) GetGroups(w http.ResponseWriter, r *http.Request) {
method GetTags (line 784) | func (ep *endpoints) GetTags(w http.ResponseWriter, r *http.Request) {
method ListClusters (line 791) | func (ep *endpoints) ListClusters(w http.ResponseWriter, r *http.Reque...
method ListWorkers (line 804) | func (ep *endpoints) ListWorkers(w http.ResponseWriter, r *http.Reques...
method GetWorker (line 827) | func (ep *endpoints) GetWorker(w http.ResponseWriter, r *http.Request) {
method UpdateWorker (line 838) | func (ep *endpoints) UpdateWorker(w http.ResponseWriter, r *http.Reque...
method BatchUpdateWorkers (line 858) | func (ep *endpoints) BatchUpdateWorkers(w http.ResponseWriter, r *http...
method getStringBoolVal (line 876) | func (ep *endpoints) getStringBoolVal(s string) bool {
method CreateTemplateRunByName (line 887) | func (ep *endpoints) CreateTemplateRunByName(w http.ResponseWriter, r ...
method CreateTemplateRun (line 930) | func (ep *endpoints) CreateTemplateRun(w http.ResponseWriter, r *http....
method ListTemplates (line 972) | func (ep *endpoints) ListTemplates(w http.ResponseWriter, r *http.Requ...
method GetTemplate (line 1011) | func (ep *endpoints) GetTemplate(w http.ResponseWriter, r *http.Reques...
method CreateTemplate (line 1028) | func (ep *endpoints) CreateTemplate(w http.ResponseWriter, r *http.Req...
method GetCluster (line 1050) | func (ep *endpoints) GetCluster(w http.ResponseWriter, r *http.Request) {
method UpdateCluster (line 1061) | func (ep *endpoints) UpdateCluster(w http.ResponseWriter, r *http.Requ...
method DeleteCluster (line 1080) | func (ep *endpoints) DeleteCluster(w http.ResponseWriter, r *http.Requ...
method HealthCheck (line 1091) | func (ep *endpoints) HealthCheck(w http.ResponseWriter, r *http.Reques...
method CreateCluster (line 1099) | func (ep *endpoints) CreateCluster(w http.ResponseWriter, r *http.Requ...
method GetRunStatus (line 1117) | func (ep *endpoints) GetRunStatus(w http.ResponseWriter, r *http.Reque...
type listRequest (line 30) | type listRequest struct
FILE: flotilla/endpoints_test.go
function setUp (line 17) | func setUp(t *testing.T) *muxtrace.Router {
function TestEndpoints_CreateDefinition (line 54) | func TestEndpoints_CreateDefinition(t *testing.T) {
function TestEndpoints_UpdateDefinition (line 84) | func TestEndpoints_UpdateDefinition(t *testing.T) {
function TestEndpoints_CreateRun (line 114) | func TestEndpoints_CreateRun(t *testing.T) {
function TestEndpoints_CreateRun2 (line 148) | func TestEndpoints_CreateRun2(t *testing.T) {
function TestEndpoints_CreateRun4 (line 186) | func TestEndpoints_CreateRun4(t *testing.T) {
function TestEndpoints_CreateRunByAlias (line 229) | func TestEndpoints_CreateRunByAlias(t *testing.T) {
function TestEndpoints_DeleteDefinition (line 267) | func TestEndpoints_DeleteDefinition(t *testing.T) {
function TestEndpoints_GetDefinition (line 294) | func TestEndpoints_GetDefinition(t *testing.T) {
function TestEndpoints_GetDefinitionByAlias (line 326) | func TestEndpoints_GetDefinitionByAlias(t *testing.T) {
function TestEndpoints_GetGroups (line 358) | func TestEndpoints_GetGroups(t *testing.T) {
function TestEndpoints_GetLogs (line 395) | func TestEndpoints_GetLogs(t *testing.T) {
function TestEndpoints_GetRun (line 423) | func TestEndpoints_GetRun(t *testing.T) {
function TestEndpoints_GetRun2 (line 451) | func TestEndpoints_GetRun2(t *testing.T) {
function TestEndpoints_GetTags (line 484) | func TestEndpoints_GetTags(t *testing.T) {
function TestEndpoints_ListDefinitions (line 521) | func TestEndpoints_ListDefinitions(t *testing.T) {
function TestEndpoints_ListRuns (line 597) | func TestEndpoints_ListRuns(t *testing.T) {
function TestEndpoints_StopRun (line 695) | func TestEndpoints_StopRun(t *testing.T) {
function TestEndpoints_ListClusters (line 722) | func TestEndpoints_ListClusters(t *testing.T) {
function TestEndpoints_GetCluster (line 773) | func TestEndpoints_GetCluster(t *testing.T) {
function TestEndpoints_UpdateCluster (line 805) | func TestEndpoints_UpdateCluster(t *testing.T) {
function TestEndpoints_DeleteCluster (line 834) | func TestEndpoints_DeleteCluster(t *testing.T) {
function TestEndpoints_CreateCluster (line 862) | func TestEndpoints_CreateCluster(t *testing.T) {
FILE: flotilla/router.go
function NewRouter (line 8) | func NewRouter(ep endpoints) *muxtrace.Router {
FILE: log/event.go
type EventSink (line 14) | type EventSink interface
type LocalEventSink (line 22) | type LocalEventSink struct
method Receive (line 35) | func (localSink *LocalEventSink) Receive(keyvals ...interface{}) error {
function NewLocalEventSink (line 27) | func NewLocalEventSink() *LocalEventSink {
type HTTPEventSink (line 44) | type HTTPEventSink struct
method headers (line 69) | func (httpsink *HTTPEventSink) headers() map[string]string {
method constructMessage (line 75) | func (httpsink *HTTPEventSink) constructMessage(keyvals ...interface{}...
method Receive (line 98) | func (httpsink *HTTPEventSink) Receive(keyvals ...interface{}) error {
type HTTPEvent (line 54) | type HTTPEvent struct
function NewHTTPSink (line 62) | func NewHTTPSink(host string, path string, method string) HTTPEventSink {
FILE: log/event_test.go
type TestDomainSpecificEvent (line 11) | type TestDomainSpecificEvent struct
function TestHTTPEventSink_Receive (line 19) | func TestHTTPEventSink_Receive(t *testing.T) {
FILE: log/logger.go
type Logger (line 10) | type Logger interface
type logger (line 15) | type logger struct
method Log (line 27) | func (l *logger) Log(keyvals ...interface{}) error {
method Event (line 35) | func (l *logger) Event(keyvals ...interface{}) error {
function NewLogger (line 23) | func NewLogger(wrapped log.Logger, sinks []EventSink) Logger {
FILE: log/logger_test.go
type testLogger (line 7) | type testLogger struct
method Log (line 11) | func (tl *testLogger) Log(keyvals ...interface{}) error {
type testSink (line 16) | type testSink struct
method Receive (line 20) | func (ts *testSink) Receive(keyvals ...interface{}) error {
function TestLogger_Log (line 25) | func TestLogger_Log(t *testing.T) {
function TestLogger_Event (line 42) | func TestLogger_Event(t *testing.T) {
FILE: main.go
function main (line 22) | func main() {
FILE: queue/manager.go
type Manager (line 12) | type Manager interface
type RunReceipt (line 28) | type RunReceipt struct
type StatusReceipt (line 38) | type StatusReceipt struct
function NewQueueManager (line 44) | func NewQueueManager(conf config.Config, name string) (Manager, error) {
FILE: queue/sqs_manager.go
type SQSManager (line 19) | type SQSManager struct
method Name (line 37) | func (qm *SQSManager) Name() string {
method Initialize (line 42) | func (qm *SQSManager) Initialize(conf config.Config, engine string) er...
method QurlFor (line 76) | func (qm *SQSManager) QurlFor(name string, prefixed bool) (string, err...
method getOrCreateQueue (line 90) | func (qm *SQSManager) getOrCreateQueue(name string, prefixed bool) (st...
method messageFromRun (line 115) | func (qm *SQSManager) messageFromRun(run state.Run) (*string, error) {
method runFromMessage (line 124) | func (qm *SQSManager) runFromMessage(message *sqs.Message) (state.Run,...
method statusFromMessage (line 142) | func (qm *SQSManager) statusFromMessage(message *sqs.Message) (string,...
method Enqueue (line 157) | func (qm *SQSManager) Enqueue(ctx context.Context, qURL string, run st...
method ReceiveRun (line 203) | func (qm *SQSManager) ReceiveRun(ctx context.Context, qURL string) (Ru...
method ReceiveStatus (line 269) | func (qm *SQSManager) ReceiveStatus(qURL string) (StatusReceipt, error) {
method ReceiveCloudTrail (line 306) | func (qm *SQSManager) ReceiveCloudTrail(qURL string) (state.CloudTrail...
method ReceiveEMREvent (line 338) | func (qm *SQSManager) ReceiveEMREvent(qURL string) (state.EmrEvent, er...
method ReceiveKubernetesEvent (line 372) | func (qm *SQSManager) ReceiveKubernetesEvent(qURL string) (state.Kuber...
method ReceiveKubernetesRun (line 406) | func (qm *SQSManager) ReceiveKubernetesRun(queue string) (string, erro...
method ack (line 437) | func (qm *SQSManager) ack(qURL string, handle *string) error {
method List (line 456) | func (qm *SQSManager) List() ([]string, error) {
type sqsClient (line 27) | type sqsClient interface
FILE: queue/sqs_manager_test.go
type testSQSClient (line 13) | type testSQSClient struct
method GetQueueUrl (line 19) | func (qc *testSQSClient) GetQueueUrl(input *sqs.GetQueueUrlInput) (*sq...
method CreateQueue (line 33) | func (qc *testSQSClient) CreateQueue(input *sqs.CreateQueueInput) (*sq...
method ListQueues (line 51) | func (qc *testSQSClient) ListQueues(input *sqs.ListQueuesInput) (*sqs....
method SendMessage (line 65) | func (qc *testSQSClient) SendMessage(input *sqs.SendMessageInput) (*sq...
method ReceiveMessage (line 92) | func (qc *testSQSClient) ReceiveMessage(input *sqs.ReceiveMessageInput...
method DeleteMessage (line 129) | func (qc *testSQSClient) DeleteMessage(input *sqs.DeleteMessageInput) ...
function setUp (line 146) | func setUp(t *testing.T) SQSManager {
function TestSQSManager_List (line 167) | func TestSQSManager_List(t *testing.T) {
function TestSQSManager_Enqueue (line 176) | func TestSQSManager_Enqueue(t *testing.T) {
function TestSQSManager_QurlFor (line 191) | func TestSQSManager_QurlFor(t *testing.T) {
function TestSQSManager_ReceiveRun (line 238) | func TestSQSManager_ReceiveRun(t *testing.T) {
function TestSQSManager_ReceiveStatus (line 244) | func TestSQSManager_ReceiveStatus(t *testing.T) {
FILE: services/definition.go
type DefinitionService (line 16) | type DefinitionService interface
type definitionService (line 31) | type definitionService struct
method Create (line 49) | func (ds *definitionService) Create(ctx context.Context, definition *s...
method aliasExists (line 72) | func (ds *definitionService) aliasExists(ctx context.Context, alias st...
method Get (line 92) | func (ds *definitionService) Get(ctx context.Context, definitionID str...
method GetByAlias (line 96) | func (ds *definitionService) GetByAlias(ctx context.Context, alias str...
method List (line 101) | func (ds *definitionService) List(ctx context.Context, limit int, offs...
method Update (line 108) | func (ds *definitionService) Update(ctx context.Context, definitionID ...
method Delete (line 119) | func (ds *definitionService) Delete(ctx context.Context, definitionID ...
method ListGroups (line 123) | func (ds *definitionService) ListGroups(ctx context.Context, limit int...
method ListTags (line 127) | func (ds *definitionService) ListTags(ctx context.Context, limit int, ...
function NewDefinitionService (line 38) | func NewDefinitionService(stateManager state.Manager) (DefinitionService...
FILE: services/definition_test.go
function setUpDefinitionServiceTest (line 10) | func setUpDefinitionServiceTest(t *testing.T) (DefinitionService, *testu...
function TestDefinitionService_Create (line 31) | func TestDefinitionService_Create(t *testing.T) {
function TestDefinitionService_Create2 (line 71) | func TestDefinitionService_Create2(t *testing.T) {
function TestDefinitionService_Update (line 87) | func TestDefinitionService_Update(t *testing.T) {
function TestDefinitionService_Delete (line 108) | func TestDefinitionService_Delete(t *testing.T) {
FILE: services/execution.go
type ExecutionService (line 30) | type ExecutionService interface
type executionService (line 56) | type executionService struct
method GetEvents (line 77) | func (es *executionService) GetEvents(ctx context.Context, run state.R...
method ReservedVariables (line 166) | func (es *executionService) ReservedVariables() []string {
method CreateDefinitionRunByDefinitionID (line 175) | func (es *executionService) CreateDefinitionRunByDefinitionID(ctx cont...
method CreateDefinitionRunByAlias (line 191) | func (es *executionService) CreateDefinitionRunByAlias(ctx context.Con...
method createFromDefinition (line 207) | func (es *executionService) createFromDefinition(ctx context.Context, ...
method constructRunFromDefinition (line 270) | func (es *executionService) constructRunFromDefinition(ctx context.Con...
method constructBaseRunFromExecutable (line 302) | func (es *executionService) constructBaseRunFromExecutable(ctx context...
method constructEnviron (line 408) | func (es *executionService) constructEnviron(run state.Run, env *state...
method List (line 432) | func (es *executionService) List(
method Get (line 469) | func (es *executionService) Get(ctx context.Context, runID string) (st...
method UpdateStatus (line 486) | func (es *executionService) UpdateStatus(ctx context.Context, runID st...
method extractExitReason (line 515) | func (es *executionService) extractExitReason(runExceptions *state.Run...
method terminateWorker (line 546) | func (es *executionService) terminateWorker(jobChan <-chan state.Termi...
method Terminate (line 604) | func (es *executionService) Terminate(ctx context.Context, runID strin...
method ListClusters (line 617) | func (es *executionService) ListClusters(ctx context.Context) ([]state...
method GetDefaultCluster (line 627) | func (es *executionService) GetDefaultCluster() string {
method sanitizeExecutionRequestCommonFields (line 632) | func (es *executionService) sanitizeExecutionRequestCommonFields(field...
method createAndEnqueueRun (line 651) | func (es *executionService) createAndEnqueueRun(ctx context.Context, r...
method CreateTemplateRunByTemplateName (line 690) | func (es *executionService) CreateTemplateRunByTemplateName(ctx contex...
method CreateTemplateRunByTemplateID (line 714) | func (es *executionService) CreateTemplateRunByTemplateID(ctx context....
method createFromTemplate (line 727) | func (es *executionService) createFromTemplate(ctx context.Context, te...
method constructRunFromTemplate (line 747) | func (es *executionService) constructRunFromTemplate(ctx context.Conte...
method resolveRequestTier (line 763) | func (es *executionService) resolveRequestTier(requestedTier state.Tie...
method clusterSupportsTier (line 771) | func (es *executionService) clusterSupportsTier(cluster state.ClusterM...
method isClusterValid (line 782) | func (es *executionService) isClusterValid(clusterName string) bool {
method UpdateClusterMetadata (line 786) | func (es *executionService) UpdateClusterMetadata(ctx context.Context,...
method DeleteClusterMetadata (line 793) | func (es *executionService) DeleteClusterMetadata(ctx context.Context,...
method GetClusterByID (line 800) | func (es *executionService) GetClusterByID(ctx context.Context, cluste...
method GetRunStatus (line 808) | func (es *executionService) GetRunStatus(ctx context.Context, runID st...
function NewExecutionService (line 85) | func NewExecutionService(conf config.Config, eksExecutionEngine engine.E...
FILE: services/execution_test.go
function setUp (line 16) | func setUp(t *testing.T) (ExecutionService, *testutils.ImplementsAllTheT...
function TestExecutionService_CreateDefinitionRunByDefinitionID (line 47) | func TestExecutionService_CreateDefinitionRunByDefinitionID(t *testing.T) {
function TestExecutionService_CreateDefinitionRunByAlias (line 167) | func TestExecutionService_CreateDefinitionRunByAlias(t *testing.T) {
function TestExecutionService_List (line 265) | func TestExecutionService_List(t *testing.T) {
function TestExecutionService_List2 (line 286) | func TestExecutionService_List2(t *testing.T) {
function TestExecutionService_ListClusters (line 310) | func TestExecutionService_ListClusters(t *testing.T) {
function TestExecutionService_CreateDefinitionRunWithTier (line 335) | func TestExecutionService_CreateDefinitionRunWithTier(t *testing.T) {
function TestExecutionService_GetRunStatus (line 463) | func TestExecutionService_GetRunStatus(t *testing.T) {
function TestExecutionService_CommandHashCalculatedFromCommand (line 515) | func TestExecutionService_CommandHashCalculatedFromCommand(t *testing.T) {
function TestExecutionService_CommandHashWithSameDescriptionDifferentCommands (line 553) | func TestExecutionService_CommandHashWithSameDescriptionDifferentCommand...
function TestExecutionService_CommandHashNullWhenCommandNull (line 619) | func TestExecutionService_CommandHashNullWhenCommandNull(t *testing.T) {
function TestExecutionService_CommandHashMatchesCommand (line 660) | func TestExecutionService_CommandHashMatchesCommand(t *testing.T) {
function TestExecutionService_CommandHashStableAcrossRuns (line 704) | func TestExecutionService_CommandHashStableAcrossRuns(t *testing.T) {
function TestExecutionService_CommandHashNotSetInEndpoints (line 746) | func TestExecutionService_CommandHashNotSetInEndpoints(t *testing.T) {
function TestExecutionService_CommandHashWithOverride (line 787) | func TestExecutionService_CommandHashWithOverride(t *testing.T) {
function TestExecutionService_SparkCommandHashFromDescription (line 822) | func TestExecutionService_SparkCommandHashFromDescription(t *testing.T) {
function TestExecutionService_SparkCommandHashConsistent (line 860) | func TestExecutionService_SparkCommandHashConsistent(t *testing.T) {
function TestExecutionService_SparkVsRegularEKSHashing (line 909) | func TestExecutionService_SparkVsRegularEKSHashing(t *testing.T) {
function TestExecutionService_SparkNullDescriptionNullHash (line 981) | func TestExecutionService_SparkNullDescriptionNullHash(t *testing.T) {
FILE: services/logs.go
type LogService (line 11) | type LogService interface
type logService (line 16) | type logService struct
method Logs (line 27) | func (ls *logService) Logs(runID string, lastSeen *string, role *strin...
method LogsText (line 52) | func (ls *logService) LogsText(runID string, w http.ResponseWriter) er...
function NewLogService (line 22) | func NewLogService(sm state.Manager, lc logs.Client) (LogService, error) {
FILE: services/logs_test.go
function setUpLogServiceTest (line 10) | func setUpLogServiceTest(t *testing.T) (LogService, *testutils.Implement...
function TestLogService_Logs (line 25) | func TestLogService_Logs(t *testing.T) {
FILE: services/template.go
type TemplateService (line 14) | type TemplateService interface
type templateService (line 22) | type templateService struct
method Create (line 33) | func (ts *templateService) Create(ctx context.Context, req *state.Crea...
method GetByID (line 85) | func (ts *templateService) GetByID(ctx context.Context, id string) (st...
method GetLatestByName (line 90) | func (ts *templateService) GetLatestByName(ctx context.Context, templa...
method List (line 95) | func (ts *templateService) List(ctx context.Context, limit int, offset...
method ListLatestOnly (line 100) | func (ts *templateService) ListLatestOnly(ctx context.Context, limit i...
method diff (line 106) | func (ts *templateService) diff(prev state.Template, curr state.Templa...
method constructTemplateFromCreateTemplateRequest (line 188) | func (ts *templateService) constructTemplateFromCreateTemplateRequest(...
function NewTemplateService (line 27) | func NewTemplateService(conf config.Config, sm state.Manager) (TemplateS...
FILE: services/worker.go
type WorkerService (line 14) | type WorkerService interface
type workerService (line 21) | type workerService struct
method List (line 33) | func (ws *workerService) List(ctx context.Context, engine string) (sta...
method Get (line 37) | func (ws *workerService) Get(ctx context.Context, workerType string, e...
method Update (line 45) | func (ws *workerService) Update(ctx context.Context, workerType string...
method BatchUpdate (line 54) | func (ws *workerService) BatchUpdate(ctx context.Context, updates []st...
method validate (line 64) | func (ws *workerService) validate(workerType string) error {
function NewWorkerService (line 28) | func NewWorkerService(conf config.Config, sm state.Manager) (WorkerServi...
FILE: state/manager.go
type Manager (line 12) | type Manager interface
function NewStateManager (line 71) | func NewStateManager(conf config.Config, logger log.Logger) (Manager, er...
FILE: state/models.go
function IsValidWorkerType (line 90) | func IsValidWorkerType(workerType string) bool {
function IsValidStatus (line 96) | func IsValidStatus(status string) bool {
function NewRunID (line 105) | func NewRunID(engine *string) (string, error) {
function NewDefinitionID (line 111) | func NewDefinitionID(definition Definition) (string, error) {
function newUUIDv4 (line 119) | func newUUIDv4() (string, error) {
type EnvList (line 130) | type EnvList
type PortsList (line 135) | type PortsList
type EnvVar (line 139) | type EnvVar struct
type NodeList (line 144) | type NodeList
type Tags (line 149) | type Tags
type ExecutableResources (line 153) | type ExecutableResources struct
type ExecutableType (line 165) | type ExecutableType
constant ExecutableTypeDefinition (line 168) | ExecutableTypeDefinition ExecutableType = "task_definition"
constant ExecutableTypeTemplate (line 169) | ExecutableTypeTemplate ExecutableType = "template"
type Executable (line 172) | type Executable interface
function UnmarshalSparkExtension (line 180) | func UnmarshalSparkExtension(data []byte) (SparkExtension, error) {
type SparkExtension (line 190) | type SparkExtension struct
method Marshal (line 186) | func (r *SparkExtension) Marshal() ([]byte, error) {
type Conf (line 210) | type Conf struct
type SparkSubmitJobDriver (line 215) | type SparkSubmitJobDriver struct
type Labels (line 228) | type Labels
type ExecutionRequestCommon (line 231) | type ExecutionRequestCommon struct
type ExecutionRequestCustom (line 253) | type ExecutionRequestCustom
type ExecutionRequest (line 254) | type ExecutionRequest interface
type DefinitionExecutionRequest (line 259) | type DefinitionExecutionRequest struct
method GetExecutionRequestCommon (line 264) | func (d *DefinitionExecutionRequest) GetExecutionRequestCommon() *Exec...
method GetExecutionRequestCustom (line 269) | func (d *DefinitionExecutionRequest) GetExecutionRequestCustom() *Exec...
type TerminateJob (line 273) | type TerminateJob struct
type Definition (line 279) | type Definition struct
method GetExecutableID (line 291) | func (d Definition) GetExecutableID() *string {
method GetExecutableType (line 296) | func (d Definition) GetExecutableType() *ExecutableType {
method GetExecutableResources (line 300) | func (d Definition) GetExecutableResources() *ExecutableResources {
method GetExecutableCommand (line 304) | func (d Definition) GetExecutableCommand(req ExecutionRequest) (string...
method GetExecutableResourceName (line 308) | func (d Definition) GetExecutableResourceName() string {
method WrappedCommand (line 322) | func (d *Definition) WrappedCommand() (string, error) {
method IsValid (line 337) | func (d *Definition) IsValid() (bool, []string) {
method UpdateWith (line 355) | func (d *Definition) UpdateWith(other Definition) {
method MarshalJSON (line 400) | func (d Definition) MarshalJSON() ([]byte, error) {
type validationCondition (line 330) | type validationCondition struct
type DefinitionList (line 418) | type DefinitionList struct
method MarshalJSON (line 423) | func (dl *DefinitionList) MarshalJSON() ([]byte, error) {
type Run (line 448) | type Run struct
method UpdateWith (line 501) | func (d *Run) UpdateWith(other Run) {
method MarshalJSON (line 745) | func (r Run) MarshalJSON() ([]byte, error) {
function removeDuplicateStr (line 694) | func removeDuplicateStr(strSlice []string) []string {
type byExecutorName (line 706) | type byExecutorName
method Len (line 722) | func (s byExecutorName) Len() int {
method Key (line 725) | func (s byExecutorName) Key(i int) int {
method Swap (line 737) | func (s byExecutorName) Swap(i, j int) {
method Less (line 741) | func (s byExecutorName) Less(i, j int) bool {
type RunStatus (line 708) | type RunStatus struct
type RunList (line 816) | type RunList struct
type PodEvents (line 821) | type PodEvents
type PodEventList (line 823) | type PodEventList struct
type SpawnedRun (line 828) | type SpawnedRun struct
type SpawnedRuns (line 832) | type SpawnedRuns
type RunExceptions (line 834) | type RunExceptions
type PodEvent (line 845) | type PodEvent struct
method Equal (line 836) | func (w *PodEvent) Equal(other PodEvent) bool {
type GroupsList (line 854) | type GroupsList struct
type TagsList (line 860) | type TagsList struct
type Worker (line 866) | type Worker struct
method UpdateWith (line 873) | func (w *Worker) UpdateWith(other Worker) {
type WorkersList (line 880) | type WorkersList struct
type UserInfo (line 886) | type UserInfo struct
type TaskResources (line 892) | type TaskResources struct
type CloudTrailS3File (line 898) | type CloudTrailS3File struct
type CloudTrailNotifications (line 910) | type CloudTrailNotifications struct
method Marshal (line 905) | func (e *CloudTrailNotifications) Marshal() ([]byte, error) {
type Record (line 915) | type Record struct
method Equal (line 927) | func (w *Record) Equal(other Record) bool {
method String (line 932) | func (w *Record) String() string {
type UserIdentity (line 922) | type UserIdentity struct
constant TemplatePayloadKey (line 936) | TemplatePayloadKey = "template_payload"
type TemplatePayload (line 938) | type TemplatePayload
type TemplateExecutionRequest (line 940) | type TemplateExecutionRequest struct
method GetExecutionRequestCommon (line 947) | func (t TemplateExecutionRequest) GetExecutionRequestCommon() *Executi...
method GetExecutionRequestCustom (line 952) | func (t TemplateExecutionRequest) GetExecutionRequestCustom() *Executi...
type TemplateJSONSchema (line 959) | type TemplateJSONSchema
type Template (line 962) | type Template struct
method GetExecutableID (line 988) | func (t Template) GetExecutableID() *string {
method GetExecutableType (line 993) | func (t Template) GetExecutableType() *ExecutableType {
method GetExecutableResources (line 999) | func (t Template) GetExecutableResources() *ExecutableResources {
method GetExecutableCommand (line 1004) | func (t Template) GetExecutableCommand(req ExecutionRequest) (string, ...
method GetExecutableResourceName (line 1051) | func (t Template) GetExecutableResourceName() string {
method compositeUserAndDefaults (line 1055) | func (t Template) compositeUserAndDefaults(userPayload interface{}) (T...
method IsValid (line 1085) | func (t *Template) IsValid() (bool, []string) {
type CreateTemplateRequest (line 973) | type CreateTemplateRequest struct
type CreateTemplateResponse (line 982) | type CreateTemplateResponse struct
function NewTemplateID (line 1076) | func NewTemplateID(t Template) (string, error) {
type TemplateList (line 1106) | type TemplateList struct
method MarshalJSON (line 1112) | func (tl *TemplateList) MarshalJSON() ([]byte, error) {
type KubernetesEvent (line 1131) | type KubernetesEvent struct
method Marshal (line 1127) | func (r *KubernetesEvent) Marshal() ([]byte, error) {
type InvolvedObject (line 1147) | type InvolvedObject struct
type EventLabels (line 1158) | type EventLabels struct
type Metadata (line 1164) | type Metadata struct
type Source (line 1173) | type Source struct
function UnmarshalEmrEvents (line 1178) | func UnmarshalEmrEvents(data []byte) (EmrEvent, error) {
type EmrEvent (line 1188) | type EmrEvent struct
method Marshal (line 1184) | func (r *EmrEvent) Marshal() ([]byte, error) {
type Detail (line 1201) | type Detail struct
type LaunchRequest (line 1216) | type LaunchRequest struct
type LaunchRequestV2 (line 1222) | type LaunchRequestV2 struct
type RunTags (line 1245) | type RunTags struct
type ClusterStatus (line 1251) | type ClusterStatus
type Tier (line 1252) | type Tier
type Tiers (line 1253) | type Tiers
type Capability (line 1254) | type Capability
type Capabilities (line 1255) | type Capabilities
constant StatusActive (line 1258) | StatusActive ClusterStatus = "active"
constant StatusMaintenance (line 1259) | StatusMaintenance ClusterStatus = "maintenance"
constant StatusOffline (line 1260) | StatusOffline ClusterStatus = "offline"
type ClusterMetadata (line 1263) | type ClusterMetadata struct
function MergeMaps (line 1283) | func MergeMaps(a *map[string]interface{}, b map[string]interface{}) error {
function mergeMapsRecursive (line 1287) | func mergeMapsRecursive(a *map[string]interface{}, b map[string]interfac...
function GetLabels (line 1315) | func GetLabels(run Run) map[string]string {
function SanitizeLabel (line 1348) | func SanitizeLabel(key string) string {
FILE: state/models_test.go
function TestMergeMaps_Simple (line 10) | func TestMergeMaps_Simple(t *testing.T) {
function TestMergeMaps_Nested (line 39) | func TestMergeMaps_Nested(t *testing.T) {
function TestSanitizeLabel (line 92) | func TestSanitizeLabel(t *testing.T) {
function TestGetLabels (line 149) | func TestGetLabels(t *testing.T) {
FILE: state/pg_queries.go
constant DefinitionSelect (line 4) | DefinitionSelect = `
constant ListDefinitionsSQL (line 25) | ListDefinitionsSQL = DefinitionSelect + "\n%s %s limit $1 offset $2"
constant ListClusterStatesSQL (line 29) | ListClusterStatesSQL = `
constant GetDefinitionSQL (line 49) | GetDefinitionSQL = DefinitionSelect + "\nwhere definition_id = $1"
constant GetDefinitionByAliasSQL (line 52) | GetDefinitionByAliasSQL = DefinitionSelect + "\nwhere alias = $1"
constant TaskResourcesSelectCommandSQL (line 54) | TaskResourcesSelectCommandSQL = `
constant TaskResourcesExecutorCountSQL (line 68) | TaskResourcesExecutorCountSQL = `
constant TaskResourcesDriverOOMSQL (line 83) | TaskResourcesDriverOOMSQL = `
constant TaskIdempotenceKeyCheckSQL (line 95) | TaskIdempotenceKeyCheckSQL = `
constant TaskResourcesExecutorOOMSQL (line 107) | TaskResourcesExecutorOOMSQL = `
constant TaskResourcesExecutorNodeLifecycleSQL (line 120) | TaskResourcesExecutorNodeLifecycleSQL = `
constant TaskExecutionRuntimeCommandSQL (line 132) | TaskExecutionRuntimeCommandSQL = `
constant ListFailingNodesSQL (line 144) | ListFailingNodesSQL = `
constant PodReAttemptRate (line 162) | PodReAttemptRate = `
constant RunSelect (line 174) | RunSelect = `
constant GetRunStatusSQL (line 226) | GetRunStatusSQL = `
constant ListRunsSQL (line 244) | ListRunsSQL = RunSelect + "\n%s %s limit $1 offset $2"
constant GetRunSQL (line 247) | GetRunSQL = RunSelect + "\nwhere run_id = $1"
constant GetRunSQLByEMRJobId (line 249) | GetRunSQLByEMRJobId = RunSelect + "\nwhere spark_extension->>'emr_job_id...
constant GetRunSQLForUpdate (line 253) | GetRunSQLForUpdate = GetRunSQL + " for update"
constant GroupsSelect (line 257) | GroupsSelect = `
constant TagsSelect (line 262) | TagsSelect = `
constant ListGroupsSQL (line 267) | ListGroupsSQL = GroupsSelect + "\n%s order by group_name asc limit $1 of...
constant ListTagsSQL (line 270) | ListTagsSQL = TagsSelect + "\n%s order by text asc limit $1 offset $2"
constant WorkerSelect (line 273) | WorkerSelect = `
constant ListWorkersSQL (line 282) | ListWorkersSQL = WorkerSelect
constant GetWorkerEngine (line 284) | GetWorkerEngine = WorkerSelect + "\nwhere engine = $1"
constant GetWorkerSQL (line 288) | GetWorkerSQL = WorkerSelect + "\nwhere worker_type = $1 and engine = $2"
constant GetWorkerSQLForUpdate (line 292) | GetWorkerSQLForUpdate = GetWorkerSQL + " for update"
constant TemplateSelect (line 295) | TemplateSelect = `
constant ListTemplatesSQL (line 315) | ListTemplatesSQL = TemplateSelect + "\n%s limit $1 offset $2"
constant GetTemplateByIDSQL (line 318) | GetTemplateByIDSQL = TemplateSelect + "\nwhere template_id = $1"
constant ListTemplatesLatestOnlySQL (line 322) | ListTemplatesLatestOnlySQL = `
constant GetTemplateLatestOnlySQL (line 344) | GetTemplateLatestOnlySQL = TemplateSelect + "\nWHERE template_name = $1 ...
constant GetTemplateByVersionSQL (line 345) | GetTemplateByVersionSQL = TemplateSelect + "\nWHERE template_name = $1 A...
FILE: state/pg_state_manager.go
type SQLStateManager (line 32) | type SQLStateManager struct
method ListFailingNodes (line 38) | func (sm *SQLStateManager) ListFailingNodes(ctx context.Context) (Node...
method GetPodReAttemptRate (line 58) | func (sm *SQLStateManager) GetPodReAttemptRate(ctx context.Context) (f...
method GetNodeLifecycle (line 77) | func (sm *SQLStateManager) GetNodeLifecycle(ctx context.Context, execu...
method GetTaskHistoricalRuntime (line 97) | func (sm *SQLStateManager) GetTaskHistoricalRuntime(ctx context.Contex...
method EstimateRunResources (line 118) | func (sm *SQLStateManager) EstimateRunResources(ctx context.Context, e...
method EstimateExecutorCount (line 207) | func (sm *SQLStateManager) EstimateExecutorCount(ctx context.Context, ...
method CheckIdempotenceKey (line 227) | func (sm *SQLStateManager) CheckIdempotenceKey(ctx context.Context, id...
method ExecutorOOM (line 241) | func (sm *SQLStateManager) ExecutorOOM(ctx context.Context, executable...
method DriverOOM (line 262) | func (sm *SQLStateManager) DriverOOM(ctx context.Context, executableID...
method Name (line 284) | func (sm *SQLStateManager) Name() string {
method Initialize (line 300) | func (sm *SQLStateManager) Initialize(conf config.Config) error {
method makeWhereClause (line 346) | func (sm *SQLStateManager) makeWhereClause(filters map[string][]string...
method makeEnvWhereClause (line 376) | func (sm *SQLStateManager) makeEnvWhereClause(filters map[string]strin...
method orderBy (line 388) | func (sm *SQLStateManager) orderBy(obj IOrderable, field string, order...
method ListDefinitions (line 407) | func (sm *SQLStateManager) ListDefinitions(
method GetDefinition (line 445) | func (sm *SQLStateManager) GetDefinition(ctx context.Context, definiti...
method GetDefinitionByAlias (line 464) | func (sm *SQLStateManager) GetDefinitionByAlias(ctx context.Context, a...
method UpdateDefinition (line 486) | func (sm *SQLStateManager) UpdateDefinition(ctx context.Context, defin...
method CreateDefinition (line 599) | func (sm *SQLStateManager) CreateDefinition(ctx context.Context, d Def...
method DeleteDefinition (line 694) | func (sm *SQLStateManager) DeleteDefinition(ctx context.Context, defin...
method ListRuns (line 731) | func (sm *SQLStateManager) ListRuns(ctx context.Context, limit int, of...
method GetRun (line 773) | func (sm *SQLStateManager) GetRun(ctx context.Context, runID string) (...
method GetRunByEMRJobId (line 802) | func (sm *SQLStateManager) GetRunByEMRJobId(ctx context.Context, emrJo...
method GetResources (line 829) | func (sm *SQLStateManager) GetResources(ctx context.Context, runID str...
method UpdateRun (line 857) | func (sm *SQLStateManager) UpdateRun(ctx context.Context, runID string...
method CreateRun (line 1058) | func (sm *SQLStateManager) CreateRun(ctx context.Context, r Run) error {
method ListGroups (line 1234) | func (sm *SQLStateManager) ListGroups(ctx context.Context, limit int, ...
method ListTags (line 1263) | func (sm *SQLStateManager) ListTags(ctx context.Context, limit int, of...
method initWorkerTable (line 1294) | func (sm *SQLStateManager) initWorkerTable(c config.Config) error {
method ListWorkers (line 1339) | func (sm *SQLStateManager) ListWorkers(ctx context.Context, engine str...
method GetWorker (line 1363) | func (sm *SQLStateManager) GetWorker(ctx context.Context, workerType s...
method UpdateWorker (line 1381) | func (sm *SQLStateManager) UpdateWorker(ctx context.Context, workerTyp...
method BatchUpdateWorkers (line 1436) | func (sm *SQLStateManager) BatchUpdateWorkers(ctx context.Context, upd...
method Cleanup (line 1455) | func (sm *SQLStateManager) Cleanup() error {
method GetTemplateByID (line 1694) | func (sm *SQLStateManager) GetTemplateByID(ctx context.Context, templa...
method GetTemplateByVersion (line 1713) | func (sm *SQLStateManager) GetTemplateByVersion(ctx context.Context, t...
method GetLatestTemplateByTemplateName (line 1734) | func (sm *SQLStateManager) GetLatestTemplateByTemplateName(ctx context...
method ListTemplates (line 1753) | func (sm *SQLStateManager) ListTemplates(ctx context.Context, limit in...
method ListTemplatesLatestOnly (line 1783) | func (sm *SQLStateManager) ListTemplatesLatestOnly(ctx context.Context...
method CreateTemplate (line 1806) | func (sm *SQLStateManager) CreateTemplate(ctx context.Context, t Templ...
method GetExecutableByTypeAndID (line 1846) | func (sm *SQLStateManager) GetExecutableByTypeAndID(ctx context.Contex...
method logStatusUpdate (line 1865) | func (sm *SQLStateManager) logStatusUpdate(update Run) {
method ListClusterStates (line 1940) | func (sm *SQLStateManager) ListClusterStates(ctx context.Context) ([]C...
method UpdateClusterMetadata (line 1953) | func (sm *SQLStateManager) UpdateClusterMetadata(ctx context.Context, ...
method DeleteClusterMetadata (line 2051) | func (sm *SQLStateManager) DeleteClusterMetadata(ctx context.Context, ...
method GetClusterByID (line 2080) | func (sm *SQLStateManager) GetClusterByID(ctx context.Context, cluster...
method GetRunStatus (line 2238) | func (sm *SQLStateManager) GetRunStatus(ctx context.Context, runID str...
type IOrderable (line 1459) | type IOrderable interface
method ValidOrderField (line 1465) | func (d *Definition) ValidOrderField(field string) bool {
method ValidOrderFields (line 1474) | func (d *Definition) ValidOrderFields() []string {
method DefaultOrderField (line 1478) | func (d *Definition) DefaultOrderField() string {
method ValidOrderField (line 1482) | func (r *Run) ValidOrderField(field string) bool {
method ValidOrderFields (line 1491) | func (r *Run) ValidOrderFields() []string {
method DefaultOrderField (line 1495) | func (r *Run) DefaultOrderField() string {
method ValidOrderField (line 1499) | func (t *Template) ValidOrderField(field string) bool {
method ValidOrderFields (line 1508) | func (t *Template) ValidOrderFields() []string {
method DefaultOrderField (line 1513) | func (t *Template) DefaultOrderField() string {
method Scan (line 1518) | func (e *EnvList) Scan(value interface{}) error {
method Value (line 1527) | func (e *EnvList) Value() (driver.Value, error) {
method Scan (line 1533) | func (e *PodEvents) Scan(value interface{}) error {
method Value (line 1542) | func (e SpawnedRuns) Value() (driver.Value, error) {
method Scan (line 1547) | func (e *SpawnedRuns) Scan(value interface{}) error {
method Value (line 1556) | func (e SparkExtension) Value() (driver.Value, error) {
method Scan (line 1561) | func (e *SparkExtension) Scan(value interface{}) error {
method Value (line 1570) | func (e RunExceptions) Value() (driver.Value, error) {
method Scan (line 1575) | func (e *RunExceptions) Scan(value interface{}) error {
method Value (line 1584) | func (e PodEvents) Value() (driver.Value, error) {
method Scan (line 1590) | func (e *PortsList) Scan(value interface{}) error {
method Value (line 1599) | func (e PortsList) Value() (driver.Value, error) {
method Scan (line 1605) | func (e *Tags) Scan(value interface{}) error {
method Value (line 1614) | func (e Tags) Value() (driver.Value, error) {
method Scan (line 1620) | func (e *CloudTrailNotifications) Scan(value interface{}) error {
method Value (line 1629) | func (e CloudTrailNotifications) Value() (driver.Value, error) {
method Scan (line 1635) | func (e *ExecutionRequestCustom) Scan(value interface{}) error {
method Value (line 1644) | func (e ExecutionRequestCustom) Value() (driver.Value, error) {
method Scan (line 1650) | func (tjs *TemplateJSONSchema) Scan(value interface{}) error {
method Value (line 1659) | func (tjs TemplateJSONSchema) Value() (driver.Value, error) {
method Scan (line 1665) | func (tjs *TemplatePayload) Scan(value interface{}) error {
method Value (line 1674) | func (tjs TemplatePayload) Value() (driver.Value, error) {
method Value (line 1680) | func (e Labels) Value() (driver.Value, error) {
method Scan (line 1685) | func (e *Labels) Scan(value interface{}) error {
function ScanStringArray (line 2114) | func ScanStringArray(arr *[]string, value interface{}) error {
method Scan (line 2147) | func (arr *Tiers) Scan(value interface{}) error {
method Value (line 2185) | func (arr Tiers) Value() (driver.Value, error) {
method Scan (line 2197) | func (arr *Capabilities) Scan(value interface{}) error {
method Value (line 2231) | func (arr Capabilities) Value() (driver.Value, error) {
FILE: state/pg_state_manager_test.go
function getDB (line 22) | func getDB(conf config.Config) *sqlx.DB {
function setUp (line 34) | func setUp() Manager {
function insertDefinitions (line 59) | func insertDefinitions(db *sqlx.DB) {
function tearDown (line 129) | func tearDown() {
function TestSQLStateManager_ListDefinitions (line 142) | func TestSQLStateManager_ListDefinitions(t *testing.T) {
function TestSQLStateManager_GetDefinition (line 202) | func TestSQLStateManager_GetDefinition(t *testing.T) {
function TestSQLStateManager_GetDefinitionByAlias (line 221) | func TestSQLStateManager_GetDefinitionByAlias(t *testing.T) {
function TestSQLStateManager_CreateDefinition (line 240) | func TestSQLStateManager_CreateDefinition(t *testing.T) {
function TestSQLStateManager_UpdateDefinition (line 280) | func TestSQLStateManager_UpdateDefinition(t *testing.T) {
function TestSQLStateManager_DeleteDefinition (line 331) | func TestSQLStateManager_DeleteDefinition(t *testing.T) {
function TestSQLStateManager_ListRuns (line 347) | func TestSQLStateManager_ListRuns(t *testing.T) {
function TestSQLStateManager_ListRuns2 (line 415) | func TestSQLStateManager_ListRuns2(t *testing.T) {
function TestSQLStateManager_ListRuns3 (line 444) | func TestSQLStateManager_ListRuns3(t *testing.T) {
function TestSQLStateManager_GetRun (line 472) | func TestSQLStateManager_GetRun(t *testing.T) {
function TestSQLStateManager_CreateRun (line 491) | func TestSQLStateManager_CreateRun(t *testing.T) {
function TestSQLStateManager_UpdateRun (line 608) | func TestSQLStateManager_UpdateRun(t *testing.T) {
function TestSQLStateManager_UpdateWorker (line 691) | func TestSQLStateManager_UpdateWorker(t *testing.T) {
function TestSQLStateManager_ListClusterStates (line 738) | func TestSQLStateManager_ListClusterStates(t *testing.T) {
function TestStringArray_Scan (line 749) | func TestStringArray_Scan(t *testing.T) {
function TestStringArray_Value (line 817) | func TestStringArray_Value(t *testing.T) {
function TestStringArray_RoundTrip (line 862) | func TestStringArray_RoundTrip(t *testing.T) {
function TestCapabilities_Scan (line 907) | func TestCapabilities_Scan(t *testing.T) {
function TestCapabilities_Value (line 969) | func TestCapabilities_Value(t *testing.T) {
function TestCapabilities_RoundTrip (line 1012) | func TestCapabilities_RoundTrip(t *testing.T) {
function tearDownClusters (line 1062) | func tearDownClusters() {
function TestSQLStateManager_UpdateClusterMetadata (line 1070) | func TestSQLStateManager_UpdateClusterMetadata(t *testing.T) {
function TestSQLStateManager_DeleteClusterMetadata (line 1137) | func TestSQLStateManager_DeleteClusterMetadata(t *testing.T) {
FILE: testutils/mocks.go
type ImplementsAllTheThings (line 20) | type ImplementsAllTheThings struct
method GetResources (line 40) | func (iatt *ImplementsAllTheThings) GetResources(ctx context.Context, ...
method ListClusters (line 49) | func (iatt *ImplementsAllTheThings) ListClusters() ([]state.ClusterMet...
method ListClusterStates (line 54) | func (i *ImplementsAllTheThings) ListClusterStates(ctx context.Context...
method GetClusterByID (line 60) | func (i *ImplementsAllTheThings) GetClusterByID(ctx context.Context, c...
method DeleteClusterMetadata (line 65) | func (i *ImplementsAllTheThings) DeleteClusterMetadata(ctx context.Con...
method UpdateClusterMetadata (line 70) | func (i *ImplementsAllTheThings) UpdateClusterMetadata(ctx context.Con...
method LogsText (line 75) | func (iatt *ImplementsAllTheThings) LogsText(executable state.Executab...
method Log (line 80) | func (iatt *ImplementsAllTheThings) Log(keyvals ...interface{}) error {
method Event (line 85) | func (iatt *ImplementsAllTheThings) Event(keyvals ...interface{}) error {
method Name (line 91) | func (iatt *ImplementsAllTheThings) Name() string {
method Initialize (line 97) | func (iatt *ImplementsAllTheThings) Initialize(conf config.Config) err...
method Cleanup (line 103) | func (iatt *ImplementsAllTheThings) Cleanup() error {
method ListFailingNodes (line 108) | func (iatt *ImplementsAllTheThings) ListFailingNodes(ctx context.Conte...
method GetPodReAttemptRate (line 114) | func (iatt *ImplementsAllTheThings) GetPodReAttemptRate(ctx context.Co...
method GetNodeLifecycle (line 119) | func (iatt *ImplementsAllTheThings) GetNodeLifecycle(ctx context.Conte...
method GetTaskHistoricalRuntime (line 124) | func (iatt *ImplementsAllTheThings) GetTaskHistoricalRuntime(ctx conte...
method ListDefinitions (line 130) | func (iatt *ImplementsAllTheThings) ListDefinitions(
method GetDefinition (line 144) | func (iatt *ImplementsAllTheThings) GetDefinition(ctx context.Context,...
method GetDefinitionByAlias (line 155) | func (iatt *ImplementsAllTheThings) GetDefinitionByAlias(ctx context.C...
method UpdateDefinition (line 166) | func (iatt *ImplementsAllTheThings) UpdateDefinition(ctx context.Conte...
method CreateDefinition (line 175) | func (iatt *ImplementsAllTheThings) CreateDefinition(ctx context.Conte...
method DeleteDefinition (line 182) | func (iatt *ImplementsAllTheThings) DeleteDefinition(ctx context.Conte...
method ListRuns (line 189) | func (iatt *ImplementsAllTheThings) ListRuns(ctx context.Context, limi...
method GetRun (line 199) | func (iatt *ImplementsAllTheThings) GetRun(ctx context.Context, runID ...
method GetRunByEMRJobId (line 209) | func (iatt *ImplementsAllTheThings) GetRunByEMRJobId(ctx context.Conte...
method CreateRun (line 220) | func (iatt *ImplementsAllTheThings) CreateRun(ctx context.Context, r s...
method EstimateRunResources (line 226) | func (iatt *ImplementsAllTheThings) EstimateRunResources(ctx context.C...
method EstimateExecutorCount (line 231) | func (iatt *ImplementsAllTheThings) EstimateExecutorCount(ctx context....
method ExecutorOOM (line 236) | func (iatt *ImplementsAllTheThings) ExecutorOOM(ctx context.Context, e...
method DriverOOM (line 240) | func (iatt *ImplementsAllTheThings) DriverOOM(ctx context.Context, exe...
method UpdateRun (line 246) | func (iatt *ImplementsAllTheThings) UpdateRun(ctx context.Context, run...
method ListGroups (line 255) | func (iatt *ImplementsAllTheThings) ListGroups(ctx context.Context, li...
method ListTags (line 261) | func (iatt *ImplementsAllTheThings) ListTags(ctx context.Context, limi...
method initWorkerTable (line 267) | func (iatt *ImplementsAllTheThings) initWorkerTable(c config.Config) e...
method ListWorkers (line 273) | func (iatt *ImplementsAllTheThings) ListWorkers(ctx context.Context, e...
method CheckIdempotenceKey (line 278) | func (iatt *ImplementsAllTheThings) CheckIdempotenceKey(ctx context.Co...
method GetWorker (line 284) | func (iatt *ImplementsAllTheThings) GetWorker(ctx context.Context, wor...
method UpdateWorker (line 290) | func (iatt *ImplementsAllTheThings) UpdateWorker(ctx context.Context, ...
method BatchUpdateWorkers (line 296) | func (iatt *ImplementsAllTheThings) BatchUpdateWorkers(ctx context.Con...
method QurlFor (line 302) | func (iatt *ImplementsAllTheThings) QurlFor(name string, prefixed bool...
method Enqueue (line 308) | func (iatt *ImplementsAllTheThings) Enqueue(ctx context.Context, run s...
method ReceiveRun (line 315) | func (iatt *ImplementsAllTheThings) ReceiveRun(qURL string) (queue.Run...
method ReceiveStatus (line 334) | func (iatt *ImplementsAllTheThings) ReceiveStatus(qURL string) (queue....
method List (line 355) | func (iatt *ImplementsAllTheThings) List() ([]string, error) {
method GetEvents (line 366) | func (iatt *ImplementsAllTheThings) GetEvents(ctx context.Context, run...
method FetchUpdateStatus (line 375) | func (iatt *ImplementsAllTheThings) FetchUpdateStatus(ctx context.Cont...
method FetchPodMetrics (line 381) | func (iatt *ImplementsAllTheThings) FetchPodMetrics(ctx context.Contex...
method CanBeRun (line 387) | func (iatt *ImplementsAllTheThings) CanBeRun(clusterName string, execu...
method IsImageValid (line 396) | func (iatt *ImplementsAllTheThings) IsImageValid(imageRef string) (boo...
method PollRunStatus (line 404) | func (iatt *ImplementsAllTheThings) PollRunStatus(ctx context.Context)...
method PollRuns (line 410) | func (iatt *ImplementsAllTheThings) PollRuns(ctx context.Context) ([]e...
method PollStatus (line 432) | func (iatt *ImplementsAllTheThings) PollStatus(ctx context.Context) (e...
method Execute (line 453) | func (iatt *ImplementsAllTheThings) Execute(ctx context.Context, execu...
method Terminate (line 459) | func (iatt *ImplementsAllTheThings) Terminate(ctx context.Context, run...
method Define (line 465) | func (iatt *ImplementsAllTheThings) Define(ctx context.Context, defini...
method Deregister (line 472) | func (iatt *ImplementsAllTheThings) Deregister(ctx context.Context, de...
method Logs (line 478) | func (iatt *ImplementsAllTheThings) Logs(executable state.Executable, ...
method GetExecutableByTypeAndID (line 484) | func (iatt *ImplementsAllTheThings) GetExecutableByTypeAndID(ctx conte...
method ListTemplates (line 497) | func (iatt *ImplementsAllTheThings) ListTemplates(ctx context.Context,...
method ListTemplatesLatestOnly (line 507) | func (iatt *ImplementsAllTheThings) ListTemplatesLatestOnly(ctx contex...
method GetTemplateByVersion (line 518) | func (iatt *ImplementsAllTheThings) GetTemplateByVersion(ctx context.C...
method GetTemplateByID (line 538) | func (iatt *ImplementsAllTheThings) GetTemplateByID(ctx context.Contex...
method GetLatestTemplateByTemplateName (line 549) | func (iatt *ImplementsAllTheThings) GetLatestTemplateByTemplateName(ct...
method CreateTemplate (line 571) | func (iatt *ImplementsAllTheThings) CreateTemplate(ctx context.Context...
method GetRunStatus (line 577) | func (iatt *ImplementsAllTheThings) GetRunStatus(ctx context.Context, ...
FILE: tracing/tracing.go
function TraceJob (line 11) | func TraceJob(ctx context.Context, operationName string, runID string) (...
function TagRunInfo (line 23) | func TagRunInfo(span tracer.Span,
type TextMapCarrier (line 40) | type TextMapCarrier
method ForeachKey (line 43) | func (c TextMapCarrier) ForeachKey(handler func(key, val string) error...
method Set (line 53) | func (c TextMapCarrier) Set(key, val string) {
FILE: ui/src/components/ARASwitch.tsx
type Props (line 9) | type Props = {
class ARASwitch (line 13) | class ARASwitch extends React.Component<Props> {
method constructor (line 14) | constructor(props: Props) {
method handleChange (line 19) | handleChange() {
method isEnabled (line 44) | isEnabled() {
method render (line 48) | render() {
type ConnectedProps (line 71) | type ConnectedProps = {
FILE: ui/src/components/App.tsx
class App (line 20) | class App extends React.Component<ConnectedProps<typeof connector>> {
method componentDidMount (line 21) | componentDidMount() {
method checkOnboardingStatus (line 25) | checkOnboardingStatus() {
method render (line 52) | render() {
FILE: ui/src/components/Attribute.tsx
type Props (line 5) | type Props = { rawValue: string }
type State (line 6) | type State = { isCopied: boolean }
class CopyableAttributeValue (line 8) | class CopyableAttributeValue extends React.Component<Props, State> {
method constructor (line 9) | constructor(props: Props) {
method handleCopy (line 18) | handleCopy() {
method render (line 22) | render() {
FILE: ui/src/components/BaseTaskForm.tsx
type Props (line 43) | type Props = Pick<
FILE: ui/src/components/CloudtrailRecords.tsx
type Props (line 5) | type Props = {
FILE: ui/src/components/CreateTaskForm.tsx
type Props (line 26) | type Props = Pick<
type ConnectedProps (line 75) | type ConnectedProps = RouteComponentProps & {
FILE: ui/src/components/DeleteTaskButton.tsx
type Args (line 9) | type Args = { definitionID: string }
type Props (line 10) | type Props = ChildProps<any, Args> & ConnectedProps
type State (line 11) | type State = { isOpen: boolean }
class DeleteTaskButton (line 13) | class DeleteTaskButton extends React.Component<Props, State> {
method constructor (line 14) | constructor(props: Props) {
method handleSubmitClick (line 24) | handleSubmitClick() {
method openDialog (line 28) | openDialog() {
method closeDialog (line 32) | closeDialog() {
method render (line 36) | render() {
type ConnectedProps (line 72) | type ConnectedProps = {
FILE: ui/src/components/Duration.tsx
type Props (line 5) | type Props = {
type State (line 11) | type State = {
class Duration (line 15) | class Duration extends React.Component<Props, State> {
method constructor (line 17) | constructor(props: Props) {
method componentDidMount (line 26) | componentDidMount() {
method componentWillUnmount (line 36) | componentWillUnmount() {
method process (line 40) | process() {
method render (line 45) | render() {
FILE: ui/src/components/EnvFieldArray.tsx
type Props (line 10) | type Props = {
FILE: ui/src/components/EnvQueryFilter.tsx
type Props (line 8) | type Props = {
type State (line 13) | type State = {
class EnvQueryFilter (line 18) | class EnvQueryFilter extends React.Component<Props, State> {
method constructor (line 21) | constructor(props: Props) {
method serialize (line 36) | serialize(env: Env): string {
method deserialize (line 40) | deserialize(str: string): Env {
method handleNameChange (line 48) | handleNameChange(i: number, evt: React.ChangeEvent<HTMLInputElement>) {
method handleValueChange (line 56) | handleValueChange(i: number, evt: React.ChangeEvent<HTMLInputElement>) {
method handleRemove (line 64) | handleRemove(i: number) {
method handleNewNameChange (line 71) | handleNewNameChange(evt: React.ChangeEvent<HTMLInputElement>) {
method handleNewValueChange (line 75) | handleNewValueChange(evt: React.ChangeEvent<HTMLInputElement>) {
method handleAddNewEnv (line 79) | handleAddNewEnv() {
method shouldDisableAddNewEnvButton (line 90) | shouldDisableAddNewEnvButton(): boolean {
method render (line 95) | render() {
FILE: ui/src/components/ListRequest.tsx
constant DEFAULT_PROPS (line 7) | const DEFAULT_PROPS = {
type Props (line 11) | type Props<Response, Args> = RequestChildProps<Response, Args> &
type ChildProps (line 18) | type ChildProps<Response, Args> = Omit<
class ListRequest (line 31) | class ListRequest<Response, Args> extends React.Component<
method componentDidMount (line 36) | componentDidMount() {
method componentDidUpdate (line 45) | componentDidUpdate(prevProps: Props<Response, Args>) {
method request (line 51) | request() {
method updateSort (line 60) | updateSort(sortKey: string): void {
method updatePage (line 95) | updatePage(n: number): void {
method updateFilter (line 103) | updateFilter(key: string, value: any): void {
method getChildProps (line 107) | getChildProps(): ChildProps<Response, Args> {
method render (line 124) | render() {
type ConnectedProps (line 129) | type ConnectedProps<Response, Args> = {
class ConnectedListRequest (line 136) | class ConnectedListRequest<Response, Args> extends React.Component<
method render (line 140) | render() {
FILE: ui/src/components/Log.tsx
type Props (line 9) | type Props = {
class Log (line 15) | class Log extends React.Component<Props> {
method componentDidMount (line 18) | componentDidMount() {
method componentDidUpdate (line 24) | componentDidUpdate(prevProps: Props) {
method shouldScrollToBottom (line 46) | shouldScrollToBottom(prev: Props, next: Props) {
method render (line 52) | render() {
FILE: ui/src/components/LogProcessor.tsx
type ConnectedProps (line 10) | type ConnectedProps = {
type Props (line 15) | type Props = ConnectedProps & {
type State (line 20) | type State = {
class LogProcessor (line 25) | class LogProcessor extends React.Component<Props, State> {
method constructor (line 27) | constructor(props: Props) {
method componentDidMount (line 47) | componentDidMount() {
method componentDidUpdate (line 51) | componentDidUpdate(prevProps: Props) {
method processLogs (line 67) | processLogs(): void {
method render (line 80) | render() {
FILE: ui/src/components/LogRequesterCloudWatchLogs.tsx
type Props (line 18) | type Props = {
type State (line 23) | type State = {
class LogRequesterCloudWatchLogs (line 37) | class LogRequesterCloudWatchLogs extends React.Component<Props, State> {
method componentDidMount (line 41) | componentDidMount() {
method componentDidUpdate (line 45) | componentDidUpdate(prevProps: Props) {
method componentWillUnmount (line 60) | componentWillUnmount() {
method initialize (line 79) | initialize() {
method handleRunIDChange (line 91) | handleRunIDChange() {
method render (line 158) | render() {
FILE: ui/src/components/LogRequesterS3.tsx
type Props (line 20) | type Props = {
type State (line 25) | type State = {
class LogRequesterS3 (line 37) | class LogRequesterS3 extends React.PureComponent<Props, State> {
method componentDidMount (line 42) | componentDidMount() {
method componentDidUpdate (line 46) | componentDidUpdate(prevProps: Props) {
method componentWillUnmount (line 64) | componentWillUnmount() {
method initialize (line 89) | initialize() {
method handleRunIDChange (line 97) | handleRunIDChange() {
method render (line 132) | render() {
FILE: ui/src/components/LogVirtualized.tsx
type Props (line 15) | type Props = {
type State (line 22) | type State = {
type KeyCode (line 30) | enum KeyCode {
class LogVirtualized (line 37) | class LogVirtualized extends React.Component<Props, State> {
method constructor (line 46) | constructor(props: Props) {
method componentDidMount (line 63) | componentDidMount() {
method componentDidUpdate (line 72) | componentDidUpdate(prevProps: Props, prevState: State) {
method componentWillUnmount (line 88) | componentWillUnmount() {
method search (line 97) | search(q: string): void {
method handleCursorChange (line 124) | handleCursorChange(): void {
method handleIncrementCursor (line 134) | handleIncrementCursor(): void {
method handleDecrementCursor (line 145) | handleDecrementCursor(): void {
method handleKeydown (line 156) | handleKeydown(evt: KeyboardEvent) {
method resetSearchState (line 185) | resetSearchState(): void {
method searchInputFocus (line 194) | searchInputFocus() {
method scrollTo (line 200) | scrollTo(
method render (line 210) | render() {
FILE: ui/src/components/LogVirtualizedSearch.tsx
type Props (line 4) | type Props = {
FILE: ui/src/components/Pagination.tsx
type Props (line 4) | type Props = {
FILE: ui/src/components/QueryParams.tsx
type Props (line 5) | type Props = RouteComponentProps & {
type ChildProps (line 9) | type ChildProps = {
class QueryParams (line 14) | class QueryParams extends React.Component<Props> {
method setQuery (line 15) | setQuery(query: object, shouldReplace?: boolean): void {
method getQuery (line 25) | getQuery(): object {
method getChildProps (line 35) | getChildProps(): ChildProps {
method render (line 42) | render() {
FILE: ui/src/components/Request.tsx
type RequestStatus (line 4) | enum RequestStatus {
type Props (line 10) | type Props<ResponseType, ArgsType> = {
type State (line 19) | type State<ResponseType> = {
type ChildProps (line 27) | type ChildProps<ResponseType, ArgsType> = State<ResponseType> & {
class Request (line 31) | class Request<ResponseType, ArgsType> extends React.Component<
method componentDidMount (line 48) | componentDidMount() {
method request (line 54) | request(args: ArgsType): void {
method render (line 85) | render() {
FILE: ui/src/components/Run.tsx
type Props (line 53) | type Props = QPChildProps &
class Run (line 58) | class Run extends React.Component<Props> {
method constructor (line 61) | constructor(props: Props) {
method componentDidMount (line 66) | componentDidMount() {
method componentDidUpdate (line 73) | componentDidUpdate(prevProps: Props) {
method componentWillUnmount (line 91) | componentWillUnmount() {
method request (line 95) | request() {
method setRequestInterval (line 101) | setRequestInterval() {
method clearRequestInterval (line 108) | clearRequestInterval() {
method getActiveTabId (line 112) | getActiveTabId(): RunTabId {
method setActiveTabId (line 135) | setActiveTabId(id: RunTabId): void {
method getExecutableLinkName (line 139) | getExecutableLinkName(): string {
method getExecutableLinkURL (line 152) | getExecutableLinkURL(): string {
method render (line 165) | render() {
FILE: ui/src/components/RunEvents.tsx
type Props (line 11) | type Props = {
FILE: ui/src/components/Runs.tsx
type Props (line 35) | type Props = ListRequestChildProps<
FILE: ui/src/components/SortableTh.tsx
type Props (line 4) | type Props = {
FILE: ui/src/components/StopRunButton.tsx
type Args (line 9) | type Args = { definitionID: string; runID: string }
type Props (line 10) | type Props = ChildProps<any, Args> & ConnectedProps
type State (line 11) | type State = { isOpen: boolean }
class StopRunButton (line 13) | class StopRunButton extends React.Component<Props, State> {
method constructor (line 14) | constructor(props: Props) {
method openDialog (line 25) | openDialog() {
method closeDialog (line 29) | closeDialog() {
method handleSubmitClick (line 33) | handleSubmitClick() {
method render (line 41) | render() {
type ConnectedProps (line 76) | type ConnectedProps = {
FILE: ui/src/components/Table.tsx
type Column (line 7) | type Column<ItemType> = {
type Props (line 13) | type Props<ItemType> = {
class Table (line 22) | class Table<ItemType> extends React.Component<Props<ItemType>> {
method render (line 23) | render() {
FILE: ui/src/components/Task.tsx
type TaskCtx (line 14) | type TaskCtx = ChildProps<TaskShape, { definitionID: string }> & {
type ConnectedProps (line 87) | type ConnectedProps = RouteComponentProps<{ definitionID: string }>
FILE: ui/src/components/TaskExecutionForm.tsx
type Props (line 55) | type Props = RequestChildProps<
FILE: ui/src/components/TaskRuns.tsx
type Props (line 36) | type Props = ListRequestChildProps<
FILE: ui/src/components/Tasks.tsx
type Props (line 25) | type Props = ListRequestChildProps<
FILE: ui/src/components/Template.tsx
type TemplateCtx (line 9) | type TemplateCtx = ChildProps<TemplateShape, { templateID: string }> & {
type ConnectedProps (line 40) | type ConnectedProps = RouteComponentProps<{ templateID: string }>
FILE: ui/src/components/TemplateExecutionForm.tsx
type Props (line 55) | type Props = RequestChildProps<
FILE: ui/src/components/TemplateHistoryTable.tsx
type Props (line 36) | type Props = ListRequestChildProps<
FILE: ui/src/components/TemplateRunForm.tsx
type Props (line 76) | type Props = RequestChildProps<
class RunForm (line 112) | class RunForm extends React.Component<Props> {
method onSubmit (line 121) | onSubmit(jsonschemaForm: any) {
method render (line 132) | render() {
FILE: ui/src/components/Templates.tsx
type Props (line 21) | type Props = ListRequestChildProps<
FILE: ui/src/components/Toggler.tsx
type Props (line 3) | type Props = {
type State (line 7) | type State = {
type ChildProps (line 11) | type ChildProps = {
class Toggler (line 16) | class Toggler extends React.Component<Props, State> {
method toggleVisibility (line 21) | toggleVisibility() {
method getChildProps (line 25) | getChildProps(): ChildProps {
method render (line 32) | render() {
FILE: ui/src/components/UpdateTaskForm.tsx
type Props (line 22) | type Props = Pick<
type ConnectedProps (line 60) | type ConnectedProps = RouteComponentProps & {
FILE: ui/src/components/ViewHeader.tsx
type Props (line 5) | type Props = {
FILE: ui/src/components/__tests__/ListRequest.spec.tsx
constant DEFAULT_PROPS (line 7) | const DEFAULT_PROPS: Props<any, any> = {
FILE: ui/src/components/__tests__/Run.spec.tsx
type RunInstance (line 17) | type RunInstance = {}
FILE: ui/src/constants.ts
constant PAGE_SIZE (line 5) | const PAGE_SIZE = 20
constant RUN_FETCH_INTERVAL_MS (line 6) | const RUN_FETCH_INTERVAL_MS = 5000 // 5 sec
constant LOG_FETCH_INTERVAL_MS (line 7) | const LOG_FETCH_INTERVAL_MS = 10000 // 10 sec
constant KILL_LOG_POLLING_TIMEOUT_MS (line 8) | const KILL_LOG_POLLING_TIMEOUT_MS = 120000 // 2 mins
constant RUN_TAB_ID_QUERY_KEY (line 9) | const RUN_TAB_ID_QUERY_KEY = "rt"
constant LOG_SEARCH_QUERY_KEY (line 10) | const LOG_SEARCH_QUERY_KEY = "log_search"
constant RUN_STATUS_COLOR_MAP (line 11) | const RUN_STATUS_COLOR_MAP = new Map<
constant LOCAL_STORAGE_SETTINGS_KEY (line 23) | const LOCAL_STORAGE_SETTINGS_KEY = "settings"
constant LOCAL_STORAGE_IS_ONBOARDED_KEY (line 24) | const LOCAL_STORAGE_IS_ONBOARDED_KEY = "is_onboarded"
constant CHAR_TO_PX_RATIO (line 25) | const CHAR_TO_PX_RATIO = 40 / 300
constant JSON_VIEW_PROPS (line 26) | const JSON_VIEW_PROPS: Partial<ReactJsonViewProps> = {
FILE: ui/src/helpers/FlotillaClient.ts
type IInitOpts (line 32) | interface IInitOpts {
class FlotillaClient (line 37) | class FlotillaClient {
method constructor (line 40) | constructor({ baseURL, headers = {} }: IInitOpts) {
method request (line 303) | private request<T>({ method, url, params, data }: RequestArgs): Promis...
FILE: ui/src/helpers/constructDefaultObjectFromJsonSchema.ts
constant DEFAULT_ARRAY (line 3) | const DEFAULT_ARRAY: any[] = []
constant DEFAULT_STRING (line 4) | const DEFAULT_STRING = ""
constant DEFAULT_NUM (line 5) | const DEFAULT_NUM = 0
constant DEFAULT_BOOL (line 6) | const DEFAULT_BOOL = false
function constructDefaultObjectFromJsonSchema (line 8) | function constructDefaultObjectFromJsonSchema(
function helper (line 27) | function helper(properties: object, root: { [k: string]: any }): void {
FILE: ui/src/helpers/getInitialValuesForExecutionForm.ts
function getInitialValuesForTaskExecutionForm (line 18) | function getInitialValuesForTaskExecutionForm(
function getInitialValuesForTemplateExecutionForm (line 32) | function getInitialValuesForTemplateExecutionForm(
function getInitialValuesForCommonExecutionFields (line 48) | function getInitialValuesForCommonExecutionFields(
FILE: ui/src/helpers/testHelpers.ts
function createMockRouteComponentProps (line 13) | function createMockRouteComponentProps<MatchParams>({
FILE: ui/src/state/runView.ts
type RunViewReducer (line 3) | type RunViewReducer = {
method toggleAutoscroll (line 19) | toggleAutoscroll(state) {
method setHasLogs (line 23) | setHasLogs(state) {
method toggleIsLogRequestIntervalActive (line 27) | toggleIsLogRequestIntervalActive(
FILE: ui/src/state/settings.ts
type Settings (line 7) | type Settings = {
type SettingsReducer (line 12) | type SettingsReducer = {
method initStart (line 44) | initStart() {}
method initSuccess (line 45) | initSuccess(state, { payload }: PayloadAction<any>) {
method initFailure (line 48) | initFailure() {}
method updateStart (line 49) | updateStart(state) {
method updateSuccess (line 52) | updateSuccess(state, { payload }: PayloadAction<Settings>) {
method updateFailure (line 57) | updateFailure(state) {
method toggleDialogVisibilityChange (line 60) | toggleDialogVisibilityChange(
FILE: ui/src/state/store.ts
type RootState (line 34) | type RootState = ReturnType<typeof rootReducer>
type AppDispatch (line 35) | type AppDispatch = typeof store.dispatch
type AppThunk (line 36) | type AppThunk = ThunkAction<void, RootState, null, Action<string>>
FILE: ui/src/types.ts
type Env (line 3) | type Env = {
type Task (line 8) | type Task = {
type RunInstance (line 18) | type RunInstance = {
type CloudTrailNotifications (line 23) | type CloudTrailNotifications = {}
type ExecutableType (line 25) | enum ExecutableType {
type Run (line 30) | type Run = {
type RunLog (line 64) | type RunLog = {
type RunLogRaw (line 69) | type RunLogRaw = string
type HTTPMethod (line 75) | enum HTTPMethod {
type SortOrder (line 82) | enum SortOrder {
type RunStatus (line 87) | enum RunStatus {
type EnhancedRunStatus (line 95) | enum EnhancedRunStatus {
type SelectOption (line 107) | type SelectOption = { label: string; value: string }
type SelectProps (line 109) | type SelectProps = {
type MultiSelectProps (line 115) | type MultiSelectProps = {
type RequestArgs (line 126) | type RequestArgs = {
type ListRequestArgs (line 133) | type ListRequestArgs = {
type ListResponse (line 140) | type ListResponse = {
type ListTaskRunsParams (line 148) | type ListTaskRunsParams = Omit<ListRunParams, "alias">
type ListTaskRunsResponse (line 149) | type ListTaskRunsResponse = Omit<ListRunResponse, "alias">
type ListTaskParams (line 151) | type ListTaskParams = ListRequestArgs & {
type ListTaskResponse (line 157) | type ListTaskResponse = ListResponse & {
type CreateTaskPayload (line 164) | type CreateTaskPayload = UpdateTaskPayload & { alias: string }
type UpdateTaskPayload (line 166) | type UpdateTaskPayload = {
type ExecutionEngine (line 177) | enum ExecutionEngine {
type NodeLifecycle (line 184) | enum NodeLifecycle {
type LaunchRequestV2 (line 191) | type LaunchRequestV2 = {
type ListRunParams (line 203) | type ListRunParams = ListRequestArgs & {
type ListRunResponse (line 210) | type ListRunResponse = ListResponse & {
type ListClustersResponse (line 218) | type ListClustersResponse = ListResponse & { clusters: string[] | null }
type ListGroupsResponse (line 219) | type ListGroupsResponse = ListResponse & { groups: string[] | null }
type ListTagsResponse (line 220) | type ListTagsResponse = ListResponse & { tags: string[] | null }
type FieldSpec (line 222) | type FieldSpec = {
type PodEvent (line 229) | type PodEvent = {
type ListRunEventsResponse (line 237) | type ListRunEventsResponse = {
type RunTabId (line 242) | enum RunTabId {
type ExecutableResources (line 249) | type ExecutableResources = {
type Template (line 261) | type Template = {
type ListTemplateParams (line 271) | type ListTemplateParams = ListRequestArgs & {
type ListTemplateResponse (line 275) | type ListTemplateResponse = ListResponse & {
type TemplateExecutionRequest (line 279) | type TemplateExecutionRequest = {
type ExecutionRequestCommon (line 283) | type ExecutionRequestCommon = {
type CloudtrailRecord (line 293) | type CloudtrailRecord = {
type CloudtrailRecords (line 298) | type CloudtrailRecords = {
type ListTemplateHistoryParams (line 314) | type ListTemplateHistoryParams = Omit<ListRunParams, "alias">
type ListTemplateHistoryResponse (line 315) | type ListTemplateHistoryResponse = Omit<ListRunResponse, "alias">
type Executable (line 316) | type Executable = Template | Task
FILE: ui/src/workers/index.ts
class WebWorker (line 1) | class WebWorker {
method constructor (line 2) | constructor(worker: any) {
FILE: utils/dd_tracing.go
function TraceJob (line 12) | func TraceJob(ctx context.Context, operationName string, runID string) (...
function TagJobRun (line 17) | func TagJobRun(span tracer.Span, run state.Run) {
FILE: utils/utils.go
function StringSliceContains (line 11) | func StringSliceContains(s []string, e string) bool {
function SetupRedisClient (line 20) | func SetupRedisClient(c config.Config) (*redis.Client, error) {
FILE: worker/events_worker.go
type eventsWorker (line 23) | type eventsWorker struct
method Initialize (line 44) | func (ew *eventsWorker) Initialize(conf config.Config, sm state.Manage...
method GetTomb (line 82) | func (ew *eventsWorker) GetTomb() *tomb.Tomb {
method Run (line 86) | func (ew *eventsWorker) Run(ctx context.Context) error {
method runOnceEMR (line 102) | func (ew *eventsWorker) runOnceEMR(ctx context.Context) {
method processEventEMR (line 113) | func (ew *eventsWorker) processEventEMR(ctx context.Context, emrEvent ...
method runOnce (line 184) | func (ew *eventsWorker) runOnce(ctx context.Context) {
method processEMRPodEvents (line 195) | func (ew *eventsWorker) processEMRPodEvents(ctx context.Context, kuber...
method setEMRMetricsUri (line 354) | func (ew *eventsWorker) setEMRMetricsUri(run *state.Run) {
method setEKSMetricsUri (line 366) | func (ew *eventsWorker) setEKSMetricsUri(run *state.Run) {
method processEvent (line 390) | func (ew *eventsWorker) processEvent(ctx context.Context, kubernetesEv...
method parsePodName (line 446) | func (ew *eventsWorker) parsePodName(kubernetesEvent state.KubernetesE...
method applySlidingWindow (line 457) | func (ew *eventsWorker) applySlidingWindow(events state.PodEvents, new...
FILE: worker/events_worker_test.go
function TestEventsWorker_applySlidingWindow (line 11) | func TestEventsWorker_applySlidingWindow(t *testing.T) {
FILE: worker/retry_worker.go
type retryWorker (line 17) | type retryWorker struct
method Initialize (line 27) | func (rw *retryWorker) Initialize(conf config.Config, sm state.Manager...
method GetTomb (line 38) | func (rw *retryWorker) GetTomb() *tomb.Tomb {
method Run (line 43) | func (rw *retryWorker) Run(ctx context.Context) error {
method runOnce (line 56) | func (rw *retryWorker) runOnce(ctx context.Context) {
FILE: worker/retry_worker_test.go
function setUpRetryWorkerTest (line 13) | func setUpRetryWorkerTest(t *testing.T) (*retryWorker, *testutils.Implem...
function TestRetryWorker_Run (line 41) | func TestRetryWorker_Run(t *testing.T) {
FILE: worker/status_worker.go
type statusWorker (line 26) | type statusWorker struct
method Initialize (line 42) | func (sw *statusWorker) Initialize(conf config.Config, sm state.Manage...
method GetTomb (line 63) | func (sw *statusWorker) GetTomb() *tomb.Tomb {
method Run (line 68) | func (sw *statusWorker) Run(ctx context.Context) error {
method runTimeouts (line 84) | func (sw *statusWorker) runTimeouts(ctx context.Context) {
method processTimeouts (line 103) | func (sw *statusWorker) processTimeouts(runs []state.Run) {
method runOnceEKS (line 135) | func (sw *statusWorker) runOnceEKS(ctx context.Context) {
method processEKSRuns (line 154) | func (sw *statusWorker) processEKSRuns(ctx context.Context, runs []sta...
method acquireLock (line 190) | func (sw *statusWorker) acquireLock(run state.Run, purpose string, exp...
method processEKSRun (line 206) | func (sw *statusWorker) processEKSRun(ctx context.Context, run state.R...
method cleanupRun (line 307) | func (sw *statusWorker) cleanupRun(ctx context.Context, runID string) {
method extractExceptions (line 321) | func (sw *statusWorker) extractExceptions(ctx context.Context, runID s...
method processEKSRunMetrics (line 353) | func (sw *statusWorker) processEKSRunMetrics(ctx context.Context, run ...
method logStatusUpdate (line 368) | func (sw *statusWorker) logStatusUpdate(update state.Run) {
method findRun (line 441) | func (sw *statusWorker) findRun(ctx context.Context, taskArn string) (...
FILE: worker/status_worker_test.go
function setUpStatusWorkerTest (line 13) | func setUpStatusWorkerTest(t *testing.T) (*statusWorker, *testutils.Impl...
function setUpStatusWorkerTest2 (line 69) | func setUpStatusWorkerTest2(t *testing.T) (*statusWorker, *testutils.Imp...
FILE: worker/submit_worker.go
type submitWorker (line 21) | type submitWorker struct
method Initialize (line 33) | func (sw *submitWorker) Initialize(conf config.Config, sm state.Manage...
method GetTomb (line 46) | func (sw *submitWorker) GetTomb() *tomb.Tomb {
method Run (line 51) | func (sw *submitWorker) Run(ctx context.Context) error {
method runOnce (line 63) | func (sw *submitWorker) runOnce(ctx context.Context) {
method logFailedToGetExecutableMessage (line 237) | func (sw *submitWorker) logFailedToGetExecutableMessage(run state.Run,...
FILE: worker/submit_worker_test.go
function setUpSubmitWorkerTest1 (line 15) | func setUpSubmitWorkerTest1(t *testing.T) (*submitWorker, *testutils.Imp...
function setUpSubmitWorkerTest2 (line 48) | func setUpSubmitWorkerTest2(t *testing.T) (*submitWorker, *testutils.Imp...
function setUpSubmitWorkerTest3 (line 81) | func setUpSubmitWorkerTest3(t *testing.T) (*submitWorker, *testutils.Imp...
function TestSubmitWorker_Run (line 109) | func TestSubmitWorker_Run(t *testing.T) {
function TestSubmitWorker_Run2 (line 130) | func TestSubmitWorker_Run2(t *testing.T) {
function TestSubmitWorker_Run3 (line 148) | func TestSubmitWorker_Run3(t *testing.T) {
function TestSubmitWorker_Run4 (line 167) | func TestSubmitWorker_Run4(t *testing.T) {
function TestSubmitWorker_Run5 (line 195) | func TestSubmitWorker_Run5(t *testing.T) {
FILE: worker/worker.go
type Worker (line 18) | type Worker interface
function NewWorker (line 25) | func NewWorker(workerType string, log flotillaLog.Logger, conf config.Co...
function GetPollInterval (line 51) | func GetPollInterval(workerType string, conf config.Config) (time.Durati...
FILE: worker/worker_manager.go
type workerManager (line 18) | type workerManager struct
method Initialize (line 32) | func (wm *workerManager) Initialize(
method GetTomb (line 61) | func (wm *workerManager) GetTomb() *tomb.Tomb {
method InitializeWorkers (line 68) | func (wm *workerManager) InitializeWorkers(ctx context.Context) error {
method Run (line 99) | func (wm *workerManager) Run(ctx context.Context) error {
method runOnce (line 114) | func (wm *workerManager) runOnce(ctx context.Context) error {
method updateWorkerCount (line 139) | func (wm *workerManager) updateWorkerCount(
method removeWorker (line 168) | func (wm *workerManager) removeWorker(ctx context.Context, workerType ...
method addWorker (line 185) | func (wm *workerManager) addWorker(ctx context.Context, workerType str...
FILE: worker/worker_test.go
function TestGetPollInterval (line 11) | func TestGetPollInterval(t *testing.T) {
Condensed preview — 224 files, each showing path, character count, and a content snippet. Download the .json file or copy for the full structured content (1,299K chars).
[
{
"path": ".circleci/config.yml",
"chars": 1499,
"preview": "---\nversion: 2\njobs:\n build:\n working_directory: ~/go/src/github.com/stitchfix/flotilla-os\n docker:\n - image"
},
{
"path": ".github/CODEOWNERS",
"chars": 153,
"preview": "# This file uses the GitHub CODEOWNERS convention to assign PR reviewers:\n# https://help.github.com/articles/about-codeo"
},
{
"path": ".github/PULL_REQUEST_TEMPLATE.md",
"chars": 24,
"preview": "## PROBLEM\n\n\n## SOLUTION"
},
{
"path": ".gitignore",
"chars": 419,
"preview": "# Binaries for programs and plugins\n*.exe\n*.dll\n*.so\n*.dylib\n\n# Test binary, build with `go test -c`\n*.test\n\n# Output of"
},
{
"path": ".migrations/V20200123054713__initial_table_create.sql",
"chars": 4435,
"preview": "--\n-- Definitions\n--\nCREATE TABLE IF NOT EXISTS task_def (\n definition_id character varying PRIMARY KEY,\n alias charac"
},
{
"path": ".migrations/V20200123054714__add_spark_extension.sql",
"chars": 65,
"preview": "ALTER TABLE task ADD COLUMN IF NOT EXISTS spark_extension JSONB;\n"
},
{
"path": ".migrations/V20200205133700__executable.sql",
"chars": 116,
"preview": "ALTER TABLE task\n ADD COLUMN executable_id VARCHAR,\n ADD COLUMN executable_type VARCHAR DEFAULT 'task_definition';"
},
{
"path": ".migrations/V20200206115000__template.sql",
"chars": 518,
"preview": "CREATE TABLE template (\n template_id VARCHAR PRIMARY KEY,\n type VARCHAR NOT NULL,\n version INTEGER NOT NULL,\n schema"
},
{
"path": ".migrations/V20200210154600__template_refactor.sql",
"chars": 207,
"preview": "ALTER TABLE template DROP CONSTRAINT template_type_version;\nALTER TABLE template RENAME COLUMN type to template_name;\nAL"
},
{
"path": ".migrations/V20200211160100__task_col_fix.sql",
"chars": 85,
"preview": "ALTER TABLE task RENAME COLUMN executable_request_custom to execution_request_custom;"
},
{
"path": ".migrations/V20200211161900__template_indicies.sql",
"chars": 140,
"preview": "CREATE INDEX IF NOT EXISTS ix_template_id ON template(template_id);\nCREATE INDEX IF NOT EXISTS ix_template_name ON templ"
},
{
"path": ".migrations/V20200212101900__template.sql",
"chars": 106,
"preview": "ALTER TABLE template ADD COLUMN default_payload JSONB;\nALTER TABLE template ADD COLUMN avatar_uri VARCHAR;"
},
{
"path": ".migrations/V20200213101400__task_indexes.sql",
"chars": 322,
"preview": "CREATE INDEX IF NOT EXISTS ix_task_executable_id ON task(executable_id);\nCREATE INDEX IF NOT EXISTS ix_task_executable_i"
},
{
"path": ".migrations/V20200213125200__rename_default_payload.sql",
"chars": 63,
"preview": "ALTER TABLE template RENAME COLUMN default_payload to defaults;"
},
{
"path": ".migrations/V20200225125200__add_limits.sql",
"chars": 97,
"preview": "ALTER TABLE task ADD COLUMN memory_limit integer;\nALTER TABLE task ADD COLUMN cpu_limit integer;\n"
},
{
"path": ".migrations/V20200325125200__add_attempts.sql",
"chars": 51,
"preview": "ALTER TABLE task ADD COLUMN attempt_count integer;\n"
},
{
"path": ".migrations/V20200325125201__add_spawned.sql",
"chars": 48,
"preview": "ALTER TABLE task ADD COLUMN spawned_runs jsonb;\n"
},
{
"path": ".migrations/V20200625125201__add_run_exceptions.sql",
"chars": 50,
"preview": "ALTER TABLE task ADD COLUMN run_exceptions jsonb;\n"
},
{
"path": ".migrations/V20210083054714__metrics_uri.sql",
"chars": 63,
"preview": "ALTER TABLE task ADD COLUMN IF NOT EXISTS metrics_uri varchar;\n"
},
{
"path": ".migrations/V20210427125201__add_active_deadline_seconds.sql",
"chars": 61,
"preview": "ALTER TABLE task ADD COLUMN active_deadline_seconds integer;\n"
},
{
"path": ".migrations/V20210807125201__drop_index_container_name.sql",
"chars": 64,
"preview": "alter table task_def alter column container_name drop not null;\n"
},
{
"path": ".migrations/V20211007125201__add_description.sql",
"chars": 63,
"preview": "ALTER TABLE task ADD COLUMN IF NOT EXISTS description varchar;\n"
},
{
"path": ".migrations/V20220907125201__add_idempotence.sql",
"chars": 67,
"preview": "ALTER TABLE task ADD COLUMN IF NOT EXISTS idempotence_key varchar;\n"
},
{
"path": ".migrations/V20220907125202__add_arch.sql",
"chars": 56,
"preview": "ALTER TABLE task ADD COLUMN IF NOT EXISTS arch varchar;\n"
},
{
"path": ".migrations/V20221215125203__add_labels.sql",
"chars": 56,
"preview": "ALTER TABLE task ADD COLUMN IF NOT EXISTS labels jsonb;\n"
},
{
"path": ".migrations/V20230718115000__add_ephemeral_storage.sql",
"chars": 142,
"preview": "ALTER TABLE task_def ADD COLUMN IF NOT EXISTS ephemeral_storage INTEGER;\nALTER TABLE task ADD COLUMN IF NOT EXISTS ephem"
},
{
"path": ".migrations/V20231013191711__add_requires_docker.sql",
"chars": 168,
"preview": "ALTER TABLE task_def ADD COLUMN IF NOT EXISTS requires_docker BOOLEAN DEFAULT(false);\nALTER TABLE task ADD COLUMN IF NOT"
},
{
"path": ".migrations/V20231122141100__add_target_cluster.sql",
"chars": 69,
"preview": "ALTER TABLE task_def ADD COLUMN IF NOT EXISTS target_cluster VARCHAR;"
},
{
"path": ".migrations/V20240205132100__add_service_account.sql",
"chars": 66,
"preview": "ALTER TABLE task ADD COLUMN IF NOT EXISTS service_account VARCHAR;"
},
{
"path": ".migrations/V20250122141100__add_cluster_routing.sql",
"chars": 1192,
"preview": "DO $$\nBEGIN\n IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'cluster_status') THEN\n CREATE TYPE cluster_"
},
{
"path": ".migrations/dev.conf",
"chars": 182,
"preview": "flyway.url=jdbc:postgresql://127.0.0.1:5432/flotilla\nflyway.user=flotilla\nflyway.password=flotilla\nflyway.cleanDisabled="
},
{
"path": "ARA_METRICS_COMPARISON.md",
"chars": 11548,
"preview": "# ARA Metrics Implementation Comparison\n\nComparing `ez/ara-metrics` (HEAD) vs `ez/ara-metrics-alt`\n\n## Overview\n\nBoth im"
},
{
"path": "Dockerfile",
"chars": 296,
"preview": "FROM golang:latest\n\nRUN mkdir -p /go/src/github.com/stitchfix/flotilla-os\nADD . /go/src/github.com/stitchfix/flotilla-os"
},
{
"path": "LICENSE",
"chars": 11357,
"preview": " Apache License\n Version 2.0, January 2004\n "
},
{
"path": "README.html",
"chars": 47653,
"preview": "<!DOCTYPE html><html><head><meta charset=\"utf-8\"><meta name=\"viewport\" content=\"width=device-width, initial-scale=1\"><st"
},
{
"path": "README.md",
"chars": 12604,
"preview": "<img src=\"https://user-images.githubusercontent.com/10807627/36499489-47bf1b6e-16f6-11e8-9a22-6e356c603a37.png\" height=\""
},
{
"path": "ara-impact-report-staging.md",
"chars": 12478,
"preview": "# ARA Impact Analysis Report - STAGING Environment\n## 10-Day Analysis of Adaptive Resource Allocation (Dec 7-17, 2025)\n\n"
},
{
"path": "ara-impact-report.md",
"chars": 17023,
"preview": "# ARA Impact Analysis Report\n## 10-Day Analysis of Adaptive Resource Allocation (Dec 7-17, 2025)\n\n### Executive Summary\n"
},
{
"path": "clients/cluster/cluster.go",
"chars": 1015,
"preview": "package cluster\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/pkg/errors\"\n\t\"github.com/stitchfix/flotilla-os/config\"\n\t\"github.com/stitc"
},
{
"path": "clients/cluster/eks_cluster_client.go",
"chars": 798,
"preview": "package cluster\n\nimport (\n\t\"github.com/stitchfix/flotilla-os/config\"\n\t\"github.com/stitchfix/flotilla-os/state\"\n)\n\n// EKS"
},
{
"path": "clients/httpclient/client.go",
"chars": 4501,
"preview": "package httpclient\n\nimport (\n\t\"bytes\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io\"\n\t\"net/http\"\n\t\"net/url\"\n\t\"strings\"\n\t\"time\"\n)\n\ntype Re"
},
{
"path": "clients/httpclient/client_test.go",
"chars": 3615,
"preview": "package httpclient\n\nimport (\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"net/http\"\n\t\"net/http/httptest\"\n\t\"testing\"\n\t\"time\"\n)\n\nty"
},
{
"path": "clients/logs/eks_cloudwatch_logs_client.go",
"chars": 6005,
"preview": "package logs\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"github.com/aws/aws-sdk-go/aws\"\n\t\"github.com/aws/aws-sdk-go/aws/awserr\"\n"
},
{
"path": "clients/logs/eks_s3_logs_client.go",
"chars": 10817,
"preview": "package logs\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"compress/gzip\"\n\t\"context\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"github.com/aws/aws-sdk-go"
},
{
"path": "clients/logs/logs.go",
"chars": 1991,
"preview": "package logs\n\nimport (\n\t\"fmt\"\n\t\"github.com/aws/aws-sdk-go/service/cloudwatchlogs\"\n\t\"github.com/pkg/errors\"\n\t\"github.com/"
},
{
"path": "clients/metrics/datadog_metrics_client.go",
"chars": 2667,
"preview": "package metrics\n\nimport (\n\t\"fmt\"\n\t\"github.com/DataDog/datadog-go/v5/statsd\"\n\t\"github.com/stitchfix/flotilla-os/config\"\n\t"
},
{
"path": "clients/metrics/metrics.go",
"chars": 7068,
"preview": "package metrics\n\nimport (\n\t\"fmt\"\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/pkg/errors\"\n\t\"github.com/stitchfix/flotilla-os/config\"\n)\n"
},
{
"path": "clients/middleware/client.go",
"chars": 415,
"preview": "package middleware\n\nimport (\n\t\"github.com/stitchfix/flotilla-os/state\"\n\t\"net/http\"\n)\n\ntype Client interface {\n\tAnnotateL"
},
{
"path": "conf/config.yml",
"chars": 1220,
"preview": "aws_default_region: us-east-1\ncluster_client: eks\ncreate_database_schema: true\ndatabase_url: postgresql://flotilla:floti"
},
{
"path": "config/config.go",
"chars": 2223,
"preview": "package config\n\nimport (\n\t\"github.com/pkg/errors\"\n\t\"github.com/spf13/viper\"\n\t\"strings\"\n)\n\n//\n// Config interface to wrap"
},
{
"path": "config/config_test.go",
"chars": 575,
"preview": "package config\n\nimport (\n\t\"os\"\n\t\"testing\"\n)\n\nfunc TestNewConfig(t *testing.T) {\n\tvar c Config\n\tc, _ = NewConfig(nil)\n\n\tt"
},
{
"path": "datadog-ara-dashboard-api.json",
"chars": 26423,
"preview": "{\n \"title\": \"Flotilla ARA (Auto Resource Adjustment) Metrics\",\n \"description\": \"Dashboard tracking Auto Resource Adjus"
},
{
"path": "docker-compose.yml",
"chars": 790,
"preview": "version: '3'\nservices:\n ui:\n build:\n context: ./ui\n args:\n FLOTILLA_API: http://localhost:3000/api/"
},
{
"path": "docs/ara-command-hash-bug-report.md",
"chars": 17043,
"preview": "# ARA command_hash Bug Report\n\n## Executive Summary\n\nThe Auto Resource Adjustment (ARA) feature has a **critical bug** w"
},
{
"path": "docs/ara-command-hash-fix-locations.md",
"chars": 13072,
"preview": "# ARA command_hash Fix: Implementation Locations\n\n## ✅ STATUS: IMPLEMENTED\n\n**All code changes have been completed.** Th"
},
{
"path": "docs/ara-command-hash-history.md",
"chars": 9671,
"preview": "# History of command_hash Implementation\n\n## Timeline of Changes\n\n### January 17, 2020 - Original Design (Commit a5d7e0f"
},
{
"path": "docs/ara-instrumentation.md",
"chars": 10983,
"preview": "# ARA Instrumentation Guide\n\n## Overview\n\nThis document describes the instrumentation added to measure Auto Resource Adj"
},
{
"path": "docs/ara.md",
"chars": 1811,
"preview": "*Adaptive Resource Allocation for Kubernetes Pods*\n\nAt StitchFix we empower our data scientists to deploy their models a"
},
{
"path": "exceptions/errors.go",
"chars": 687,
"preview": "package exceptions\n\n//\n// MalformedInput describes malformed or otherwise incorrect input\n//\ntype MalformedInput struct "
},
{
"path": "execution/adapter/eks_adapter.go",
"chars": 25292,
"preview": "package adapter\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"fmt\"\n\t\"os\"\n\t\"regexp\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/aws/aws-sdk-go/aw"
},
{
"path": "execution/adapter/eks_adapter_test.go",
"chars": 20293,
"preview": "package adapter\n\nimport (\n\t\"context\"\n\t\"database/sql\"\n\t\"errors\"\n\t\"testing\"\n\n\t\"github.com/stitchfix/flotilla-os/config\"\n\t\""
},
{
"path": "execution/engine/dcm.go",
"chars": 6041,
"preview": "package engine\n\nimport (\n\t\"context\"\n\t\"github.com/aws/aws-sdk-go/aws\"\n\t\"github.com/aws/aws-sdk-go/aws/session\"\n\t\"github.c"
},
{
"path": "execution/engine/eks_engine.go",
"chars": 21263,
"preview": "package engine\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"fmt\"\n\t\"github.com/go-redis/redis\"\n\t\"github.com/stitchfix/flotilla-os/util"
},
{
"path": "execution/engine/emr_engine.go",
"chars": 38140,
"preview": "package engine\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"os\"\n\t\"github.com/aws/aws-sdk-go/aws\"\n\t\"github.com"
},
{
"path": "execution/engine/engine.go",
"chars": 2225,
"preview": "package engine\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"github.com/pkg/errors\"\n\t\"github.com/stitchfix/flotilla-os/config\"\n\t\"github."
},
{
"path": "flotilla/app.go",
"chars": 5331,
"preview": "package flotilla\n\nimport (\n\t\"context\"\n\t\"github.com/stitchfix/flotilla-os/clients/middleware\"\n\t\"github.com/stitchfix/flot"
},
{
"path": "flotilla/endpoints.go",
"chars": 32434,
"preview": "package flotilla\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"net/http\"\n\t\"net/url\"\n\t\"strconv\"\n\t\"strings\"\n\n\t\"github.com/gorilla/mu"
},
{
"path": "flotilla/endpoints_test.go",
"chars": 23539,
"preview": "package flotilla\n\nimport (\n\t\"bytes\"\n\t\"encoding/json\"\n\t\"net/http/httptest\"\n\t\"testing\"\n\n\t\"github.com/stitchfix/flotilla-os"
},
{
"path": "flotilla/router.go",
"chars": 4824,
"preview": "package flotilla\n\nimport (\n\tmuxtrace \"gopkg.in/DataDog/dd-trace-go.v1/contrib/gorilla/mux\"\n)\n\n// NewRouter creates and r"
},
{
"path": "go.mod",
"chars": 4509,
"preview": "module github.com/stitchfix/flotilla-os\n\ngo 1.26.1\n\nrequire (\n\tgithub.com/DataDog/datadog-go/v5 v5.1.0\n\tgithub.com/Maste"
},
{
"path": "go.sum",
"chars": 121314,
"preview": "cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=\ncloud.google.com/go v0.34.0/go.mod h1"
},
{
"path": "log/event.go",
"chars": 2392,
"preview": "package log\n\nimport (\n\t\"errors\"\n\t\"github.com/stitchfix/flotilla-os/clients/httpclient\"\n\t\"log\"\n\t\"os\"\n\t\"time\"\n)\n\n//\n// Eve"
},
{
"path": "log/event_test.go",
"chars": 940,
"preview": "package log\n\nimport (\n\t\"encoding/json\"\n\t\"net/http\"\n\t\"net/http/httptest\"\n\t\"testing\"\n\t\"time\"\n)\n\ntype TestDomainSpecificEve"
},
{
"path": "log/logger.go",
"chars": 987,
"preview": "package log\n\nimport \"github.com/go-kit/kit/log\"\n\n//\n// Logger interface, supports log messages and \"events\"\n// where an "
},
{
"path": "log/logger_test.go",
"chars": 1289,
"preview": "package log\n\nimport (\n\t\"testing\"\n)\n\ntype testLogger struct {\n\tkeyvals []interface{}\n}\n\nfunc (tl *testLogger) Log(keyvals"
},
{
"path": "main.go",
"chars": 4166,
"preview": "package main\n\nimport (\n\t\"fmt\"\n\tgklog \"github.com/go-kit/kit/log\"\n\t\"github.com/pkg/errors\"\n\t\"github.com/stitchfix/flotill"
},
{
"path": "queue/manager.go",
"chars": 1886,
"preview": "package queue\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"github.com/pkg/errors\"\n\t\"github.com/stitchfix/flotilla-os/config\"\n\t\"github.c"
},
{
"path": "queue/sqs_manager.go",
"chars": 13390,
"preview": "package queue\n\nimport (\n\t\"context\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"github.com/aws/aws-sdk-go/aws\"\n\t\"github.com/aws/aws-sdk-go/"
},
{
"path": "queue/sqs_manager_test.go",
"chars": 6219,
"preview": "package queue\n\nimport (\n\t\"context\"\n\t\"encoding/json\"\n\t\"errors\"\n\t\"github.com/aws/aws-sdk-go/service/sqs\"\n\t\"github.com/stit"
},
{
"path": "services/definition.go",
"chars": 4443,
"preview": "package services\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"github.com/stitchfix/flotilla-os/exceptions\"\n\t\"github.com/stitchfix/floti"
},
{
"path": "services/definition_test.go",
"chars": 3621,
"preview": "package services\n\nimport (\n\t\"context\"\n\t\"github.com/stitchfix/flotilla-os/state\"\n\t\"github.com/stitchfix/flotilla-os/testu"
},
{
"path": "services/execution.go",
"chars": 27811,
"preview": "package services\n\nimport (\n\t\"context\"\n\t\"crypto/md5\"\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"math/rand\"\n\t\"regexp\"\n\t\"slices\"\n\t"
},
{
"path": "services/execution_test.go",
"chars": 29664,
"preview": "package services\n\nimport (\n\t\"context\"\n\t\"crypto/md5\"\n\t\"fmt\"\n\t\"log\"\n\t\"testing\"\n\n\t\"github.com/aws/aws-sdk-go/aws\"\n\t\"github."
},
{
"path": "services/logs.go",
"chars": 2092,
"preview": "package services\n\nimport (\n\t\"context\"\n\t\"github.com/aws/aws-sdk-go/aws\"\n\t\"github.com/stitchfix/flotilla-os/clients/logs\"\n"
},
{
"path": "services/logs_test.go",
"chars": 1939,
"preview": "package services\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stitchfix/flotilla-os/state\"\n\t\"github.com/stitchfix/flotilla-os/test"
},
{
"path": "services/template.go",
"chars": 6366,
"preview": "package services\n\nimport (\n\t\"context\"\n\t\"reflect\"\n\t\"strings\"\n\n\t\"github.com/stitchfix/flotilla-os/config\"\n\t\"github.com/sti"
},
{
"path": "services/worker.go",
"chars": 2239,
"preview": "package services\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"github.com/stitchfix/flotilla-os/config\"\n\t\"github.com/stitchfix/flotilla-"
},
{
"path": "state/manager.go",
"chars": 4278,
"preview": "package state\n\nimport (\n\t\"context\"\n\t\"github.com/pkg/errors\"\n\t\"github.com/stitchfix/flotilla-os/config\"\n\t\"github.com/stit"
},
{
"path": "state/models.go",
"chars": 40898,
"preview": "package state\n\nimport (\n\t\"bytes\"\n\t\"database/sql\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"os\"\n\t\"reflect\"\n\t\"regexp\"\n\t\"sort\"\n\t\"strconv\"\n\t"
},
{
"path": "state/models_test.go",
"chars": 4228,
"preview": "package state\n\nimport (\n\t\"os\"\n\t\"reflect\"\n\t\"strings\"\n\t\"testing\"\n)\n\nfunc TestMergeMaps_Simple(t *testing.T) {\n\tmapA := map"
},
{
"path": "state/pg_queries.go",
"chars": 12008,
"preview": "package state\n\n// DefinitionSelect postgres specific query for definitions\nconst DefinitionSelect = `\nselect td.definiti"
},
{
"path": "state/pg_state_manager.go",
"chars": 61665,
"preview": "package state\n\nimport (\n\t\"context\"\n\t\"database/sql/driver\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"time\"\n\n\t\"github.com/stitchfix/flotil"
},
{
"path": "state/pg_state_manager_test.go",
"chars": 30102,
"preview": "package state\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"log\"\n\t\"os\"\n\t\"testing\"\n\t\"time\"\n\n\tgklog \"github.com/go-kit/kit/log\"\n\tflotillaL"
},
{
"path": "testutils/mocks.go",
"chars": 20215,
"preview": "package testutils\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"math\"\n\t\"net/http\"\n\t\"testing\"\n\n\t\"github.com/aws/aws-sdk-go/aws\"\n\n\t\"github"
},
{
"path": "tracing/tracing.go",
"chars": 1231,
"preview": "package tracing\n\nimport (\n\t\"context\"\n\t\"time\"\n\n\t\"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer\"\n)\n\n// TraceJob starts or"
},
{
"path": "ui/.gitignore",
"chars": 327,
"preview": "# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.\n\n# dependencies\n/node_modules\n/.pn"
},
{
"path": "ui/.prettierrc",
"chars": 46,
"preview": "{\n \"trailingComma\": \"es5\",\n \"semi\": false\n}\n"
},
{
"path": "ui/Dockerfile",
"chars": 179,
"preview": "FROM node:carbon\nWORKDIR /usr/src/app\nADD . /usr/src/app\nRUN npm install -g serve\nRUN npm install\nARG FLOTILLA_API\nARG D"
},
{
"path": "ui/README.md",
"chars": 731,
"preview": "# Flotilla UI\n\nThe Flotilla UI is a React application bundled along with the rest of Flotilla. If you are running the en"
},
{
"path": "ui/package.json",
"chars": 2460,
"preview": "{\n \"name\": \"flotilla\",\n \"version\": \"5.1.1\",\n \"dependencies\": {\n \"@blueprintjs/core\": \"3.15.1\",\n \"@blueprintjs/d"
},
{
"path": "ui/public/index.html",
"chars": 496,
"preview": "<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\" />\n <link rel=\"icon\" href=\"%PUBLIC_URL%/favicon.i"
},
{
"path": "ui/src/api.ts",
"chars": 679,
"preview": "import FlotillaClient from \"./helpers/FlotillaClient\"\n\nconst err =\n \"Base URL undefined. If you are running this in dev"
},
{
"path": "ui/src/components/ARASwitch.tsx",
"chars": 2449,
"preview": "import * as React from \"react\"\nimport { get } from \"lodash\"\nimport { Tag, Colors, Checkbox, Intent } from \"@blueprintjs/"
},
{
"path": "ui/src/components/App.tsx",
"chars": 2432,
"preview": "import * as React from \"react\"\nimport { BrowserRouter, Route, Switch, Redirect } from \"react-router-dom\"\nimport Tasks fr"
},
{
"path": "ui/src/components/Attribute.tsx",
"chars": 2041,
"preview": "import * as React from \"react\"\nimport { Tag, Tooltip, Icon, Intent } from \"@blueprintjs/core\"\nimport CopyToClipboard fro"
},
{
"path": "ui/src/components/AutoscrollSwitch.tsx",
"chars": 575,
"preview": "import * as React from \"react\"\nimport { useDispatch, useSelector } from \"react-redux\"\nimport { Switch } from \"@blueprint"
},
{
"path": "ui/src/components/BaseTaskForm.tsx",
"chars": 3354,
"preview": "import * as React from \"react\"\nimport { FormGroup, Classes } from \"@blueprintjs/core\"\nimport { FastField, FormikProps } "
},
{
"path": "ui/src/components/CloudtrailRecords.tsx",
"chars": 621,
"preview": "import * as React from \"react\"\nimport { CloudtrailRecord } from \"../types\"\nimport { HTMLTable } from \"@blueprintjs/core\""
},
{
"path": "ui/src/components/ClusterSelect.tsx",
"chars": 1598,
"preview": "import * as React from \"react\"\nimport { get, isArray } from \"lodash\"\nimport Creatable from \"react-select/lib/Creatable\"\n"
},
{
"path": "ui/src/components/CreateTaskForm.tsx",
"chars": 3527,
"preview": "import * as React from \"react\"\nimport { RouteComponentProps } from \"react-router-dom\"\nimport { Button, Intent, FormGroup"
},
{
"path": "ui/src/components/DeleteTaskButton.tsx",
"chars": 2699,
"preview": "import * as React from \"react\"\nimport { Button, Dialog, Intent, Classes } from \"@blueprintjs/core\"\nimport { withRouter, "
},
{
"path": "ui/src/components/Duration.tsx",
"chars": 1186,
"preview": "import * as React from \"react\"\nimport prettyMS from \"pretty-ms\"\nimport calculateDuration from \"../helpers/calculateDurat"
},
{
"path": "ui/src/components/EngineTag.tsx",
"chars": 240,
"preview": "import * as React from \"react\"\nimport { Tag } from \"@blueprintjs/core\"\nimport { ExecutionEngine } from \"../types\"\n\nconst"
},
{
"path": "ui/src/components/EnvFieldArray.tsx",
"chars": 2298,
"preview": "import * as React from \"react\"\nimport { FieldArray, FastField, FormikErrors } from \"formik\"\nimport { get } from \"lodash\""
},
{
"path": "ui/src/components/EnvList.tsx",
"chars": 493,
"preview": "import * as React from \"react\"\nimport { isEmpty, isArray } from \"lodash\"\nimport { Env } from \"../types\"\nimport Attribute"
},
{
"path": "ui/src/components/EnvQueryFilter.tsx",
"chars": 5050,
"preview": "import * as React from \"react\"\nimport { Button, FormGroup, Classes, Intent } from \"@blueprintjs/core\"\nimport { Env } fro"
},
{
"path": "ui/src/components/ErrorCallout.tsx",
"chars": 845,
"preview": "import * as React from \"react\"\nimport { Callout, Intent } from \"@blueprintjs/core\"\nimport { get } from \"lodash\"\nimport {"
},
{
"path": "ui/src/components/FieldError.tsx",
"chars": 223,
"preview": "import * as React from \"react\"\nimport { Colors } from \"@blueprintjs/core\"\n\nconst FieldError: React.FunctionComponent = ("
},
{
"path": "ui/src/components/GenericMultiSelect.tsx",
"chars": 796,
"preview": "import * as React from \"react\"\nimport { isArray } from \"lodash\"\nimport Creatable from \"react-select/lib/Creatable\"\nimpor"
},
{
"path": "ui/src/components/GroupNameSelect.tsx",
"chars": 2230,
"preview": "import * as React from \"react\"\nimport { get } from \"lodash\"\nimport Creatable from \"react-select/lib/Creatable\"\nimport Re"
},
{
"path": "ui/src/components/ISO8601AttributeValue.tsx",
"chars": 903,
"preview": "import * as React from \"react\"\nimport moment from \"moment\"\nimport { Classes } from \"@blueprintjs/core\"\n\nconst ISO8601Att"
},
{
"path": "ui/src/components/ListFiltersDropdown.tsx",
"chars": 475,
"preview": "import * as React from \"react\"\nimport { Button, Tooltip, Popover, Position, Card } from \"@blueprintjs/core\"\n\nconst ListF"
},
{
"path": "ui/src/components/ListRequest.tsx",
"chars": 4378,
"preview": "import * as React from \"react\"\nimport { get, isEqual, isEmpty, Omit } from \"lodash\"\nimport Request, { ChildProps as Requ"
},
{
"path": "ui/src/components/Log.tsx",
"chars": 1853,
"preview": "import * as React from \"react\"\nimport { connect, ConnectedProps } from \"react-redux\"\nimport Ansi from \"ansi-to-react\"\nim"
},
{
"path": "ui/src/components/LogProcessor.tsx",
"chars": 3091,
"preview": "import * as React from \"react\"\nimport { get } from \"lodash\"\nimport ReactResizeDetector from \"react-resize-detector\"\nimpo"
},
{
"path": "ui/src/components/LogRequesterCloudWatchLogs.tsx",
"chars": 4304,
"preview": "import * as React from \"react\"\nimport { has, isEmpty } from \"lodash\"\nimport { connect, ConnectedProps } from \"react-redu"
},
{
"path": "ui/src/components/LogRequesterS3.tsx",
"chars": 3669,
"preview": "import * as React from \"react\"\nimport { connect, ConnectedProps } from \"react-redux\"\nimport api from \"../api\"\nimport Log"
},
{
"path": "ui/src/components/LogVirtualized.tsx",
"chars": 7198,
"preview": "import * as React from \"react\"\nimport { FixedSizeList as List } from \"react-window\"\nimport { connect, ConnectedProps } f"
},
{
"path": "ui/src/components/LogVirtualizedRow.tsx",
"chars": 1344,
"preview": "import * as React from \"react\"\nimport Ansi from \"ansi-to-react\"\nimport { get } from \"lodash\"\nimport { ListChildComponent"
},
{
"path": "ui/src/components/LogVirtualizedSearch.tsx",
"chars": 1533,
"preview": "import * as React from \"react\"\nimport { ButtonGroup, Button, Spinner } from \"@blueprintjs/core\"\n\ntype Props = {\n onChan"
},
{
"path": "ui/src/components/Navigation.tsx",
"chars": 1330,
"preview": "import * as React from \"react\"\nimport { Link, NavLink } from \"react-router-dom\"\nimport {\n ButtonGroup,\n Navbar,\n Navb"
},
{
"path": "ui/src/components/NodeLifecycleSelect.tsx",
"chars": 864,
"preview": "import * as React from \"react\"\nimport Select from \"react-select\"\nimport { SelectOption, SelectProps, NodeLifecycle } fro"
},
{
"path": "ui/src/components/Pagination.tsx",
"chars": 927,
"preview": "import * as React from \"react\"\nimport { Button, ButtonGroup } from \"@blueprintjs/core\"\n\nexport type Props = {\n updatePa"
},
{
"path": "ui/src/components/QueryParams.tsx",
"chars": 1093,
"preview": "import * as React from \"react\"\nimport * as qs from \"qs\"\nimport { withRouter, RouteComponentProps } from \"react-router-do"
},
{
"path": "ui/src/components/Request.tsx",
"chars": 2073,
"preview": "import * as React from \"react\"\nimport { AxiosError } from \"axios\"\n\nexport enum RequestStatus {\n READY = \"READY\",\n NOT_"
},
{
"path": "ui/src/components/ResourceUsageValue.tsx",
"chars": 992,
"preview": "import { Tooltip, Colors } from \"@blueprintjs/core\"\n\n\nconst isLessThanPct = (x: number, y: number, pct: number): boolean"
},
{
"path": "ui/src/components/Run.tsx",
"chars": 12564,
"preview": "import * as React from \"react\"\nimport { connect, ConnectedProps } from \"react-redux\"\nimport { get } from \"lodash\"\nimport"
},
{
"path": "ui/src/components/RunAttributes.tsx",
"chars": 3448,
"preview": "import * as React from \"react\"\nimport { Card, Pre, Tag } from \"@blueprintjs/core\"\nimport { Run, ExecutionEngine } from \""
},
{
"path": "ui/src/components/RunDebugAttributes.tsx",
"chars": 2847,
"preview": "import * as React from \"react\"\nimport { Card, Icon } from \"@blueprintjs/core\"\nimport urljoin from \"url-join\"\nimport { Ru"
},
{
"path": "ui/src/components/RunEvents.tsx",
"chars": 2611,
"preview": "import * as React from \"react\"\nimport { RunStatus, RunTabId } from \"../types\"\nimport Request, { RequestStatus } from \"./"
},
{
"path": "ui/src/components/RunSidebar.tsx",
"chars": 1377,
"preview": "import * as React from \"react\"\nimport { get } from \"lodash\"\nimport { Card } from \"@blueprintjs/core\"\nimport JsonView fro"
},
{
"path": "ui/src/components/RunStatusSelect.tsx",
"chars": 1007,
"preview": "import * as React from \"react\"\nimport { isArray } from \"lodash\"\nimport Select from \"react-select\"\nimport { SelectOption,"
},
{
"path": "ui/src/components/RunTag.tsx",
"chars": 571,
"preview": "import * as React from \"react\"\nimport { Run } from \"../types\"\nimport { Tag, Colors } from \"@blueprintjs/core\"\nimport { R"
},
{
"path": "ui/src/components/Runs.tsx",
"chars": 7421,
"preview": "import * as React from \"react\"\nimport { Link } from \"react-router-dom\"\nimport { get, omit, isArray, isString } from \"lod"
},
{
"path": "ui/src/components/SettingsButton.tsx",
"chars": 3542,
"preview": "import * as React from \"react\"\nimport { useSelector, useDispatch } from \"react-redux\"\nimport { Formik, Form, FastField, "
},
{
"path": "ui/src/components/SortableTh.tsx",
"chars": 675,
"preview": "import * as React from \"react\"\nimport { SortOrder } from \"../types\"\n\nexport type Props = {\n isSortable: boolean\n isAct"
},
{
"path": "ui/src/components/StopRunButton.tsx",
"chars": 2803,
"preview": "import * as React from \"react\"\nimport { Button, Dialog, Intent, Classes } from \"@blueprintjs/core\"\nimport Request, { Chi"
},
{
"path": "ui/src/components/Table.tsx",
"chars": 1864,
"preview": "import * as React from \"react\"\nimport { HTMLTable, Callout } from \"@blueprintjs/core\"\nimport { isArray } from \"lodash\"\ni"
},
{
"path": "ui/src/components/TagsSelect.tsx",
"chars": 1310,
"preview": "import * as React from \"react\"\nimport { get, isArray } from \"lodash\"\nimport Creatable from \"react-select/lib/Creatable\"\n"
},
{
"path": "ui/src/components/Task.tsx",
"chars": 3428,
"preview": "import * as React from \"react\"\nimport { Switch, Route, RouteComponentProps } from \"react-router-dom\"\nimport { get } from"
},
{
"path": "ui/src/components/TaskDetails.tsx",
"chars": 7265,
"preview": "import * as React from \"react\"\nimport { Link } from \"react-router-dom\"\nimport {\n Collapse,\n Card,\n ButtonGroup,\n Pre"
},
{
"path": "ui/src/components/TaskExecutionForm.tsx",
"chars": 8283,
"preview": "import * as React from \"react\"\nimport { Formik, Form, FastField, Field } from \"formik\"\nimport * as Yup from \"yup\"\nimport"
},
{
"path": "ui/src/components/TaskRuns.tsx",
"chars": 7906,
"preview": "import * as React from \"react\"\nimport { Link } from \"react-router-dom\"\nimport { get, omit, isArray, isString } from \"lod"
},
{
"path": "ui/src/components/Tasks.tsx",
"chars": 5011,
"preview": "import * as React from \"react\"\nimport { Link } from \"react-router-dom\"\nimport { get, omit } from \"lodash\"\nimport { Debou"
},
{
"path": "ui/src/components/Template.tsx",
"chars": 1627,
"preview": "import * as React from \"react\"\nimport { Switch, Route, RouteComponentProps } from \"react-router-dom\"\nimport Request, { C"
},
{
"path": "ui/src/components/TemplateDetails.tsx",
"chars": 5644,
"preview": "import * as React from \"react\"\nimport { Link } from \"react-router-dom\"\nimport {\n Collapse,\n Card,\n ButtonGroup,\n Cla"
},
{
"path": "ui/src/components/TemplateExecutionForm.tsx",
"chars": 9311,
"preview": "import * as React from \"react\"\nimport { Formik, Form, FastField, Field } from \"formik\"\nimport * as Yup from \"yup\"\nimport"
},
{
"path": "ui/src/components/TemplateHistoryTable.tsx",
"chars": 7983,
"preview": "import * as React from \"react\"\nimport { Link } from \"react-router-dom\"\nimport { get, omit, isArray, isString } from \"lod"
},
{
"path": "ui/src/components/TemplateRunForm.tsx",
"chars": 11733,
"preview": "import * as React from \"react\"\nimport { Formik, Form, FastField, Field } from \"formik\"\nimport * as Yup from \"yup\"\nimport"
},
{
"path": "ui/src/components/Templates.tsx",
"chars": 3055,
"preview": "import * as React from \"react\"\nimport { Link } from \"react-router-dom\"\nimport { get, omit } from \"lodash\"\nimport { Spinn"
},
{
"path": "ui/src/components/Toaster.ts",
"chars": 125,
"preview": "import { Position, Toaster } from \"@blueprintjs/core\"\n\nexport default Toaster.create({\n position: Position.BOTTOM_RIGHT"
},
{
"path": "ui/src/components/Toggler.tsx",
"chars": 644,
"preview": "import * as React from \"react\"\n\ntype Props = {\n children: (props: ChildProps) => React.ReactNode\n}\n\ntype State = {\n is"
},
{
"path": "ui/src/components/UpdateTaskForm.tsx",
"chars": 4355,
"preview": "import * as React from \"react\"\nimport { RouteComponentProps } from \"react-router-dom\"\nimport { Button, Intent, Spinner }"
},
{
"path": "ui/src/components/ViewHeader.tsx",
"chars": 781,
"preview": "import * as React from \"react\"\nimport { Link } from \"react-router-dom\"\nimport { Breadcrumbs, IBreadcrumbProps, Classes }"
},
{
"path": "ui/src/components/__tests__/BaseTaskForm.spec.tsx",
"chars": 4805,
"preview": "import * as React from \"react\"\nimport { mount } from \"enzyme\"\nimport { Formik, FastField } from \"formik\"\nimport { FormGr"
},
{
"path": "ui/src/components/__tests__/ClusterSelect.spec.tsx",
"chars": 2432,
"preview": "import React from \"react\"\nimport { mount } from \"enzyme\"\nimport Creatable from \"react-select/lib/Creatable\"\nimport Conne"
},
{
"path": "ui/src/components/__tests__/CreateTaskForm.spec.tsx",
"chars": 3150,
"preview": "import * as React from \"react\"\nimport flushPromiseQueue from \"flush-promises\"\nimport { mount, ReactWrapper } from \"enzym"
},
{
"path": "ui/src/components/__tests__/DeleteTaskButton.spec.tsx",
"chars": 1402,
"preview": "import * as React from \"react\"\nimport { MemoryRouter } from \"react-router-dom\"\nimport { mount } from \"enzyme\"\nimport Con"
},
{
"path": "ui/src/components/__tests__/EnvFieldArray.spec.tsx",
"chars": 1926,
"preview": "import * as React from \"react\"\nimport { mount, ReactWrapper } from \"enzyme\"\nimport { Formik, FastField } from \"formik\"\ni"
},
{
"path": "ui/src/components/__tests__/GroupNameSelect.spec.tsx",
"chars": 1640,
"preview": "import React from \"react\"\nimport { mount } from \"enzyme\"\nimport Creatable from \"react-select/lib/Creatable\"\nimport Conne"
},
{
"path": "ui/src/components/__tests__/ListRequest.spec.tsx",
"chars": 8182,
"preview": "import * as React from \"react\"\nimport { mount, ReactWrapper } from \"enzyme\"\nimport { ListRequest, Props, ChildProps } fr"
},
{
"path": "ui/src/components/__tests__/LogProcessor.spec.tsx",
"chars": 696,
"preview": "import * as React from \"react\"\nimport { shallow } from \"enzyme\"\nimport { LogProcessor } from \"../LogProcessor\"\n\njest.moc"
},
{
"path": "ui/src/components/__tests__/LogVirtualized.spec.tsx",
"chars": 3834,
"preview": "import * as React from \"react\"\nimport { mount, shallow } from \"enzyme\"\nimport { LogVirtualized, Props } from \"../LogVirt"
},
{
"path": "ui/src/components/__tests__/LogVirtualizedSearch.spec.tsx",
"chars": 1957,
"preview": "import * as React from \"react\"\nimport { mount, ReactWrapper } from \"enzyme\"\nimport { DebounceInput } from \"react-debounc"
},
{
"path": "ui/src/components/__tests__/Pagination.spec.tsx",
"chars": 1878,
"preview": "import * as React from \"react\"\nimport { mount, ReactWrapper } from \"enzyme\"\nimport { Button, ButtonGroup } from \"@bluepr"
},
{
"path": "ui/src/components/__tests__/QueryParams.spec.tsx",
"chars": 773,
"preview": "import * as React from \"react\"\nimport { mount } from \"enzyme\"\nimport { MemoryRouter } from \"react-router-dom\"\nimport qs "
},
{
"path": "ui/src/components/__tests__/Request.spec.tsx",
"chars": 3729,
"preview": "import * as React from \"react\"\nimport { mount, shallow } from \"enzyme\"\nimport flushPromises from \"flush-promises\"\nimport"
},
{
"path": "ui/src/components/__tests__/Run.spec.tsx",
"chars": 4731,
"preview": "import * as React from \"react\"\nimport { mount } from \"enzyme\"\nimport { MemoryRouter } from \"react-router-dom\"\nimport { R"
},
{
"path": "ui/src/components/__tests__/Runs.spec.tsx",
"chars": 3441,
"preview": "import React from \"react\"\nimport { mount } from \"enzyme\"\nimport { MemoryRouter } from \"react-router-dom\"\nimport Connecte"
},
{
"path": "ui/src/components/__tests__/StopRunButton.spec.tsx",
"chars": 1393,
"preview": "import * as React from \"react\"\nimport { MemoryRouter } from \"react-router-dom\"\nimport { mount } from \"enzyme\"\nimport Con"
},
{
"path": "ui/src/components/__tests__/TaskRuns.spec.tsx",
"chars": 3780,
"preview": "import React from \"react\"\nimport { mount } from \"enzyme\"\nimport { MemoryRouter } from \"react-router-dom\"\nimport Connecte"
},
{
"path": "ui/src/components/__tests__/Tasks.spec.tsx",
"chars": 3481,
"preview": "import React from \"react\"\nimport { mount } from \"enzyme\"\nimport { MemoryRouter } from \"react-router-dom\"\nimport Connecte"
},
{
"path": "ui/src/components/__tests__/UpdateTaskForm.spec.tsx",
"chars": 3432,
"preview": "import * as React from \"react\"\nimport flushPromiseQueue from \"flush-promises\"\nimport { mount, ReactWrapper } from \"enzym"
},
{
"path": "ui/src/constants.ts",
"chars": 1300,
"preview": "import { EnhancedRunStatus, RunStatus } from \"./types\"\nimport { Colors } from \"@blueprintjs/core\"\nimport { ReactJsonView"
},
{
"path": "ui/src/helpers/FlotillaClient.ts",
"chars": 7683,
"preview": "import axios, { AxiosInstance, AxiosError, AxiosResponse } from \"axios\"\nimport * as qs from \"qs\"\nimport { has, omit, Omi"
},
{
"path": "ui/src/helpers/__mocks__/FlotillaClient.ts",
"chars": 6206,
"preview": "import {\n CreateTaskPayload,\n ListClustersResponse,\n ListGroupsResponse,\n ListRunParams,\n ListRunResponse,\n ListTa"
},
{
"path": "ui/src/helpers/__tests__/FlotillaClient.spec.ts",
"chars": 5978,
"preview": "import axios from \"axios\"\nimport MockAdapter from \"axios-mock-adapter\"\nimport FlotillaClient from \"../FlotillaClient\"\nim"
},
{
"path": "ui/src/helpers/__tests__/getInitialValuesForTaskRun.spec.ts",
"chars": 812,
"preview": "import getInitialValuesForTaskRun from \"../getInitialValuesForTaskRun\"\nimport { createMockTaskObject } from \"../testHelp"
},
{
"path": "ui/src/helpers/__tests__/pageToOffsetLimit.spec.ts",
"chars": 343,
"preview": "import pageToOffsetLimit from \"../pageToOffsetLimit\"\n\ndescribe(\"pageToOffsetLimit\", () => {\n it(\"works correctly\", () ="
},
{
"path": "ui/src/helpers/calculateDuration.ts",
"chars": 259,
"preview": "const calculateDuration = (\n start: string,\n end: string | null | undefined\n): number => {\n const s = Date.parse(star"
},
{
"path": "ui/src/helpers/constructDefaultObjectFromJsonSchema.ts",
"chars": 1311,
"preview": "import { get, isObject } from \"lodash\"\n\nconst DEFAULT_ARRAY: any[] = []\nconst DEFAULT_STRING = \"\"\nconst DEFAULT_NUM = 0\n"
},
{
"path": "ui/src/helpers/getEnhancedRunStatus.ts",
"chars": 371,
"preview": "import { Run, RunStatus, EnhancedRunStatus } from \"../types\"\n\nconst getEnhancedRunStatus = (run: Run): EnhancedRunStatus"
},
{
"path": "ui/src/helpers/getInitialValuesForExecutionForm.ts",
"chars": 2548,
"preview": "import { get } from \"lodash\"\nimport getOwnerIdRunTagFromCookies from \"./getOwnerIdRunTagFromCookies\"\nimport {\n Executab"
},
{
"path": "ui/src/helpers/getOwnerIdRunTagFromCookies.ts",
"chars": 921,
"preview": "import { get, has } from \"lodash\"\nimport cookie from \"cookie\"\n\nconst getOwnerIdRunTagFromCookies = (): string => {\n // "
},
{
"path": "ui/src/helpers/pageToOffsetLimit.ts",
"chars": 167,
"preview": "const pageToOffsetLimit = ({\n page,\n limit,\n}: {\n page: number\n limit: number\n}) => ({\n offset: (page - 1) * limit,"
},
{
"path": "ui/src/helpers/runFormHelpers.ts",
"chars": 1067,
"preview": "import { FieldSpec } from \"../types\"\n\nexport const clusterFieldSpec: FieldSpec = {\n name: \"cluster\",\n label: \"Cluster\""
},
{
"path": "ui/src/helpers/selectHelpers.ts",
"chars": 1769,
"preview": "import { isArray } from \"lodash\"\nimport { SelectOption } from \"../types\"\nimport { ValueType, Theme } from \"react-select/"
},
{
"path": "ui/src/helpers/taskFormHelpers.ts",
"chars": 1345,
"preview": "import { FieldSpec } from \"../types\"\n\nexport const aliasFieldSpec: FieldSpec = {\n name: \"alias\",\n label: \"alias\",\n de"
},
{
"path": "ui/src/helpers/testHelpers.ts",
"chars": 2325,
"preview": "import { FormikActions } from \"formik\"\nimport { createMemoryHistory, createLocation } from \"history\"\nimport { RouteCompo"
},
{
"path": "ui/src/index.css",
"chars": 7810,
"preview": ":root {\n --spacing: 12px;\n --blueprint-navbar-height: 50px;\n --default-border-color: #293742;\n --default-border: 1px"
}
]
// ... and 24 more files (download for full content)
About this extraction
This page contains the full source code of the stitchfix/flotilla-os GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 224 files (1.1 MB), approximately 356.8k tokens, and a symbol index with 1257 extracted functions, classes, methods, constants, and types. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.