Repository: cortexlabs/cortex Branch: master Commit: dc48c02ed50a Files: 702 Total size: 5.2 MB Directory structure: gitextract_xza_bns1/ ├── .circleci/ │ └── config.yml ├── .dockerignore ├── .gitbook.yaml ├── .github/ │ ├── ISSUE_TEMPLATE/ │ │ ├── bug-report.md │ │ ├── feature-request.md │ │ └── question.md │ └── pull_request_template.md ├── .gitignore ├── CONTRIBUTING.md ├── LICENSE ├── Makefile ├── README.md ├── build/ │ ├── amend-image.sh │ ├── amend-images.sh │ ├── build-image.sh │ ├── build-images.sh │ ├── cli.sh │ ├── generate_ami_mapping.go │ ├── images.sh │ ├── lint-docs.sh │ ├── lint.sh │ ├── push-image.sh │ ├── push-images.sh │ └── test.sh ├── cli/ │ ├── cluster/ │ │ ├── delete.go │ │ ├── deploy.go │ │ ├── errors.go │ │ ├── get.go │ │ ├── info.go │ │ ├── lib_http_client.go │ │ ├── logs.go │ │ └── refresh.go │ ├── cmd/ │ │ ├── cluster.go │ │ ├── completion.go │ │ ├── const.go │ │ ├── delete.go │ │ ├── deploy.go │ │ ├── describe.go │ │ ├── env.go │ │ ├── errors.go │ │ ├── get.go │ │ ├── lib_apis.go │ │ ├── lib_async_apis.go │ │ ├── lib_aws_creds.go │ │ ├── lib_batch_apis.go │ │ ├── lib_cli_config.go │ │ ├── lib_client_id.go │ │ ├── lib_cluster_config.go │ │ ├── lib_manager.go │ │ ├── lib_realtime_apis.go │ │ ├── lib_task_apis.go │ │ ├── lib_traffic_splitters.go │ │ ├── lib_watch.go │ │ ├── logs.go │ │ ├── refresh.go │ │ ├── root.go │ │ └── version.go │ ├── lib/ │ │ └── routines/ │ │ └── routines.go │ ├── main.go │ └── types/ │ ├── cliconfig/ │ │ ├── cli_config.go │ │ ├── config_key.go │ │ ├── environment.go │ │ └── errors.go │ └── flags/ │ ├── errors.go │ └── output_type.go ├── cmd/ │ ├── activator/ │ │ └── main.go │ ├── async-gateway/ │ │ └── main.go │ ├── autoscaler/ │ │ └── main.go │ ├── dequeuer/ │ │ └── main.go │ ├── enqueuer/ │ │ └── main.go │ ├── operator/ │ │ └── main.go │ └── proxy/ │ └── main.go ├── dev/ │ ├── build_cli.sh │ ├── create_user.py │ ├── delete_ecr_repos.py │ ├── export_images.sh │ ├── find_missing_docs_links.py │ ├── format.sh │ ├── generate_cli_md.sh │ ├── generate_python_client_md.sh │ ├── get_operator_url.py │ ├── load.go │ ├── minimum_aws_policy.json │ ├── operator_local.sh │ ├── prometheus.md │ ├── registry.sh │ ├── update_cli_config.py │ ├── util.sh │ └── versions.md ├── docs/ │ ├── README.md │ ├── clients/ │ │ ├── cli.md │ │ ├── install.md │ │ ├── python.md │ │ └── uninstall.md │ ├── clusters/ │ │ ├── advanced/ │ │ │ ├── kubectl.md │ │ │ ├── registry.md │ │ │ └── self-hosted-images.md │ │ ├── instances/ │ │ │ ├── multi.md │ │ │ └── spot.md │ │ ├── management/ │ │ │ ├── auth.md │ │ │ ├── create.md │ │ │ ├── delete.md │ │ │ ├── environments.md │ │ │ ├── production.md │ │ │ └── update.md │ │ ├── networking/ │ │ │ ├── api-gateway.md │ │ │ ├── custom-domain.md │ │ │ ├── https.md │ │ │ ├── load-balancers.md │ │ │ └── vpc-peering.md │ │ └── observability/ │ │ ├── alerting.md │ │ ├── logging.md │ │ └── metrics.md │ ├── overview.md │ ├── start.md │ ├── summary.md │ └── workloads/ │ ├── async/ │ │ ├── async.md │ │ ├── autoscaling.md │ │ ├── configuration.md │ │ ├── containers.md │ │ ├── example.md │ │ └── statuses.md │ ├── batch/ │ │ ├── batch.md │ │ ├── configuration.md │ │ ├── containers.md │ │ ├── example.md │ │ ├── jobs.md │ │ └── statuses.md │ ├── realtime/ │ │ ├── autoscaling.md │ │ ├── configuration.md │ │ ├── containers.md │ │ ├── example.md │ │ ├── metrics.md │ │ ├── realtime.md │ │ ├── statuses.md │ │ ├── traffic-splitter.md │ │ └── troubleshooting.md │ └── task/ │ ├── configuration.md │ ├── containers.md │ ├── example.md │ ├── jobs.md │ ├── statuses.md │ └── task.md ├── get-cli.sh ├── go.mod ├── go.sum ├── images/ │ ├── activator/ │ │ └── Dockerfile │ ├── async-gateway/ │ │ └── Dockerfile │ ├── autoscaler/ │ │ └── Dockerfile │ ├── cluster-autoscaler/ │ │ └── Dockerfile │ ├── controller-manager/ │ │ └── Dockerfile │ ├── dequeuer/ │ │ └── Dockerfile │ ├── enqueuer/ │ │ └── Dockerfile │ ├── event-exporter/ │ │ └── Dockerfile │ ├── fluent-bit/ │ │ └── Dockerfile │ ├── grafana/ │ │ └── Dockerfile │ ├── istio-pilot/ │ │ └── Dockerfile │ ├── istio-proxy/ │ │ └── Dockerfile │ ├── kube-rbac-proxy/ │ │ └── Dockerfile │ ├── kubexit/ │ │ └── Dockerfile │ ├── manager/ │ │ └── Dockerfile │ ├── metrics-server/ │ │ └── Dockerfile │ ├── neuron-device-plugin/ │ │ └── Dockerfile │ ├── neuron-scheduler/ │ │ └── Dockerfile │ ├── nvidia-device-plugin/ │ │ └── Dockerfile │ ├── operator/ │ │ └── Dockerfile │ ├── prometheus/ │ │ └── Dockerfile │ ├── prometheus-config-reloader/ │ │ └── Dockerfile │ ├── prometheus-dcgm-exporter/ │ │ └── Dockerfile │ ├── prometheus-kube-state-metrics/ │ │ └── Dockerfile │ ├── prometheus-node-exporter/ │ │ └── Dockerfile │ ├── prometheus-operator/ │ │ └── Dockerfile │ ├── prometheus-statsd-exporter/ │ │ └── Dockerfile │ └── proxy/ │ └── Dockerfile ├── manager/ │ ├── check_cortex_version.sh │ ├── cluster_config_env.py │ ├── debug.sh │ ├── generate_eks.py │ ├── get_api_load_balancer_state.py │ ├── get_operator_load_balancer_state.py │ ├── get_operator_target_group_status.py │ ├── helpers.py │ ├── install.sh │ ├── manifests/ │ │ ├── activator.yaml.j2 │ │ ├── ami.json │ │ ├── apis.yaml.j2 │ │ ├── async-gateway.yaml.j2 │ │ ├── autoscaler.yaml.j2 │ │ ├── cluster-autoscaler.yaml.j2 │ │ ├── default_cortex_cli_config.yaml │ │ ├── event-exporter.yaml │ │ ├── fluent-bit.yaml.j2 │ │ ├── grafana/ │ │ │ ├── grafana-dashboard-async.yaml │ │ │ ├── grafana-dashboard-batch.yaml │ │ │ ├── grafana-dashboard-cluster.yaml │ │ │ ├── grafana-dashboard-control-plane.yaml │ │ │ ├── grafana-dashboard-nodes.yaml │ │ │ ├── grafana-dashboard-realtime.yaml │ │ │ ├── grafana-dashboard-task.yaml │ │ │ └── grafana.yaml.j2 │ │ ├── inferentia.yaml │ │ ├── istio.yaml.j2 │ │ ├── kube-proxy.patch.yaml │ │ ├── metrics-server.yaml │ │ ├── namespaces.yaml │ │ ├── nvidia.yaml │ │ ├── operator.yaml.j2 │ │ ├── prometheus-additional-scrape-configs.yaml.j2 │ │ ├── prometheus-dcgm-exporter.yaml │ │ ├── prometheus-kube-state-metrics.yaml │ │ ├── prometheus-kubelet-exporter.yaml │ │ ├── prometheus-monitoring.yaml │ │ ├── prometheus-node-exporter.yaml │ │ ├── prometheus-operator.yaml │ │ └── prometheus-statsd-exporter.yaml │ ├── refresh.sh │ ├── render_template.py │ ├── requirements.txt │ ├── uninstall.sh │ └── upgrade_kube_proxy_mode.py ├── pkg/ │ ├── activator/ │ │ ├── activator.go │ │ ├── activator_test.go │ │ ├── api_activator.go │ │ ├── api_activator_test.go │ │ ├── handler.go │ │ ├── handler_test.go │ │ ├── helpers.go │ │ └── request_stats.go │ ├── async-gateway/ │ │ ├── endpoint.go │ │ ├── queue.go │ │ ├── service.go │ │ ├── storage.go │ │ └── types.go │ ├── autoscaler/ │ │ ├── async_scaler.go │ │ ├── autoscaler.go │ │ ├── autoscaler_test.go │ │ ├── client.go │ │ ├── handler.go │ │ ├── realtime_scaler.go │ │ ├── recommendations.go │ │ └── scaler_func.go │ ├── config/ │ │ └── config.go │ ├── consts/ │ │ └── consts.go │ ├── crds/ │ │ ├── Makefile │ │ ├── PROJECT │ │ ├── apis/ │ │ │ └── batch/ │ │ │ └── v1alpha1/ │ │ │ ├── batchjob_metrics.go │ │ │ ├── batchjob_types.go │ │ │ ├── groupversion_info.go │ │ │ └── zz_generated.deepcopy.go │ │ ├── config/ │ │ │ ├── crd/ │ │ │ │ ├── bases/ │ │ │ │ │ └── batch.cortex.dev_batchjobs.yaml │ │ │ │ ├── kustomization.yaml │ │ │ │ ├── kustomizeconfig.yaml │ │ │ │ └── patches/ │ │ │ │ ├── cainjection_in_batchjobs.yaml │ │ │ │ └── webhook_in_batchjobs.yaml │ │ │ ├── default/ │ │ │ │ ├── kustomization.yaml │ │ │ │ ├── manager_auth_proxy_patch.yaml │ │ │ │ └── manager_config_patch.yaml │ │ │ ├── manager/ │ │ │ │ ├── controller_manager_config.yaml │ │ │ │ ├── kustomization.yaml │ │ │ │ └── manager.yaml │ │ │ ├── prometheus/ │ │ │ │ ├── kustomization.yaml │ │ │ │ └── monitor.yaml │ │ │ ├── rbac/ │ │ │ │ ├── auth_proxy_client_clusterrole.yaml │ │ │ │ ├── auth_proxy_role.yaml │ │ │ │ ├── auth_proxy_role_binding.yaml │ │ │ │ ├── auth_proxy_service.yaml │ │ │ │ ├── batchjob_editor_role.yaml │ │ │ │ ├── batchjob_viewer_role.yaml │ │ │ │ ├── kustomization.yaml │ │ │ │ ├── leader_election_role.yaml │ │ │ │ ├── leader_election_role_binding.yaml │ │ │ │ ├── role.yaml │ │ │ │ ├── role_binding.yaml │ │ │ │ └── service_account.yaml │ │ │ └── samples/ │ │ │ └── batch_v1alpha1_batchjob.yaml │ │ ├── controllers/ │ │ │ ├── batch/ │ │ │ │ ├── batchjob_controller.go │ │ │ │ ├── batchjob_controller_config.go │ │ │ │ ├── batchjob_controller_helpers.go │ │ │ │ ├── batchjob_controller_test.go │ │ │ │ └── suite_test.go │ │ │ └── errors.go │ │ ├── hack/ │ │ │ ├── boilerplate.go.txt │ │ │ └── run_manager.sh │ │ └── main.go │ ├── dequeuer/ │ │ ├── async_handler.go │ │ ├── async_handler_test.go │ │ ├── async_stats.go │ │ ├── async_stats_test.go │ │ ├── batch_handler.go │ │ ├── batch_handler_test.go │ │ ├── dequeuer.go │ │ ├── dequeuer_test.go │ │ ├── errors.go │ │ ├── http_handler.go │ │ ├── message_handler.go │ │ ├── probes.go │ │ ├── probes_test.go │ │ ├── queue_attributes.go │ │ └── request_stats.go │ ├── enqueuer/ │ │ ├── enqueuer.go │ │ ├── errors.go │ │ ├── helpers.go │ │ └── uploader.go │ ├── health/ │ │ └── health.go │ ├── lib/ │ │ ├── archive/ │ │ │ ├── archive_test.go │ │ │ ├── archiver.go │ │ │ ├── errors.go │ │ │ ├── input.go │ │ │ ├── tar.go │ │ │ ├── tgz.go │ │ │ └── zip.go │ │ ├── aws/ │ │ │ ├── acm.go │ │ │ ├── apigateway.go │ │ │ ├── autoscaling.go │ │ │ ├── aws.go │ │ │ ├── clients.go │ │ │ ├── cloudformation.go │ │ │ ├── cloudwatch.go │ │ │ ├── credentials.go │ │ │ ├── ec2.go │ │ │ ├── ec2_test.go │ │ │ ├── ecr.go │ │ │ ├── eks.go │ │ │ ├── elb.go │ │ │ ├── elbv2.go │ │ │ ├── errors.go │ │ │ ├── gen_resource_metadata.py │ │ │ ├── iam.go │ │ │ ├── resource_metadata.go │ │ │ ├── s3.go │ │ │ ├── servicequotas.go │ │ │ ├── sqs.go │ │ │ └── sts.go │ │ ├── cast/ │ │ │ ├── interface.go │ │ │ └── interface_test.go │ │ ├── configreader/ │ │ │ ├── bool.go │ │ │ ├── bool_list.go │ │ │ ├── bool_ptr.go │ │ │ ├── errors.go │ │ │ ├── float32.go │ │ │ ├── float32_list.go │ │ │ ├── float32_ptr.go │ │ │ ├── float64.go │ │ │ ├── float64_list.go │ │ │ ├── float64_ptr.go │ │ │ ├── int.go │ │ │ ├── int32.go │ │ │ ├── int32_list.go │ │ │ ├── int32_ptr.go │ │ │ ├── int64.go │ │ │ ├── int64_list.go │ │ │ ├── int64_ptr.go │ │ │ ├── int_list.go │ │ │ ├── int_ptr.go │ │ │ ├── interface.go │ │ │ ├── interface_map.go │ │ │ ├── interface_map_list.go │ │ │ ├── interface_test.go │ │ │ ├── reader.go │ │ │ ├── reader_test.go │ │ │ ├── string.go │ │ │ ├── string_list.go │ │ │ ├── string_map.go │ │ │ ├── string_ptr.go │ │ │ ├── types.go │ │ │ └── validators.go │ │ ├── console/ │ │ │ └── format.go │ │ ├── cron/ │ │ │ └── cron.go │ │ ├── debug/ │ │ │ └── debug.go │ │ ├── docker/ │ │ │ ├── docker.go │ │ │ └── errors.go │ │ ├── errors/ │ │ │ ├── error.go │ │ │ ├── errors.go │ │ │ ├── message.go │ │ │ ├── multi.go │ │ │ └── stack.go │ │ ├── exit/ │ │ │ └── exit.go │ │ ├── files/ │ │ │ ├── errors.go │ │ │ ├── files.go │ │ │ └── files_test.go │ │ ├── hash/ │ │ │ └── hash.go │ │ ├── json/ │ │ │ ├── errors.go │ │ │ └── json.go │ │ ├── k8s/ │ │ │ ├── configmap.go │ │ │ ├── deployment.go │ │ │ ├── errors.go │ │ │ ├── hpa.go │ │ │ ├── ingress.go │ │ │ ├── job.go │ │ │ ├── k8s.go │ │ │ ├── node.go │ │ │ ├── parsers.go │ │ │ ├── pod.go │ │ │ ├── quantity.go │ │ │ ├── secret.go │ │ │ ├── service.go │ │ │ ├── virtual_service.go │ │ │ └── volume.go │ │ ├── logging/ │ │ │ ├── errors.go │ │ │ └── logging.go │ │ ├── maps/ │ │ │ ├── interface.go │ │ │ └── string.go │ │ ├── math/ │ │ │ ├── float32.go │ │ │ ├── float64.go │ │ │ ├── int.go │ │ │ ├── int32.go │ │ │ └── int64.go │ │ ├── msgpack/ │ │ │ ├── errors.go │ │ │ └── msgpack.go │ │ ├── parallel/ │ │ │ ├── parallel.go │ │ │ └── parallel_test.go │ │ ├── pointer/ │ │ │ ├── equal.go │ │ │ ├── pointer.go │ │ │ └── pointer_test.go │ │ ├── print/ │ │ │ └── print.go │ │ ├── prompt/ │ │ │ ├── errors.go │ │ │ └── prompt.go │ │ ├── random/ │ │ │ └── random.go │ │ ├── regex/ │ │ │ ├── regex.go │ │ │ └── regex_test.go │ │ ├── requests/ │ │ │ ├── errors.go │ │ │ └── requests.go │ │ ├── sets/ │ │ │ └── strset/ │ │ │ ├── strset.go │ │ │ ├── strset_test.go │ │ │ └── threadsafe/ │ │ │ ├── strset.go │ │ │ └── strset_test.go │ │ ├── slices/ │ │ │ ├── bool.go │ │ │ ├── errors.go │ │ │ ├── float32.go │ │ │ ├── float64.go │ │ │ ├── float64_ptr.go │ │ │ ├── float64_ptr_test.go │ │ │ ├── int.go │ │ │ ├── int32.go │ │ │ ├── int64.go │ │ │ ├── sort.go │ │ │ ├── string.go │ │ │ └── string_test.go │ │ ├── strings/ │ │ │ ├── operations.go │ │ │ ├── operations_test.go │ │ │ ├── parse.go │ │ │ ├── stringify.go │ │ │ └── stringify_test.go │ │ ├── structs/ │ │ │ ├── deepcopy.go │ │ │ └── deepcopy_test.go │ │ ├── table/ │ │ │ ├── errors.go │ │ │ ├── key_value.go │ │ │ └── table.go │ │ ├── telemetry/ │ │ │ ├── error_cache.go │ │ │ ├── errors.go │ │ │ └── telemetry.go │ │ ├── time/ │ │ │ └── time.go │ │ └── urls/ │ │ ├── errors.go │ │ └── urls.go │ ├── operator/ │ │ ├── endpoints/ │ │ │ ├── delete.go │ │ │ ├── deploy.go │ │ │ ├── describe.go │ │ │ ├── errors.go │ │ │ ├── get.go │ │ │ ├── get_batch_job.go │ │ │ ├── get_task_job.go │ │ │ ├── info.go │ │ │ ├── logs.go │ │ │ ├── logs_job.go │ │ │ ├── middleware.go │ │ │ ├── params.go │ │ │ ├── refresh.go │ │ │ ├── respond.go │ │ │ ├── stop_batch_job.go │ │ │ ├── stop_task_job.go │ │ │ ├── submit_batch.go │ │ │ ├── submit_task.go │ │ │ └── verify_cortex.go │ │ ├── lib/ │ │ │ ├── exit/ │ │ │ │ └── exit.go │ │ │ └── routines/ │ │ │ └── routines.go │ │ ├── operator/ │ │ │ ├── cron.go │ │ │ ├── deployed_resource.go │ │ │ ├── errors.go │ │ │ ├── k8s.go │ │ │ ├── logging.go │ │ │ ├── memory_capacity.go │ │ │ ├── storage.go │ │ │ └── workload_logging.go │ │ ├── resources/ │ │ │ ├── asyncapi/ │ │ │ │ ├── api.go │ │ │ │ ├── errors.go │ │ │ │ ├── k8s_specs.go │ │ │ │ ├── queue.go │ │ │ │ ├── queue_metrics.go │ │ │ │ └── status.go │ │ │ ├── errors.go │ │ │ ├── job/ │ │ │ │ ├── batchapi/ │ │ │ │ │ ├── api.go │ │ │ │ │ ├── errors.go │ │ │ │ │ ├── job.go │ │ │ │ │ ├── job_status.go │ │ │ │ │ ├── k8s_specs.go │ │ │ │ │ ├── queue.go │ │ │ │ │ ├── s3_iterator.go │ │ │ │ │ └── validations.go │ │ │ │ ├── cache.go │ │ │ │ ├── consts.go │ │ │ │ ├── errors.go │ │ │ │ ├── state.go │ │ │ │ ├── taskapi/ │ │ │ │ │ ├── api.go │ │ │ │ │ ├── cron.go │ │ │ │ │ ├── job.go │ │ │ │ │ ├── job_status.go │ │ │ │ │ ├── k8s_specs.go │ │ │ │ │ ├── metrics.go │ │ │ │ │ └── validations.go │ │ │ │ └── worker_stats.go │ │ │ ├── realtimeapi/ │ │ │ │ ├── api.go │ │ │ │ ├── errors.go │ │ │ │ ├── k8s_specs.go │ │ │ │ └── status.go │ │ │ ├── resources.go │ │ │ ├── trafficsplitter/ │ │ │ │ ├── api.go │ │ │ │ └── k8s_specs.go │ │ │ └── validations.go │ │ └── schema/ │ │ ├── config_key.go │ │ ├── job_submission.go │ │ └── schema.go │ ├── probe/ │ │ ├── handler.go │ │ ├── handler_test.go │ │ ├── probe.go │ │ └── probe_test.go │ ├── proxy/ │ │ ├── breaker.go │ │ ├── breaker_test.go │ │ ├── handler.go │ │ ├── handler_test.go │ │ ├── proxy.go │ │ ├── proxy_test.go │ │ └── request_stats.go │ ├── types/ │ │ ├── async/ │ │ │ ├── s3_paths.go │ │ │ └── status.go │ │ ├── clusterconfig/ │ │ │ ├── availability_zones.go │ │ │ ├── aws_policy.go │ │ │ ├── cluster_config.go │ │ │ ├── config_key.go │ │ │ ├── errors.go │ │ │ ├── load_balancer_scheme.go │ │ │ ├── load_balancer_type.go │ │ │ ├── nat_gateway_type.go │ │ │ ├── network_validations.go │ │ │ ├── subnet_visibility.go │ │ │ └── volume_types.go │ │ ├── clusterstate/ │ │ │ ├── clusterstate.go │ │ │ ├── errors.go │ │ │ └── state.go │ │ ├── metrics/ │ │ │ ├── batch_metrics.go │ │ │ ├── metrics_test.go │ │ │ └── queue_metrics.go │ │ ├── spec/ │ │ │ ├── api.go │ │ │ ├── errors.go │ │ │ ├── id_gen.go │ │ │ ├── job.go │ │ │ ├── utils.go │ │ │ └── validations.go │ │ ├── status/ │ │ │ ├── job_code.go │ │ │ ├── job_status.go │ │ │ └── status.go │ │ └── userconfig/ │ │ ├── api.go │ │ ├── config_key.go │ │ ├── kind.go │ │ ├── log_level.go │ │ └── resource.go │ └── workloads/ │ ├── configmap.go │ ├── helpers.go │ ├── init.go │ └── k8s.go ├── python/ │ └── client/ │ ├── README.md │ ├── cortex/ │ │ ├── __init__.py │ │ ├── binary/ │ │ │ └── __init__.py │ │ ├── client.py │ │ ├── consts.py │ │ ├── exceptions.py │ │ ├── telemetry.py │ │ └── util.py │ └── setup.py └── test/ ├── README.md ├── apis/ │ ├── async/ │ │ ├── hello-world/ │ │ │ ├── app/ │ │ │ │ ├── main.py │ │ │ │ └── requirements.txt │ │ │ ├── build-cpu.sh │ │ │ ├── cortex_cpu.yaml │ │ │ └── cpu.Dockerfile │ │ └── text-generator/ │ │ ├── app/ │ │ │ ├── main.py │ │ │ ├── requirements-cpu.txt │ │ │ └── requirements-gpu.txt │ │ ├── build-cpu.sh │ │ ├── build-gpu.sh │ │ ├── cortex_cpu.yaml │ │ ├── cortex_gpu.yaml │ │ ├── cpu.Dockerfile │ │ ├── expectations.yaml │ │ ├── gpu.Dockerfile │ │ └── sample.json │ ├── batch/ │ │ ├── image-classifier-alexnet/ │ │ │ ├── app/ │ │ │ │ ├── main.py │ │ │ │ ├── requirements-cpu.txt │ │ │ │ └── requirements-gpu.txt │ │ │ ├── build-cpu.sh │ │ │ ├── build-gpu.sh │ │ │ ├── cortex_cpu.yaml │ │ │ ├── cortex_gpu.yaml │ │ │ ├── cpu.Dockerfile │ │ │ ├── gpu.Dockerfile │ │ │ ├── sample.json │ │ │ └── submit.py │ │ └── sum/ │ │ ├── app/ │ │ │ ├── main.py │ │ │ └── requirements.txt │ │ ├── build-cpu.sh │ │ ├── cortex_cpu.yaml │ │ ├── cpu.Dockerfile │ │ ├── sample.json │ │ ├── sample_generator.py │ │ └── submit.py │ ├── realtime/ │ │ ├── hello-world/ │ │ │ ├── app/ │ │ │ │ ├── main.py │ │ │ │ └── requirements.txt │ │ │ ├── build-cpu.sh │ │ │ ├── cortex_cpu.yaml │ │ │ ├── cortex_cpu_arm64.yaml │ │ │ ├── cortex_scale_to_zero.yaml │ │ │ ├── cpu.Dockerfile │ │ │ └── sample.json │ │ ├── image-classifier-resnet50/ │ │ │ ├── README.md │ │ │ ├── build-cpu.sh │ │ │ ├── build-gpu.sh │ │ │ ├── build-neuron-rtd.sh │ │ │ ├── build-neuron-tf-serving.sh │ │ │ ├── client.py │ │ │ ├── client_inf.py │ │ │ ├── cortex_cpu.yaml │ │ │ ├── cortex_gpu.yaml │ │ │ ├── cortex_inf.yaml │ │ │ ├── cortex_inf_rtd.yaml │ │ │ ├── cpu.Dockerfile │ │ │ ├── gpu.Dockerfile │ │ │ ├── neuron-rtd.Dockerfile │ │ │ ├── neuron-tf-serving.Dockerfile │ │ │ └── sample.json │ │ ├── multi-container/ │ │ │ ├── app/ │ │ │ │ ├── main.py │ │ │ │ └── requirements.txt │ │ │ ├── build-tfs-cpu.sh │ │ │ ├── build-web-cpu.sh │ │ │ ├── cortex_cpu.yaml │ │ │ ├── sample.json │ │ │ ├── tfs-cpu.Dockerfile │ │ │ └── web-cpu.Dockerfile │ │ ├── prime-generator/ │ │ │ ├── app/ │ │ │ │ ├── main.py │ │ │ │ └── requirements.txt │ │ │ ├── build-cpu.sh │ │ │ ├── cortex_cpu.yaml │ │ │ ├── cpu.Dockerfile │ │ │ └── sample.json │ │ ├── sleep/ │ │ │ ├── app/ │ │ │ │ ├── main.py │ │ │ │ └── requirements.txt │ │ │ ├── build-cpu.sh │ │ │ ├── cortex_cpu.yaml │ │ │ └── cpu.Dockerfile │ │ └── text-generator/ │ │ ├── app/ │ │ │ ├── main.py │ │ │ ├── requirements-cpu.txt │ │ │ └── requirements-gpu.txt │ │ ├── build-cpu.sh │ │ ├── build-gpu.sh │ │ ├── cortex_cpu.yaml │ │ ├── cortex_gpu.yaml │ │ ├── cpu.Dockerfile │ │ ├── gpu.Dockerfile │ │ └── sample.json │ ├── task/ │ │ └── iris-classifier-trainer/ │ │ ├── app/ │ │ │ ├── main.py │ │ │ └── requirements.txt │ │ ├── build-cpu.sh │ │ ├── cortex_cpu.yaml │ │ ├── cpu.Dockerfile │ │ └── submit.py │ └── trafficsplitter/ │ └── hello-world/ │ ├── .dockerignore │ ├── cortex_cpu.yaml │ └── sample.json ├── e2e/ │ ├── README.md │ ├── e2e/ │ │ ├── __init__.py │ │ ├── cluster.py │ │ ├── exceptions.py │ │ ├── expectations.py │ │ ├── generator.py │ │ ├── tests.py │ │ └── utils.py │ ├── pytest.ini │ ├── setup.py │ └── tests/ │ ├── __init__.py │ ├── aws/ │ │ ├── __init__.py │ │ ├── conftest.py │ │ ├── test_async.py │ │ ├── test_autoscaling.py │ │ ├── test_batch.py │ │ ├── test_load.py │ │ ├── test_long_running.py │ │ ├── test_realtime.py │ │ ├── test_scale_to_zero.py │ │ └── test_task.py │ └── conftest.py └── utils/ ├── README.md ├── build-all.sh ├── build.sh └── throughput_test.py ================================================ FILE CONTENTS ================================================ ================================================ FILE: .circleci/config.yml ================================================ version: 2.1 orbs: slack: circleci/slack@4.2.0 commands: install-go: steps: - run: name: Install Go command: | sudo rm -rf /usr/local/go wget https://dl.google.com/go/go1.17.3.linux-amd64.tar.gz sudo tar -C /usr/local -xzf go1.17.3.linux-amd64.tar.gz rm -rf go*.tar.gz echo 'export PATH=$PATH:/usr/local/go/bin' >> $BASH_ENV echo 'export PATH=$PATH:~/go/bin' >> $BASH_ENV mkdir ~/go echo 'export GOPATH=~/go' >> $BASH_ENV quay-login: description: Log Docker agent into Quay.io steps: - run: name: Login to Quay command: docker login -u=$QUAY_USERNAME -p=$QUAY_PASSWORD quay.io jobs: lint: docker: - image: cimg/python:3.7 resource_class: medium steps: - checkout - install-go - restore_cache: keys: - go-mod-v1-{{ checksum "go.sum" }} - go-mod-v1- - run: name: Install Linting Tools command: | go get -u -v golang.org/x/lint/golint go get -u -v github.com/kyoh86/looppointer/cmd/looppointer pip3 install aiohttp black==20.8b1 click==8.0.4 - run: name: Lint command: make lint no_output_timeout: 20m - save_cache: key: go-mod-v1-{{ checksum "go.sum" }} paths: - "~/go/pkg/mod" test: machine: image: ubuntu-2004:202201-02 # machine executor necessary to run go integration tests resource_class: medium steps: - checkout - install-go - restore_cache: keys: - go-mod-v1-{{ checksum "go.sum" }} - go-mod-v1- - run: name: Initialize Credentials command: | echo 'export AWS_ACCESS_KEY_ID=${TEST_AWS_ACCESS_KEY_ID}' >> $BASH_ENV echo 'export AWS_SECRET_ACCESS_KEY=${TEST_AWS_SECRET_ACCESS_KEY}' >> $BASH_ENV - run: name: Generate Cluster Config command: | mkdir -p dev/config cat \<< EOF > ./dev/config/cluster.yaml cluster_name: cortex-nightly region: us-east-1 node_groups: - name: cpu instance_type: m5.large min_instances: 1 max_instances: 1 EOF - run: name: Go Tests command: make test - save_cache: key: go-mod-v1-{{ checksum "go.sum" }} paths: - "~/go/pkg/mod" build-and-upload-cli: docker: - image: cimg/python:3.7 resource_class: medium steps: - checkout - install-go - run: pip install awscli - run: make ci-build-cli - run: make ci-build-and-upload-cli build-and-push-images-amd64: machine: image: ubuntu-2004:202101-01 resource_class: medium steps: - checkout - run: name: Build CI Images (amd64) command: make ci-build-images-amd64 no_output_timeout: 20m - quay-login - run: name: Push CI Images (amd64) command: make ci-push-images-amd64 no_output_timeout: 20m build-and-push-images-arm64: machine: image: ubuntu-2004:202201-02 resource_class: arm.medium steps: - checkout - run: name: Build CI Images (arm64) command: make ci-build-images-arm64 no_output_timeout: 20m - quay-login - run: name: Push CI Images (arm64) command: make ci-push-images-arm64 no_output_timeout: 20m amend-images: docker: - image: cimg/python:3.7 environment: DOCKER_CLI_EXPERIMENTAL: enabled resource_class: medium steps: - setup_remote_docker - checkout - quay-login - run: name: Amend CI Images command: make ci-amend-images no_output_timeout: 20m cluster-up: docker: - image: cimg/python:3.7 steps: - setup_remote_docker - checkout - run: name: Install Dependencies command: | pip install boto3 pyyaml awscli pip install https://s3-us-west-2.amazonaws.com/get-cortex/master/python/cortex-master.tar.gz - run: name: Initialize Credentials command: | echo 'export AWS_ACCESS_KEY_ID=${NIGHTLY_AWS_ACCESS_KEY_ID}' >> $BASH_ENV echo 'export AWS_SECRET_ACCESS_KEY=${NIGHTLY_AWS_SECRET_ACCESS_KEY}' >> $BASH_ENV - run: name: Generate Cluster Config # using a variety of node groups to test the multi-instance-type cluster functionality command: | cat \<< EOF > ./cluster.yaml cluster_name: cortex-nightly region: us-east-1 node_groups: - name: spot instance_type: t3.medium min_instances: 16 max_instances: 16 spot: true - name: cpu instance_type: c5.xlarge min_instances: 1 max_instances: 2 - name: gpu instance_type: g4dn.xlarge min_instances: 1 max_instances: 2 - name: inferentia instance_type: inf1.xlarge min_instances: 1 max_instances: 2 - name: arm instance_type: a1.large min_instances: 1 max_instances: 2 EOF - run: name: Create/Update AWS User policy command: python ./dev/create_user.py cortex-nightly $NIGHTLY_AWS_ACCOUNT_ID us-east-1 > $BASH_ENV - run: name: Wait for new keys to propagate in AWS command: sleep 10 - run: name: Verify configuration of credentials command: aws sts get-caller-identity | jq ".Arn" | grep "dev-cortex-nightly-us-east-1" - run: name: Create Cluster command: cortex cluster up cluster.yaml --configure-env cortex -y - slack/notify: event: fail channel: "#builds" template: basic_fail_1 e2e-tests: docker: - image: cimg/python:3.7 steps: - checkout - run: name: Install Dependencies command: | pip install boto3 pyyaml awscli pip install -e ./test/e2e pip install https://s3-us-west-2.amazonaws.com/get-cortex/master/python/cortex-master.tar.gz - run: name: Initialize Credentials command: | echo 'export AWS_ACCESS_KEY_ID=${NIGHTLY_AWS_ACCESS_KEY_ID}' >> $BASH_ENV echo 'export AWS_SECRET_ACCESS_KEY=${NIGHTLY_AWS_SECRET_ACCESS_KEY}' >> $BASH_ENV - run: name: Configure Cortex CLI command: cortex env configure cortex --operator-endpoint $(python dev/get_operator_url.py cortex-nightly us-east-1) - run: name: Run E2E Tests no_output_timeout: 30m command: | pytest -v test/e2e/tests --env cortex --x86-nodegroups spot,cpu,gpu,inferentia --arm-nodegroups arm --skip-autoscaling --skip-load --skip-long-running pytest -v test/e2e/tests --env cortex --x86-nodegroups spot,cpu,gpu,inferentia -k test_autoscaling pytest -v test/e2e/tests --env cortex --x86-nodegroups spot,cpu,gpu,inferentia -k test_load - slack/notify: event: fail channel: "#builds" template: basic_fail_1 cluster-down: docker: - image: cimg/python:3.7 steps: - setup_remote_docker - checkout - run: name: Install Dependencies command: | pip install boto3 pyyaml awscli pip install https://s3-us-west-2.amazonaws.com/get-cortex/master/python/cortex-master.tar.gz - run: name: Initialize Credentials command: | echo 'export AWS_ACCESS_KEY_ID=${NIGHTLY_AWS_ACCESS_KEY_ID}' >> $BASH_ENV echo 'export AWS_SECRET_ACCESS_KEY=${NIGHTLY_AWS_SECRET_ACCESS_KEY}' >> $BASH_ENV - run: name: Create/Update AWS User policy command: python ./dev/create_user.py cortex-nightly $NIGHTLY_AWS_ACCOUNT_ID us-east-1 > $BASH_ENV - run: name: Wait for new keys to propagate in AWS command: sleep 10 - run: name: Verify configuration of credentials command: aws sts get-caller-identity | jq ".Arn" | grep "dev-cortex-nightly-us-east-1" - run: name: Delete Cluster command: cortex cluster down --name cortex-nightly --region us-east-1 -y when: always - slack/notify: event: fail channel: "#builds" template: basic_fail_1 workflows: build: jobs: - lint - test - build-and-deploy-approval: type: approval filters: branches: only: - /^[0-9]+\.[0-9]+$/ - build-and-upload-cli: requires: - lint - test - build-and-deploy-approval filters: branches: only: - master - /^[0-9]+\.[0-9]+$/ - build-and-push-images-amd64: requires: - lint - test - build-and-deploy-approval filters: branches: only: - master - /^[0-9]+\.[0-9]+$/ - build-and-push-images-arm64: requires: - lint - test - build-and-deploy-approval filters: branches: only: - master - /^[0-9]+\.[0-9]+$/ - amend-images: requires: - build-and-push-images-amd64 - build-and-push-images-arm64 filters: branches: only: - master - /^[0-9]+\.[0-9]+$/ # nightly-cluster-up: # triggers: # - schedule: # cron: "0 0 * * *" # filters: # branches: # only: # - master # jobs: # - cluster-up # nightly-e2e-tests: # triggers: # - schedule: # cron: "0 1 * * *" # filters: # branches: # only: # - master # jobs: # - e2e-tests # nightly-cluster-down: # triggers: # - schedule: # cron: "0 2 * * *" # filters: # branches: # only: # - master # jobs: # - cluster-down ================================================ FILE: .dockerignore ================================================ /vendor/ /bin/ /testbin/ /dev/ /docs/ /test/ **/.* **/*.md **/*.zip **/*.pyc **/*.pyo **/*.pyd **/__pycache__/ **/hack/ **/PROJECT **/Makefile ================================================ FILE: .gitbook.yaml ================================================ root: ./docs/ structure: readme: ./start.md summary: summary.md ================================================ FILE: .github/ISSUE_TEMPLATE/bug-report.md ================================================ --- name: Bug report about: Report a bug title: '' labels: bug assignees: '' --- ### Version (use `cortex version` to determine your version) ### Description (describe the bug) ### Configuration (paste relevant `cortex.yaml` or `cluster.yaml` configuration) ### Steps to reproduce 1. ... 2. ... 3. ... ### Expected behavior (describe the behavior you expected) ### Actual behavior (describe the behavior you experienced) ### Screenshots (optional) ### Stack traces (error output from CloudWatch Insights or from a random pod `cortex logs `) ```text ``` ### Additional context (optional) ### Suggested solution (optional) ================================================ FILE: .github/ISSUE_TEMPLATE/feature-request.md ================================================ --- name: Feature request about: Request a feature title: '' labels: enhancement assignees: '' --- ### Description (describe the feature) ### Motivation (how this will improve the product / what problem will this solve / what is the use case) ### Additional context (optional) ================================================ FILE: .github/ISSUE_TEMPLATE/question.md ================================================ --- name: Question about: Ask a question title: '' labels: question assignees: '' --- ================================================ FILE: .github/pull_request_template.md ================================================ closes # --- checklist: - [ ] run `make test` and `make lint` - [ ] test manually (i.e. build/push all images, restart operator, and re-deploy APIs) - [ ] update examples - [ ] update docs and add any new files to `summary.md` (view in [gitbook](https://cortex-labs.gitbook.io/staging/-MOmCGMADSRNQahK3Kox/) after merging) - [ ] cherry-pick into release branches if applicable - [ ] alert the dev team if the dev environment changed ================================================ FILE: .gitignore ================================================ /vendor/ /bin/ /testbin/ /dev/config/ # PYTHON __pycache__/ *.py[cod] *$py.class .python-version .env .venv *.egg-info *.pytest_cache # OSX .DS_Store ._* # MISC *.zip .unison* .vscode/ # Binaries for programs and plugins *.exe *.exe~ *.dll *.so *.dylib # Test binary, build with `go test -c` *.test # Kubernetes Generated files - skip generated files, except for vendored files !vendor/**/zz_generated.* # editor and IDE paraphernalia .idea *.swp *.swo *~ ================================================ FILE: CONTRIBUTING.md ================================================ # Contributing ## Remote development We recommend that you run your development environment on an EC2 instance due to frequent docker registry pushing. We've had a good experience using [Mutagen](https://mutagen.io/documentation/introduction) to synchronize local / remote filesystems. ## Prerequisites ### System packages To install the necessary system packages on Ubuntu, you can run these commands: ```bash sudo apt-get update sudo apt install -y apt-transport-https ca-certificates software-properties-common gnupg-agent curl zip python3 python3-pip python3-dev build-essential jq tree sudo python3 -m pip install --upgrade pip setuptools boto3 ``` ### Go To install Go on linux, run: ```bash mkdir -p ~/bin && \ wget https://dl.google.com/go/go1.17.3.linux-amd64.tar.gz && \ sudo tar -xvf go1.17.3.linux-amd64.tar.gz && \ sudo mv go /usr/local && \ rm go1.17.3.linux-amd64.tar.gz && \ echo 'export PATH="/usr/local/go/bin:$HOME/go/bin:$PATH"' >> $HOME/.bashrc ``` And then log out and back in. ### Docker To install Docker on Ubuntu, run: ```bash curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - && \ sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" && \ sudo apt update && \ sudo apt install -y docker-ce docker-ce-cli containerd.io && \ sudo usermod -aG docker $USER ``` And then log out and back in. Then, bootstrap a buildx builder: ```bash docker buildx create --driver-opt image=moby/buildkit:master --name builder --platform linux/amd64,linux/arm64 --use docker buildx inspect --bootstrap builder ``` ### kubectl To install kubectl on linux, run: ```bash curl -LO https://storage.googleapis.com/kubernetes-release/release/`curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt`/bin/linux/amd64/kubectl && \ chmod +x ./kubectl && \ sudo mv ./kubectl /usr/local/bin/kubectl ``` ### eksctl To install eksctl run: ```bash curl --silent --location "https://github.com/weaveworks/eksctl/releases/latest/download/eksctl_$(uname -s)_amd64.tar.gz" | tar xz -C /tmp && \ sudo mv /tmp/eksctl /usr/local/bin ``` ### aws-cli (v1) Follow [these instructions](https://github.com/aws/aws-cli#installation) to install aws-cli (v1). E.g. to install it globally, run: ```bash sudo python3 -m pip install awscli aws configure ``` ## Cortex dev environment ### Clone the repo Clone the project: ```bash git clone https://github.com/cortexlabs/cortex.git cd cortex ``` Run the tests: ```bash make test ``` ### Dev tools Install development tools by running: ```bash make tools ``` After the dependencies are installed, there may be a diff in `go.mod` and `go.sum`, which you can revert. Run the linter: ```bash make lint ``` We use `gofmt` for formatting Go files, `black` for Python files (line length = 100), and the VS Code [yaml extension](https://marketplace.visualstudio.com/items?itemName=redhat.vscode-yaml) for YAML files. It is recommended to enable these in your code editor, but you can also run the Go and Python formatters from the terminal: ```bash make format git diff # there should be no diff ``` ### Cluster configuration Create a config directory in the repo's root directory: ```bash mkdir dev/config ``` Create `dev/config/env.sh` with the following information: ```bash # dev/config/env.sh export AWS_ACCOUNT_ID="***" # you can find your account ID in the AWS web console; here is an example: 764403040417 export AWS_REGION="***" # you can use any AWS region you'd like, e.g. "us-west-2" export AWS_ACCESS_KEY_ID="***" # alternatively, you can remove this to use the default credentials chain on your machine export AWS_SECRET_ACCESS_KEY="***" # alternatively, you can remove this to use the default credentials chain on your machine export DEFAULT_USER_ARN="arn:aws:iam:::" # (e.g. arn:aws-us-gov:iam::123456789:user/foo) ``` Create the ECR registries: ```bash make registry-create ``` Create `dev/config/cluster.yaml`. Paste the following config, and update `region` and all registry URLs (replace `` with your AWS account ID, and replace `` with your region): ```yaml # dev/config/cluster.yaml cluster_name: cortex region: # e.g. us-west-2 node_groups: - name: worker-ng instance_type: m5.large min_instances: 1 max_instances: 5 ``` ### Building Add this to your bash profile (e.g. `~/.bash_profile`, `~/.profile` or `~/.bashrc`), replacing the placeholders accordingly: ```bash # set the default image registry export CORTEX_DEV_DEFAULT_IMAGE_REGISTRY=".dkr.ecr..amazonaws.com/cortexlabs" # enable api server monitoring in grafana export CORTEX_DEV_ADD_CONTROL_PLANE_DASHBOARD="true" # redirect analytics and error reporting to our dev environment export CORTEX_TELEMETRY_SENTRY_DSN="https://c334df915c014ffa93f2076769e5b334@sentry.io/1848098" export CORTEX_TELEMETRY_SEGMENT_WRITE_KEY="0WvoJyCey9z1W2EW7rYTPJUMRYat46dl" # instruct the Python client to use your development CLI binary (update the path to point to your cortex repo) export CORTEX_CLI_PATH="/bin/cortex" # create a cortex alias which runs your development CLI alias cortex="$CORTEX_CLI_PATH" ``` Refresh your bash profile: ```bash . ~/.bash_profile # or: `. ~/.bashrc` ``` Build the Cortex CLI: ```bash make cli # the binary will be placed in /bin/cortex cortex version # should show "master" ``` Build and push all Cortex images: ```bash make images-all ``` ## Dev workflow Here is the typical full dev workflow which covers most cases: 1. `make cluster-up` (creates a cluster using `dev/config/cluster.yaml`) 2. `make devstart` (deletes the in-cluster operator, builds the CLI, and starts the operator locally; file changes will trigger the CLI and operator to re-build) 3. Make your changes 4. `make images-dev` (only necessary if changes were made outside of the operator and CLI) 5. Test your changes e.g. via `cortex deploy` (and repeat steps 3 and 4 as necessary) 6. `make cluster-down` (deletes your cluster) If you want to switch back to the in-cluster operator: 1. `` to stop your local operator 2. `make operator-start` to restart the operator in your cluster ### Dev workflow optimizations If you are only modifying the CLI, `make cli-watch` will build the CLI and re-build it when files are changed. When doing this, you can leave the operator running in the cluster instead of running it locally. If you are only modifying the operator, `make operator-local` will build and start the operator locally, and build/restart it when files are changed. See `Makefile` for additional dev commands. ================================================ FILE: LICENSE ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: Makefile ================================================ #!make # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. SHELL := /bin/bash export BASH_ENV=./dev/config/env.sh # declare all targets as phony to avoid collisions with local files or folders .PHONY: $(MAKECMDGOALS) ####### # Dev # ####### # Cortex # build cli, start local operator, and watch for changes devstart: @$(MAKE) operator-stop || true @./dev/operator_local.sh || true cli: @mkdir -p ./bin @go build -o ./bin/cortex ./cli # build cli and watch for changes cli-watch: @rerun -watch ./pkg ./cli -run sh -c "clear && echo 'building cli...' && go build -o ./bin/cortex ./cli && clear && echo '\033[1;32mCLI built\033[0m'" || true # start local operator and watch for changes operator-local: @$(MAKE) operator-stop || true @./dev/operator_local.sh --operator-only || true # start local operator and attach the delve debugger to it (in server mode) operator-local-dbg: @$(MAKE) operator-stop || true @./dev/operator_local.sh --debug || true # configure kubectl to point to the cluster specified in dev/config/cluster.yaml kubectl: @eval $$(python3 ./manager/cluster_config_env.py ./dev/config/cluster.yaml) && eval $$(python3 ./dev/create_user.py $$CORTEX_CLUSTER_NAME $$AWS_ACCOUNT_ID $$CORTEX_REGION) && eksctl utils write-kubeconfig --cluster="$$CORTEX_CLUSTER_NAME" --region="$$CORTEX_REGION" --verbose=0 | (grep -v "saved kubeconfig as" || true); eksctl create iamidentitymapping --region $$CORTEX_REGION --cluster $$CORTEX_CLUSTER_NAME --arn $$DEFAULT_USER_ARN --group system:masters --username $$DEFAULT_USER_ARN cluster-up: @$(MAKE) images-all @$(MAKE) cli @kill $(shell pgrep -f rerun) >/dev/null 2>&1 || true @eval $$(python3 ./manager/cluster_config_env.py ./dev/config/cluster.yaml) && eval $$(python3 ./dev/create_user.py $$CORTEX_CLUSTER_NAME $$AWS_ACCOUNT_ID $$CORTEX_REGION) && sleep 10 && ./bin/cortex cluster up ./dev/config/cluster.yaml --configure-env="$$CORTEX_CLUSTER_NAME"; eksctl create iamidentitymapping --region $$CORTEX_REGION --cluster $$CORTEX_CLUSTER_NAME --arn $$DEFAULT_USER_ARN --group system:masters --username $$DEFAULT_USER_ARN @$(MAKE) kubectl cluster-up-y: @$(MAKE) images-all @$(MAKE) cli @kill $(shell pgrep -f rerun) >/dev/null 2>&1 || true @eval $$(python3 ./manager/cluster_config_env.py ./dev/config/cluster.yaml) && eval $$(python3 ./dev/create_user.py $$CORTEX_CLUSTER_NAME $$AWS_ACCOUNT_ID $$CORTEX_REGION) && sleep 10 && ./bin/cortex cluster up ./dev/config/cluster.yaml --configure-env="$$CORTEX_CLUSTER_NAME" --yes; eksctl create iamidentitymapping --region $$CORTEX_REGION --cluster $$CORTEX_CLUSTER_NAME --arn $$DEFAULT_USER_ARN --group system:masters --username $$DEFAULT_USER_ARN @$(MAKE) kubectl cluster-configure: @$(MAKE) images-manager-skip-push @$(MAKE) cli @kill $(shell pgrep -f rerun) >/dev/null 2>&1 || true @eval $$(python3 ./manager/cluster_config_env.py ./dev/config/cluster.yaml) && eval $$(python3 ./dev/create_user.py $$CORTEX_CLUSTER_NAME $$AWS_ACCOUNT_ID $$CORTEX_REGION) && sleep 10 && ./bin/cortex cluster configure ./dev/config/cluster.yaml; eksctl create iamidentitymapping --region $$CORTEX_REGION --cluster $$CORTEX_CLUSTER_NAME --arn $$DEFAULT_USER_ARN --group system:masters --username $$DEFAULT_USER_ARN @$(MAKE) kubectl cluster-configure-y: @$(MAKE) images-manager-skip-push @$(MAKE) cli @kill $(shell pgrep -f rerun) >/dev/null 2>&1 || true @eval $$(python3 ./manager/cluster_config_env.py ./dev/config/cluster.yaml) && eval $$(python3 ./dev/create_user.py $$CORTEX_CLUSTER_NAME $$AWS_ACCOUNT_ID $$CORTEX_REGION) && sleep 10 && ./bin/cortex cluster configure ./dev/config/cluster.yaml --yes; eksctl create iamidentitymapping --region $$CORTEX_REGION --cluster $$CORTEX_CLUSTER_NAME --arn $$DEFAULT_USER_ARN --group system:masters --username $$DEFAULT_USER_ARN @$(MAKE) kubectl cluster-down: @$(MAKE) images-manager-skip-push @$(MAKE) cli @kill $(shell pgrep -f rerun) >/dev/null 2>&1 || true @eval $$(python3 ./manager/cluster_config_env.py ./dev/config/cluster.yaml) && eval $$(python3 ./dev/create_user.py $$CORTEX_CLUSTER_NAME $$AWS_ACCOUNT_ID $$CORTEX_REGION) && sleep 10 && ./bin/cortex cluster down --config=./dev/config/cluster.yaml cluster-down-y: @$(MAKE) images-manager-skip-push @$(MAKE) cli @kill $(shell pgrep -f rerun) >/dev/null 2>&1 || true @eval $$(python3 ./manager/cluster_config_env.py ./dev/config/cluster.yaml) && eval $$(python3 ./dev/create_user.py $$CORTEX_CLUSTER_NAME $$AWS_ACCOUNT_ID $$CORTEX_REGION) && sleep 10 && ./bin/cortex cluster down --config=./dev/config/cluster.yaml --yes cluster-info: @$(MAKE) images-manager-skip-push @$(MAKE) cli @eval $$(python3 ./manager/cluster_config_env.py ./dev/config/cluster.yaml) && eval $$(python3 ./dev/create_user.py $$CORTEX_CLUSTER_NAME $$AWS_ACCOUNT_ID $$CORTEX_REGION) && sleep 10 && ./bin/cortex cluster info --config=./dev/config/cluster.yaml --configure-env="$$CORTEX_CLUSTER_NAME" --yes update-credentials: @eval $$(python3 ./manager/cluster_config_env.py ./dev/config/cluster.yaml) && python3 ./dev/create_user.py $$CORTEX_CLUSTER_NAME $$AWS_ACCOUNT_ID $$CORTEX_REGION # stop the in-cluster operator operator-stop: @$(MAKE) kubectl @kubectl scale --namespace=default deployments/operator --replicas=0 # start the in-cluster operator operator-start: @$(MAKE) kubectl @kubectl scale --namespace=default deployments/operator --replicas=1 @operator_pod=$$(kubectl get pods -l workloadID=operator --namespace=default -o jsonpath='{.items[0].metadata.name}') && kubectl wait --for=condition=ready pod $$operator_pod --namespace=default # restart the in-cluster operator operator-restart: @$(MAKE) kubectl @kubectl delete pods -l workloadID=operator --namespace=default @operator_pod=$$(kubectl get pods -l workloadID=operator --namespace=default -o jsonpath='{.items[0].metadata.name}') && kubectl wait --for=condition=ready pod $$operator_pod --namespace=default # build and update the in-cluster operator operator-update: @$(MAKE) kubectl @kubectl scale --namespace=default deployments/operator --replicas=0 @./dev/registry.sh update-single operator @kubectl scale --namespace=default deployments/operator --replicas=1 @operator_pod=$$(kubectl get pods -l workloadID=operator --namespace=default -o jsonpath='{.items[0].metadata.name}') && kubectl wait --for=condition=ready pod $$operator_pod --namespace=default # restart all in-cluster async-gateways async-gateway-restart: @$(MAKE) kubectl @kubectl delete pods -l cortex.dev/async=gateway --namespace=default # build and update all in-cluster async-gateways async-gateway-update: @$(MAKE) kubectl @./dev/registry.sh update-single async-gateway @kubectl delete pods -l cortex.dev/async=gateway --namespace=default # docker images images-all: @./dev/registry.sh update all images-all-multi-arch: @./dev/registry.sh update all --include-arm64-arch images-all-skip-push: @./dev/registry.sh update all --skip-push images-dev: @./dev/registry.sh update dev images-dev-multi-arch: @./dev/registry.sh update dev --include-arm64-arch images-dev-skip-push: @./dev/registry.sh update dev --skip-push images-manager-skip-push: @./dev/registry.sh update-single manager --skip-push images-clean-cache: @./dev/registry.sh clean-cache registry-create: @./dev/registry.sh create registry-clean: @./dev/registry.sh clean # Misc tools: @go get -u -v golang.org/x/lint/golint @go get -u -v github.com/kyoh86/looppointer/cmd/looppointer @go get -u -v github.com/VojtechVitek/rerun/cmd/rerun @go get -u -v github.com/go-delve/delve/cmd/dlv @python3 -m pip install aiohttp boto3 pyyaml pydoc-markdown==3.* black==20.8b1 -U @python3 -m pip install -e test/e2e format: @./dev/format.sh ######### # Tests # ######### # build test api images # make sure you login with your quay credentials build-test-api-images: @./test/utils/build-all.sh quay.io/cortexlabs-test test: @./build/test.sh go # run e2e tests on an existing cluster # read test/e2e/README.md for instructions first test-e2e: @$(MAKE) images-all @$(MAKE) operator-restart @eval $$(python3 ./manager/cluster_config_env.py ./dev/config/cluster.yaml) && CORTEX_CLI_PATH="$$(pwd)/bin/cortex" ./build/test.sh e2e -e "$$CORTEX_CLUSTER_NAME" # run e2e tests with a new cluster # read test/e2e/README.md for instructions first test-e2e-new: @$(MAKE) images-all @eval $$(python3 ./manager/cluster_config_env.py ./dev/config/cluster.yaml) && CORTEX_CLI_PATH="$$(pwd)/bin/cortex" ./build/test.sh e2e "$$(pwd)/dev/config/cluster.yaml" --create-cluster lint: @./build/lint.sh # this is a subset of lint.sh, and is only meant to be run on master lint-docs: @./build/lint-docs.sh ############### # CI Commands # ############### ci-build-images-amd64: @./build/build-images.sh amd64 quay.io docker.io ci-build-images-arm64: @./build/build-images.sh arm64 quay.io docker.io ci-push-images-amd64: @./build/push-images.sh amd64 quay.io docker.io ci-push-images-arm64: @./build/push-images.sh arm64 quay.io docker.io ci-amend-images: @./build/amend-images.sh quay.io docker.io ci-build-cli: @./build/cli.sh ci-build-and-upload-cli: @./build/cli.sh upload ================================================ FILE: README.md ================================================ **[Docs](https://docs.cortexlabs.com)** • **[Slack](https://community.cortexlabs.com)**

Note: This project is no longer actively maintained by its original authors. # Production infrastructure for machine learning at scale Deploy, manage, and scale machine learning models in production.
## Serverless workloads **Realtime** - respond to requests in real-time and autoscale based on in-flight request volumes. **Async** - process requests asynchronously and autoscale based on request queue length. **Batch** - run distributed and fault-tolerant batch processing jobs on-demand.
## Automated cluster management **Autoscaling** - elastically scale clusters with CPU and GPU instances. **Spot instances** - run workloads on spot instances with automated on-demand backups. **Environments** - create multiple clusters with different configurations.
## CI/CD and observability integrations **Provisioning** - provision clusters with declarative configuration or a Terraform provider. **Metrics** - send metrics to any monitoring tool or use pre-built Grafana dashboards. **Logs** - stream logs to any log management tool or use the pre-built CloudWatch integration.
## Built for AWS **EKS** - Cortex runs on top of EKS to scale workloads reliably and cost-effectively. **VPC** - deploy clusters into a VPC on your AWS account to keep your data private. **IAM** - integrate with IAM for authentication and authorization workflows. ================================================ FILE: build/amend-image.sh ================================================ #!/bin/bash # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -euo pipefail CORTEX_VERSION=master host_primary=$1 host_backup=$2 image=$3 hosts=( "$host_primary" "$host_backup" ) echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin for host in "${hosts[@]}"; do docker manifest create $host/cortexlabs/${image}:${CORTEX_VERSION} \ -a $host/cortexlabs/${image}:manifest-${CORTEX_VERSION}-amd64 \ -a $host/cortexlabs/${image}:manifest-${CORTEX_VERSION}-arm64 docker manifest push $host/cortexlabs/${image}:${CORTEX_VERSION} done ================================================ FILE: build/amend-images.sh ================================================ #!/bin/bash # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. >/dev/null && pwd)" source $ROOT/build/images.sh source $ROOT/dev/util.sh host_primary=$1 host_backup=$2 for image in "${multi_arch_images[@]}"; do $ROOT/build/amend-image.sh $host_primary $host_backup $image done ================================================ FILE: build/build-image.sh ================================================ #!/bin/bash # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. >/dev/null && pwd)" CORTEX_VERSION=master host_primary=$1 host_backup=$2 image=$3 is_multi_arch=$4 arch=$5 if [ "$is_multi_arch" = "true" ]; then tag="manifest-${CORTEX_VERSION}-$arch" else tag="${CORTEX_VERSION}" fi docker build $ROOT \ --build-arg TARGETOS=linux \ --build-arg TARGETARCH=$arch \ -f $ROOT/images/$image/Dockerfile \ -t $host_primary/cortexlabs/${image}:${tag} \ -t $host_backup/cortexlabs/${image}:${tag} ================================================ FILE: build/build-images.sh ================================================ #!/bin/bash # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. >/dev/null && pwd)" source $ROOT/build/images.sh source $ROOT/dev/util.sh arch=$1 host_primary=$2 host_backup=$3 for image in "${all_images[@]}"; do is_multi_arch="false" if [[ " ${multi_arch_images[*]} " =~ " $image " ]]; then is_multi_arch="true" $ROOT/build/build-image.sh $host_primary $host_backup $image $is_multi_arch $arch elif [ "$arch" = "amd64" ]; then $ROOT/build/build-image.sh $host_primary $host_backup $image $is_multi_arch $arch fi done ================================================ FILE: build/cli.sh ================================================ #!/bin/bash # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. >/dev/null && pwd)" CORTEX_VERSION=master arg1=${1:-""} upload="false" if [ "$arg1" == "upload" ]; then upload="true" fi function build_and_upload() { set -euo pipefail os=$1 echo -e "\nBuilding Cortex CLI for $os" GOOS=$os GOARCH=amd64 CGO_ENABLED=0 go build -o cortex "$ROOT/cli" if [ "$upload" == "true" ]; then echo "Uploading Cortex CLI to s3://$CLI_BUCKET_NAME/$CORTEX_VERSION/cli/$os/cortex" aws s3 cp cortex s3://$CLI_BUCKET_NAME/$CORTEX_VERSION/cli/$os/cortex --only-show-errors zip cortex.zip cortex echo "Uploading zipped Cortex CLI to s3://$CLI_BUCKET_NAME/$CORTEX_VERSION/cli/$os/cortex.zip" aws s3 cp cortex.zip s3://$CLI_BUCKET_NAME/$CORTEX_VERSION/cli/$os/cortex.zip --only-show-errors rm cortex.zip fi echo "Done ✓" rm cortex } function build_python { pushd $ROOT/python/client python setup.py sdist if [ "$upload" == "true" ]; then echo "Uploading Cortex CLI to s3://$CLI_BUCKET_NAME/$CORTEX_VERSION/python/cortex-$CORTEX_VERSION.tar.gz" aws s3 cp dist/cortex-$CORTEX_VERSION.tar.gz s3://$CLI_BUCKET_NAME/$CORTEX_VERSION/python/cortex-$CORTEX_VERSION.tar.gz fi rm -rf dist/ rm -rf cortex.egg-info/ popd } build_and_upload darwin build_and_upload linux build_python ================================================ FILE: build/generate_ami_mapping.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package main import ( "encoding/json" "fmt" "io/ioutil" "log" "os" "sort" "time" "github.com/pkg/errors" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/ec2" "github.com/aws/aws-sdk-go/service/ec2/ec2iface" ) // run with `go run build/generate_ami_mapping.go manager/manifests/ami.json` // copied from https://github.com/weaveworks/eksctl/blob/c211e68d3c8cf3c7f800768bfa0251dda17e011c/pkg/apis/eksctl.io/v1alpha5/types.go // most of this code can be removed once eksctl can be imported: https://github.com/weaveworks/eksctl/issues/813 const ( eksResourceAccountStandard = "602401143452" // eksResourceAccountAPEast1 defines the AWS EKS account ID that provides node resources in ap-east-1 region eksResourceAccountAPEast1 = "800184023465" // eksResourceAccountMESouth1 defines the AWS EKS account ID that provides node resources in me-south-1 region eksResourceAccountMESouth1 = "558608220178" // eksResourceAccountCNNorthWest1 defines the AWS EKS account ID that provides node resources in cn-northwest-1 region eksResourceAccountCNNorthWest1 = "961992271922" // eksResourceAccountCNNorth1 defines the AWS EKS account ID that provides node resources in cn-north-1 eksResourceAccountCNNorth1 = "918309763551" // eksResourceAccountAFSouth1 defines the AWS EKS account ID that provides node resources in af-south-1 eksResourceAccountAFSouth1 = "877085696533" // eksResourceAccountEUSouth1 defines the AWS EKS account ID that provides node resources in eu-south-1 eksResourceAccountEUSouth1 = "590381155156" // eksResourceAccountUSGovWest1 defines the AWS EKS account ID that provides node resources in us-gov-west-1 eksResourceAccountUSGovWest1 = "013241004608" // eksResourceAccountUSGovEast1 defines the AWS EKS account ID that provides node resources in us-gov-east-1 eksResourceAccountUSGovEast1 = "151742754352" ) // Regions const ( // RegionUSWest1 represents the US West Region North California RegionUSWest1 = "us-west-1" // RegionUSWest2 represents the US West Region Oregon RegionUSWest2 = "us-west-2" // RegionUSEast1 represents the US East Region North Virginia RegionUSEast1 = "us-east-1" // RegionUSEast2 represents the US East Region Ohio RegionUSEast2 = "us-east-2" // RegionCACentral1 represents the Canada Central Region RegionCACentral1 = "ca-central-1" // RegionEUWest1 represents the EU West Region Ireland RegionEUWest1 = "eu-west-1" // RegionEUWest2 represents the EU West Region London RegionEUWest2 = "eu-west-2" // RegionEUWest3 represents the EU West Region Paris RegionEUWest3 = "eu-west-3" // RegionEUNorth1 represents the EU North Region Stockholm RegionEUNorth1 = "eu-north-1" // RegionEUCentral1 represents the EU Central Region Frankfurt RegionEUCentral1 = "eu-central-1" // RegionEUSouth1 represents te Eu South Region Milan RegionEUSouth1 = "eu-south-1" // RegionAPNorthEast1 represents the Asia-Pacific North East Region Tokyo RegionAPNorthEast1 = "ap-northeast-1" // RegionAPNorthEast2 represents the Asia-Pacific North East Region Seoul RegionAPNorthEast2 = "ap-northeast-2" // RegionAPNorthEast3 represents the Asia-Pacific North East region Osaka RegionAPNorthEast3 = "ap-northeast-3" // RegionAPSouthEast1 represents the Asia-Pacific South East Region Singapore RegionAPSouthEast1 = "ap-southeast-1" // RegionAPSouthEast2 represents the Asia-Pacific South East Region Sydney RegionAPSouthEast2 = "ap-southeast-2" // RegionAPSouth1 represents the Asia-Pacific South Region Mumbai RegionAPSouth1 = "ap-south-1" // RegionAPEast1 represents the Asia Pacific Region Hong Kong RegionAPEast1 = "ap-east-1" // RegionMESouth1 represents the Middle East Region Bahrain RegionMESouth1 = "me-south-1" // RegionSAEast1 represents the South America Region Sao Paulo RegionSAEast1 = "sa-east-1" // RegionAFSouth1 represents the Africa Region Cape Town RegionAFSouth1 = "af-south-1" // RegionCNNorthwest1 represents the China region Ningxia RegionCNNorthwest1 = "cn-northwest-1" // RegionCNNorth1 represents the China region Beijing RegionCNNorth1 = "cn-north-1" // RegionUSGovWest1 represents the region GovCloud (US-West) RegionUSGovWest1 = "us-gov-west-1" // RegionUSGovEast1 represents the region GovCloud (US-East) RegionUSGovEast1 = "us-gov-east-1" // DefaultRegion defines the default region, where to deploy the EKS cluster DefaultRegion = RegionUSWest2 ) // SupportedRegions are the regions where EKS is available func SupportedRegions() []string { return []string{ RegionUSWest1, RegionUSWest2, RegionUSEast1, RegionUSEast2, RegionCACentral1, RegionEUWest1, RegionEUWest2, RegionEUWest3, RegionEUNorth1, RegionEUCentral1, RegionEUSouth1, RegionAPNorthEast1, RegionAPNorthEast2, RegionAPNorthEast3, RegionAPSouthEast1, RegionAPSouthEast2, RegionAPSouth1, RegionAPEast1, RegionMESouth1, RegionSAEast1, RegionAFSouth1, RegionUSGovWest1, RegionUSGovEast1, // RegionCNNorthwest1, // RegionCNNorth1, } } func EKSResourceAccountID(region string) string { switch region { case RegionAPEast1: return eksResourceAccountAPEast1 case RegionMESouth1: return eksResourceAccountMESouth1 case RegionCNNorthwest1: return eksResourceAccountCNNorthWest1 case RegionCNNorth1: return eksResourceAccountCNNorth1 case RegionUSGovWest1: return eksResourceAccountUSGovWest1 case RegionUSGovEast1: return eksResourceAccountUSGovEast1 case RegionAFSouth1: return eksResourceAccountAFSouth1 case RegionEUSouth1: return eksResourceAccountEUSouth1 default: return eksResourceAccountStandard } } func main() { if len(os.Args) > 3 { fmt.Println("usage: go run generate_ami_mapping.go public|govcloud") os.Exit(1) } destFile := os.Args[1] cloudType := os.Args[2] if cloudType != "public" && cloudType != "govcloud" { log.Fatalf("%s is not a valid value; specify public or govcloud", cloudType) } k8sVersionMap := map[string]map[string]map[string]string{} if _, err := os.Stat(destFile); !os.IsNotExist(err) { jsonBytes, err := ioutil.ReadFile(destFile) if err != nil { log.Fatal(err.Error()) } json.Unmarshal(jsonBytes, &k8sVersionMap) } k8sVersion := "1.22" if k8sVersionMap[k8sVersion] == nil { k8sVersionMap[k8sVersion] = map[string]map[string]string{} } for _, region := range SupportedRegions() { if (cloudType == "govcloud") != (region == RegionUSGovEast1 || region == RegionUSGovWest1) { // cloudType == "govcloud" xor (region is us govclouds) continue } fmt.Print(region) sess := session.New(&aws.Config{Region: aws.String(region)}) svc := ec2.New(sess) cpuAmd64AMI, err := FindImage(svc, EKSResourceAccountID(region), fmt.Sprintf("amazon-eks-node-%s-v*", k8sVersion)) if err != nil { log.Fatal(err.Error()) } cpuArm64AMI, err := FindImage(svc, EKSResourceAccountID(region), fmt.Sprintf("amazon-eks-arm64-node-%s-v*", k8sVersion)) if err != nil { log.Fatal(err.Error()) } acceleratedAmd64AMI, err := FindImage(svc, EKSResourceAccountID(region), fmt.Sprintf("amazon-eks-gpu-node-%s-v*", k8sVersion)) if err != nil { log.Fatal(err.Error()) } if k8sVersionMap[k8sVersion][region] == nil { k8sVersionMap[k8sVersion][region] = map[string]string{} } k8sVersionMap[k8sVersion][region] = map[string]string{ "cpu_amd64": cpuAmd64AMI, "cpu_arm64": cpuArm64AMI, "accelerated_amd64": acceleratedAmd64AMI, } fmt.Println(" ✓") } marshalledBytes, err := json.MarshalIndent(k8sVersionMap, "", "\t") if err != nil { log.Fatal(err.Error()) } marshalledBytes = append(marshalledBytes, []byte("\n")...) err = ioutil.WriteFile(destFile, marshalledBytes, 0664) if err != nil { log.Fatal(err.Error()) } } func FindImage(ec2api ec2iface.EC2API, ownerAccount, namePattern string) (string, error) { input := &ec2.DescribeImagesInput{ Owners: []*string{&ownerAccount}, Filters: []*ec2.Filter{ { Name: aws.String("name"), Values: []*string{&namePattern}, }, { Name: aws.String("virtualization-type"), Values: []*string{aws.String("hvm")}, }, { Name: aws.String("root-device-type"), Values: []*string{aws.String("ebs")}, }, { Name: aws.String("is-public"), Values: []*string{aws.String("true")}, }, { Name: aws.String("state"), Values: []*string{aws.String("available")}, }, }, } output, err := ec2api.DescribeImages(input) if err != nil { return "", errors.Wrapf(err, "error querying AWS for images") } if len(output.Images) < 1 { return "", nil } if len(output.Images) == 1 { return *output.Images[0].ImageId, nil } // Sort images so newest is first sort.Slice(output.Images, func(i, j int) bool { //nolint:gosec creationLeft, _ := time.Parse(time.RFC3339, *output.Images[i].CreationDate) //nolint:gosec creationRight, _ := time.Parse(time.RFC3339, *output.Images[j].CreationDate) return creationLeft.After(creationRight) }) return *output.Images[0].ImageId, nil } ================================================ FILE: build/images.sh ================================================ #!/bin/bash # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # images to build/push for development and CI commands # each image should appear exactly once on this page set -euo pipefail dev_images=( "manager" "proxy" "async-gateway" "enqueuer" "dequeuer" "autoscaler" "activator" ) non_dev_images=( "cluster-autoscaler" "operator" "controller-manager" "istio-proxy" "istio-pilot" "fluent-bit" "prometheus" "prometheus-config-reloader" "prometheus-operator" "prometheus-statsd-exporter" "prometheus-dcgm-exporter" "prometheus-kube-state-metrics" "prometheus-node-exporter" "kube-rbac-proxy" "grafana" "event-exporter" "metrics-server" "nvidia-device-plugin" "neuron-device-plugin" "neuron-scheduler" "kubexit" ) # for linux/amd64 and linux/arm64 multi_arch_images=( "proxy" "async-gateway" "enqueuer" "dequeuer" "fluent-bit" "prometheus-node-exporter" "kube-rbac-proxy" "kubexit" ) all_images=( "${dev_images[@]}" "${non_dev_images[@]}" ) ================================================ FILE: build/lint-docs.sh ================================================ #!/bin/bash # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # This is a subset of lint.sh, and is only meant to be run on master set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. >/dev/null && pwd)" # Check docs links output=$(python3 $ROOT/dev/find_missing_docs_links.py) if [[ $output ]]; then echo "docs file(s) have broken links:" echo "$output" exit 1 fi # Check for trailing whitespace output=$(cd "$ROOT/docs" && find . -type f \ -exec egrep -l " +$" {} \;) if [[ $output ]]; then echo "File(s) have lines with trailing whitespace:" echo "$output" exit 1 fi # Check for missing new line at end of file output=$(cd "$ROOT/docs" && find . -type f \ -print0 | \ xargs -0 -L1 bash -c 'test "$(tail -c 1 "$0")" && echo "No new line at end of $0"' || true) if [[ $output ]]; then echo "$output" exit 1 fi # Check for multiple new lines at end of file output=$(cd "$ROOT/docs" && find . -type f \ -print0 | \ xargs -0 -L1 bash -c 'test "$(tail -c 2 "$0")" || echo "Multiple new lines at end of $0"' || true) if [[ $output ]]; then echo "$output" exit 1 fi # Check for new line(s) at beginning of file output=$(cd "$ROOT/docs" && find . -type f \ -print0 | \ xargs -0 -L1 bash -c 'test "$(head -c 1 "$0")" || echo "New line at beginning of $0"' || true) if [[ $output ]]; then echo "$output" exit 1 fi ================================================ FILE: build/lint.sh ================================================ #!/bin/bash # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. >/dev/null && pwd)" git_branch="${CIRCLE_BRANCH:-""}" if [ "$git_branch" = "" ]; then git_branch=$(cd "$ROOT" && git rev-parse --abbrev-ref HEAD) fi is_release_branch="false" if echo "$git_branch" | grep -Eq ^[0-9]+.[0-9]+$; then is_release_branch="true" fi if ! command -v golint >/dev/null 2>&1; then echo "golint must be installed" exit 1 fi if ! command -v looppointer >/dev/null 2>&1; then echo "looppointer must be installed" exit 1 fi if ! command -v gofmt >/dev/null 2>&1; then echo "gofmt must be installed" exit 1 fi if ! command -v black >/dev/null 2>&1; then echo "black must be installed" exit 1 fi go mod tidy go vet "$ROOT/..." output=$(golint "$ROOT/..." | grep -v "comment" || true) if [[ $output ]]; then echo "$output" exit 1 fi output=$(looppointer "$ROOT/...") if [[ $output ]]; then echo "$output" exit 1 fi output=$(gofmt -s -l "$ROOT") if [[ $output ]]; then echo "go files not properly formatted:" echo "$output" exit 1 fi output=$(black --quiet --diff --line-length=100 "$ROOT") if [[ $output ]]; then echo "python files not properly formatted:" echo "$output" black --version exit 1 fi # Check for missing license output=$(cd "$ROOT" && find . -type f \ ! -path "./vendor/*" \ ! -path "**/.vscode/*" \ ! -path "**/.idea/*" \ ! -path "**/.history/*" \ ! -path "**/testbin/*" \ ! -path "**/__pycache__/*" \ ! -path "**/.pytest_cache/*" \ ! -path "**/*.egg-info/*" \ ! -path "./test/*" \ ! -path "./dev/config/*" \ ! -path "**/bin/*" \ ! -path "./.circleci/*" \ ! -path "./.git/*" \ ! -path "./pkg/crds/config/*" \ ! -path "**/tmp/*" \ ! -name LICENSE \ ! -name "*requirements.txt" \ ! -name "go.*" \ ! -name "*.md" \ ! -name "*.json" \ ! -name ".*" \ ! -name "*.bin" \ ! -name "Dockerfile" \ ! -name "PROJECT" \ -exec grep -L "Copyright 2022 Cortex Labs, Inc" {} \;) if [[ $output ]]; then echo "File(s) are missing Cortex license:" echo "$output" exit 1 fi if [ "$is_release_branch" = "true" ]; then # Check for occurrences of "master" which should be changed to the version number output=$(cd "$ROOT" && find . -type f \ ! -path "./build/lint.sh" \ ! -path "./vendor/*" \ ! -path "**/.vscode/*" \ ! -path "**/.idea/*" \ ! -path "**/.history/*" \ ! -path "**/testbin/*" \ ! -path "**/__pycache__/*" \ ! -path "**/.pytest_cache/*" \ ! -path "**/*.egg-info/*" \ ! -path "./dev/config/*" \ ! -path "**/bin/*" \ ! -path "./.git/*" \ ! -name ".*" \ ! -name "*.bin" \ -exec grep -R -A 5 -e "CORTEX_VERSION" {} \;) output=$(echo "$output" | grep -e "master" || true) if [[ $output ]]; then echo 'occurrences of "master" which should be changed to the version number:' echo "$output" exit 1 fi fi # Check for trailing whitespace output=$(cd "$ROOT" && find . -type f \ ! -path "./vendor/*" \ ! -path "**/.idea/*" \ ! -path "**/.history/*" \ ! -path "**/.vscode/*" \ ! -path "**/testbin/*" \ ! -path "**/__pycache__/*" \ ! -path "**/.pytest_cache/*" \ ! -path "**/*.egg-info/*" \ ! -path "./dev/config/*" \ ! -path "**/bin/*" \ ! -path "./.git/*" \ ! -path "./pkg/crds/config/*" \ ! -name ".*" \ ! -name "*.bin" \ ! -name "*.wav" \ -exec egrep -l " +$" {} \;) if [[ $output ]]; then echo "File(s) have lines with trailing whitespace:" echo "$output" exit 1 fi # Check for missing new line at end of file output=$(cd "$ROOT" && find . -type f \ ! -path "./vendor/*" \ ! -path "**/.idea/*" \ ! -path "**/.history/*" \ ! -path "**/.vscode/*" \ ! -path "**/testbin/*" \ ! -path "**/__pycache__/*" \ ! -path "**/.pytest_cache/*" \ ! -path "**/*.egg-info/*" \ ! -path "./dev/config/*" \ ! -path "./pkg/crds/config/*" \ ! -path "**/bin/*" \ ! -path "./.git/*" \ ! -name ".*" \ ! -name "*.bin" \ ! -name "*.wav" \ ! -name "*.json" \ -print0 | \ xargs -0 -L1 bash -c 'test "$(tail -c 1 "$0")" && echo "No new line at end of $0"' || true) if [[ $output ]]; then echo "$output" exit 1 fi # Check for multiple new lines at end of file output=$(cd "$ROOT" && find . -type f \ ! -path "./vendor/*" \ ! -path "**/.vscode/*" \ ! -path "**/.idea/*" \ ! -path "**/.history/*" \ ! -path "**/testbin/*" \ ! -path "**/__pycache__/*" \ ! -path "**/.pytest_cache/*" \ ! -path "**/*.egg-info/*" \ ! -path "./dev/config/*" \ ! -path "**/bin/*" \ ! -path "./.git/*" \ ! -name ".*" \ ! -name "*.bin" \ ! -name "*.wav" \ -print0 | \ xargs -0 -L1 bash -c 'test "$(tail -c 2 "$0")" || [ ! -s "$0" ] || echo "Multiple new lines at end of $0"' || true) if [[ $output ]]; then echo "$output" exit 1 fi # Check for new line(s) at beginning of file output=$(cd "$ROOT" && find . -type f \ ! -path "./vendor/*" \ ! -path "**/.idea/*" \ ! -path "**/.history/*" \ ! -path "**/.vscode/*" \ ! -path "**/testbin/*" \ ! -path "**/__pycache__/*" \ ! -path "**/.pytest_cache/*" \ ! -path "**/*.egg-info/*" \ ! -path "./dev/config/*" \ ! -path "./pkg/crds/config/*" \ ! -path "./bin/*" \ ! -path "./.git/*" \ ! -name ".*" \ ! -name "*.bin" \ ! -name "*.wav" \ ! -name "boilerplate.go.txt" \ -print0 | \ xargs -0 -L1 bash -c 'test "$(head -c 1 "$0")" || [ ! -s "$0" ] || echo "New line at beginning of $0"' || true) if [[ $output ]]; then echo "$output" exit 1 fi # Check that minimum_aws_policy.json is in-sync with docs output=$(python3 -c " import sys policy=open('./dev/minimum_aws_policy.json').read() doc=open('./docs/clusters/management/auth.md').read() print(policy in doc)") if [[ "$output" != "True" ]]; then echo "./dev/minimum_aws_policy.json and the policy in ./docs/clusters/management/auth.md are out of sync" exit 1 fi # Check docs links output=$(python3 $ROOT/dev/find_missing_docs_links.py) if [[ $output ]]; then echo "docs file(s) have broken links:" echo "$output" exit 1 fi ================================================ FILE: build/push-image.sh ================================================ #!/bin/bash # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. >/dev/null && pwd)" CORTEX_VERSION=master host_primary=$1 host_backup=$2 image=$3 is_multi_arch=$4 arch=$5 echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin if [ "$is_multi_arch" = "true" ]; then tag="manifest-${CORTEX_VERSION}-$arch" else tag="${CORTEX_VERSION}" fi docker push $host_primary/cortexlabs/${image}:${tag} docker push $host_backup/cortexlabs/${image}:${tag} ================================================ FILE: build/push-images.sh ================================================ #!/bin/bash # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. >/dev/null && pwd)" source $ROOT/build/images.sh source $ROOT/dev/util.sh arch=$1 host_primary=$2 host_backup=$3 for image in "${all_images[@]}"; do is_multi_arch="false" if [[ " ${multi_arch_images[*]} " =~ " $image " ]]; then is_multi_arch="true" $ROOT/build/push-image.sh $host_primary $host_backup $image $is_multi_arch $arch elif [ "$arch" = "amd64" ]; then $ROOT/build/push-image.sh $host_primary $host_backup $image $is_multi_arch $arch fi done ================================================ FILE: build/test.sh ================================================ #!/bin/bash # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. >/dev/null && pwd)" ENVTEST_ASSETS_DIR=${ROOT}/testbin cluster_env="undefined" create_cluster="no" positional_args=() while [[ $# -gt 0 ]]; do key="$1" case $key in -e|--cluster-env) cluster_env="$2" shift ;; -c|--create-cluster) create_cluster="yes" shift ;; *) positional_args+=("$1") shift ;; esac done set -- "${positional_args[@]}" positional_args=() for i in "$@"; do case $i in -e=*|--cluster-env=*) cluster_env="${i#*=}" shift ;; -c|--create-cluster) create_cluster="yes" shift ;; *) positional_args+=("$1") shift ;; esac done set -- "${positional_args[@]}" for arg in "$@"; do if [[ "$arg" == -* ]]; then echo "unknown flag: $arg" exit 1 fi done cmd=${1:-""} sub_cmd=${2:-""} function run_go_tests() { ( cd $ROOT mkdir -p ${ENVTEST_ASSETS_DIR} test -f ${ENVTEST_ASSETS_DIR}/setup-envtest.sh || curl -sSLo ${ENVTEST_ASSETS_DIR}/setup-envtest.sh https://raw.githubusercontent.com/kubernetes-sigs/controller-runtime/v0.7.2/hack/setup-envtest.sh source ${ENVTEST_ASSETS_DIR}/setup-envtest.sh; fetch_envtest_tools ${ENVTEST_ASSETS_DIR}; setup_envtest_env ${ENVTEST_ASSETS_DIR} go test -race ./... && echo "go tests passed" ) } function run_e2e_tests() { if [ "$create_cluster" = "yes" ]; then pytest $ROOT/test/e2e/tests --config "$sub_cmd" else pytest $ROOT/test/e2e/tests --env "$cluster_env" fi } if [ "$cmd" = "go" ]; then run_go_tests elif [ "$cmd" = "e2e" ]; then run_e2e_tests fi ================================================ FILE: cli/cluster/delete.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package cluster import ( "fmt" "path" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/json" "github.com/cortexlabs/cortex/pkg/lib/pointer" "github.com/cortexlabs/cortex/pkg/lib/prompt" s "github.com/cortexlabs/cortex/pkg/lib/strings" "github.com/cortexlabs/cortex/pkg/operator/schema" "github.com/cortexlabs/cortex/pkg/types/userconfig" ) func Delete(operatorConfig OperatorConfig, apiName string, keepCache bool, force bool) (schema.DeleteResponse, error) { if !force { readyReplicas := getReadyRealtimeAPIReplicasOrNil(operatorConfig, apiName) if readyReplicas != nil && *readyReplicas > 2 { prompt.YesOrExit(fmt.Sprintf("are you sure you want to delete %s (which has %d live replicas)?", apiName, *readyReplicas), "", "") } } params := map[string]string{ "apiName": apiName, "keepCache": s.Bool(keepCache), } httpRes, err := HTTPDelete(operatorConfig, "/delete/"+apiName, params) if err != nil { return schema.DeleteResponse{}, err } var deleteRes schema.DeleteResponse err = json.Unmarshal(httpRes, &deleteRes) if err != nil { return schema.DeleteResponse{}, errors.Wrap(err, "/delete", string(httpRes)) } return deleteRes, nil } func getReadyRealtimeAPIReplicasOrNil(operatorConfig OperatorConfig, apiName string) *int32 { httpRes, err := HTTPGet(operatorConfig, "/get/"+apiName) if err != nil { return nil } var apiRes schema.APIResponse if err = json.Unmarshal(httpRes, &apiRes); err != nil { return nil } if apiRes.Status == nil { return nil } return pointer.Int32(apiRes.Status.Ready) } func StopJob(operatorConfig OperatorConfig, kind userconfig.Kind, apiName string, jobID string) (schema.DeleteResponse, error) { params := map[string]string{ "apiName": apiName, "jobID": jobID, } var endpointComponent string if kind == userconfig.BatchAPIKind { endpointComponent = "batch" } else { endpointComponent = "tasks" } httpRes, err := HTTPDelete(operatorConfig, path.Join("/"+endpointComponent, apiName), params) if err != nil { return schema.DeleteResponse{}, err } var deleteRes schema.DeleteResponse err = json.Unmarshal(httpRes, &deleteRes) if err != nil { return schema.DeleteResponse{}, errors.Wrap(err, string(httpRes)) } return deleteRes, nil } ================================================ FILE: cli/cluster/deploy.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package cluster import ( "path/filepath" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/json" s "github.com/cortexlabs/cortex/pkg/lib/strings" "github.com/cortexlabs/cortex/pkg/operator/schema" ) func Deploy(operatorConfig OperatorConfig, configPath string, deploymentBytesMap map[string][]byte, force bool) ([]schema.DeployResult, error) { params := map[string]string{ "force": s.Bool(force), "configFileName": filepath.Base(configPath), } uploadInput := &HTTPUploadInput{ Bytes: deploymentBytesMap, } response, err := HTTPUpload(operatorConfig, "/deploy", uploadInput, params) if err != nil { return nil, err } var deployResults []schema.DeployResult if err := json.Unmarshal(response, &deployResults); err != nil { return nil, errors.Wrap(err, "/deploy", string(response)) } return deployResults, nil } ================================================ FILE: cli/cluster/errors.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package cluster import ( "fmt" "net/url" "strings" "github.com/cortexlabs/cortex/pkg/consts" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/urls" ) const ( _errStrCantMakeRequest = "unable to make request" _errStrRead = "unable to read" ) func errStrFailedToConnect(u url.URL) string { return "failed to connect to " + urls.TrimQueryParamsURL(u) } const ( ErrFailedToConnectOperator = "cli.failed_to_connect_operator" ErrOperatorSocketRead = "cli.operator_socket_read" ErrResponseUnknown = "cli.response_unknown" ErrOperatorResponseUnknown = "cli.operator_response_unknown" ErrOperatorStreamResponseUnknown = "cli.operator_stream_response_unknown" ) func ErrorFailedToConnectOperator(originalError error, envName string, operatorURL string) error { msg := "" if originalError != nil { msg += urls.TrimQueryParamsStr(errors.Message(originalError)) + "\n\n" } if envName == "" { msg += fmt.Sprintf("unable to connect to your cluster (operator endpoint: %s)\n\n", operatorURL) msg += "if you don't have a cluster running:\n" msg += " → to create a cluster, run `cortex cluster up`\n" msg += "\nif you have a cluster running:\n" msg += " → run `cortex cluster info --configure-env ENV_NAME` to update your environment (replace ENV_NAME with your desired environment name, and include `--config ` if you have a cluster configuration file)\n" } else { msg += fmt.Sprintf("unable to connect to your cluster in the %s environment (operator endpoint: %s)\n\n", envName, operatorURL) msg += "if you don't have a cluster running:\n" msg += fmt.Sprintf(" → if you'd like to create a cluster, run `cortex cluster up --configure-env %s`\n", envName) msg += fmt.Sprintf(" → otherwise you can ignore this message, and prevent it in the future with `cortex env delete %s`\n", envName) msg += "\nif you have a cluster running:\n" msg += fmt.Sprintf(" → run `cortex cluster info --configure-env %s` to update your environment (include `--config ` if you have a cluster configuration file)\n", envName) msg += fmt.Sprintf(" → if you set `operator_load_balancer_scheme: internal` in your cluster configuration file, your CLI must run from within a VPC that has access to your cluster's VPC (see https://docs.cortexlabs.com/v/%s/)\n", consts.CortexVersionMinor) msg += fmt.Sprintf(" → confirm that the ip address of this machine falls within the CIDR ranges specified in `operator_load_balancer_cidr_whitelist`") } return errors.WithStack(&errors.Error{ Kind: ErrFailedToConnectOperator, Message: msg, }) } func ErrorOperatorSocketRead(err error) error { return errors.WithStack(&errors.Error{ Kind: ErrOperatorSocketRead, Message: err.Error(), NoPrint: true, }) } func ErrorResponseUnknown(body string, statusCode int) error { msg := body if strings.TrimSpace(body) == "" { msg = fmt.Sprintf("empty response (status code %d)", statusCode) } return errors.WithStack(&errors.Error{ Kind: ErrResponseUnknown, Message: msg, }) } func ErrorOperatorResponseUnknown(body string, statusCode int) error { return errors.WithStack(&errors.Error{ Kind: ErrOperatorResponseUnknown, Message: fmt.Sprintf("unexpected response from operator (status code %d): %s", statusCode, body), }) } func ErrorOperatorStreamResponseUnknown(body string, statusCode int) error { return errors.WithStack(&errors.Error{ Kind: ErrOperatorStreamResponseUnknown, Message: fmt.Sprintf("unexpected response from operator (status code %d): %s", statusCode, body), }) } ================================================ FILE: cli/cluster/get.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package cluster import ( "path" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/json" "github.com/cortexlabs/cortex/pkg/operator/schema" ) func GetAPIs(operatorConfig OperatorConfig) ([]schema.APIResponse, error) { httpRes, err := HTTPGet(operatorConfig, "/get") if err != nil { return nil, err } var apisRes []schema.APIResponse if err = json.Unmarshal(httpRes, &apisRes); err != nil { return nil, errors.Wrap(err, "/get", string(httpRes)) } return apisRes, nil } func GetAPI(operatorConfig OperatorConfig, apiName string) ([]schema.APIResponse, error) { httpRes, err := HTTPGet(operatorConfig, "/get/"+apiName) if err != nil { return nil, err } var apiRes []schema.APIResponse if err = json.Unmarshal(httpRes, &apiRes); err != nil { return nil, errors.Wrap(err, "/get/"+apiName, string(httpRes)) } return apiRes, nil } func DescribeAPI(operatorConfig OperatorConfig, apiName string) ([]schema.APIResponse, error) { httpRes, err := HTTPGet(operatorConfig, "/describe/"+apiName) if err != nil { return nil, err } var apiRes []schema.APIResponse if err = json.Unmarshal(httpRes, &apiRes); err != nil { return nil, errors.Wrap(err, "/describe/"+apiName, string(httpRes)) } return apiRes, nil } func GetAPIByID(operatorConfig OperatorConfig, apiName string, apiID string) ([]schema.APIResponse, error) { httpRes, err := HTTPGet(operatorConfig, "/get/"+apiName+"/"+apiID) if err != nil { return nil, err } var apiRes []schema.APIResponse if err = json.Unmarshal(httpRes, &apiRes); err != nil { return nil, errors.Wrap(err, "/get/"+apiName+"/"+apiID, string(httpRes)) } return apiRes, nil } func GetBatchJob(operatorConfig OperatorConfig, apiName string, jobID string) (schema.BatchJobResponse, error) { endpoint := path.Join("/batch", apiName) httpRes, err := HTTPGet(operatorConfig, endpoint, map[string]string{"jobID": jobID}) if err != nil { return schema.BatchJobResponse{}, err } var jobRes schema.BatchJobResponse if err = json.Unmarshal(httpRes, &jobRes); err != nil { return schema.BatchJobResponse{}, errors.Wrap(err, endpoint, string(httpRes)) } return jobRes, nil } func GetTaskJob(operatorConfig OperatorConfig, apiName string, jobID string) (schema.TaskJobResponse, error) { endpoint := path.Join("/tasks", apiName) httpRes, err := HTTPGet(operatorConfig, endpoint, map[string]string{"jobID": jobID}) if err != nil { return schema.TaskJobResponse{}, err } var jobRes schema.TaskJobResponse if err = json.Unmarshal(httpRes, &jobRes); err != nil { return schema.TaskJobResponse{}, errors.Wrap(err, endpoint, string(httpRes)) } return jobRes, nil } ================================================ FILE: cli/cluster/info.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package cluster import ( "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/json" "github.com/cortexlabs/cortex/pkg/operator/schema" ) func Info(operatorConfig OperatorConfig) (*schema.InfoResponse, error) { httpResponse, err := HTTPGet(operatorConfig, "/info") if err != nil { return nil, errors.Wrap(err, "unable to connect to operator", "/info") } var infoResponse schema.InfoResponse err = json.Unmarshal(httpResponse, &infoResponse) if err != nil { return nil, errors.Wrap(err, "/info", string(httpResponse)) } return &infoResponse, nil } ================================================ FILE: cli/cluster/lib_http_client.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package cluster import ( "bytes" "crypto/tls" "io" "io/ioutil" "mime/multipart" "net/http" "time" "github.com/cortexlabs/cortex/pkg/consts" "github.com/cortexlabs/cortex/pkg/lib/archive" "github.com/cortexlabs/cortex/pkg/lib/aws" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/files" "github.com/cortexlabs/cortex/pkg/lib/json" "github.com/cortexlabs/cortex/pkg/operator/schema" ) type OperatorClient struct { *http.Client } type OperatorConfig struct { Telemetry bool ClientID string EnvName string OperatorEndpoint string } func HTTPGet(operatorConfig OperatorConfig, endpoint string, qParams ...map[string]string) ([]byte, error) { req, err := operatorRequest(operatorConfig, "GET", endpoint, nil, qParams...) if err != nil { return nil, err } return makeOperatorRequest(operatorConfig, req) } func HTTPPostObjAsJSON(operatorConfig OperatorConfig, endpoint string, requestData interface{}, qParams ...map[string]string) ([]byte, error) { jsonRequestData, err := json.Marshal(requestData) if err != nil { return nil, err } return HTTPPostJSON(operatorConfig, endpoint, jsonRequestData, qParams...) } func HTTPPostJSON(operatorConfig OperatorConfig, endpoint string, jsonRequestData []byte, qParams ...map[string]string) ([]byte, error) { payload := bytes.NewBuffer(jsonRequestData) req, err := operatorRequest(operatorConfig, http.MethodPost, endpoint, payload, qParams...) if err != nil { return nil, err } req.Header.Set("Content-Type", "application/json") return makeOperatorRequest(operatorConfig, req) } func HTTPPostNoBody(operatorConfig OperatorConfig, endpoint string, qParams ...map[string]string) ([]byte, error) { req, err := operatorRequest(operatorConfig, http.MethodPost, endpoint, nil, qParams...) if err != nil { return nil, err } return makeOperatorRequest(operatorConfig, req) } func HTTPDelete(operatorConfig OperatorConfig, endpoint string, qParams ...map[string]string) ([]byte, error) { req, err := operatorRequest(operatorConfig, http.MethodDelete, endpoint, nil, qParams...) if err != nil { return nil, err } return makeOperatorRequest(operatorConfig, req) } type HTTPUploadInput struct { FilePaths map[string]string Bytes map[string][]byte } func HTTPUpload(operatorConfig OperatorConfig, endpoint string, input *HTTPUploadInput, qParams ...map[string]string) ([]byte, error) { body := new(bytes.Buffer) writer := multipart.NewWriter(body) for fileName, filePath := range input.FilePaths { file, err := files.Open(filePath) if err != nil { return nil, err } defer file.Close() if err := addFileToMultipart(fileName, writer, file); err != nil { return nil, err } } for fileName, fileBytes := range input.Bytes { if err := addFileToMultipart(fileName, writer, bytes.NewReader(fileBytes)); err != nil { return nil, err } } if err := writer.Close(); err != nil { return nil, errors.Wrap(err, _errStrCantMakeRequest) } req, err := operatorRequest(operatorConfig, http.MethodPost, endpoint, body, qParams...) if err != nil { return nil, err } req.Header.Set("Content-Type", writer.FormDataContentType()) return makeOperatorRequest(operatorConfig, req) } func addFileToMultipart(fileName string, writer *multipart.Writer, reader io.Reader) error { part, err := writer.CreateFormFile(fileName, fileName) if err != nil { return errors.Wrap(err, _errStrCantMakeRequest) } if _, err = io.Copy(part, reader); err != nil { return errors.Wrap(err, _errStrCantMakeRequest) } return nil } func HTTPUploadZip(operatorConfig OperatorConfig, endpoint string, zipInput *archive.Input, fileName string, qParams ...map[string]string) ([]byte, error) { zipBytes, _, err := archive.ZipToMem(zipInput) if err != nil { return nil, errors.Wrap(err, "failed to zip configuration file") } uploadInput := &HTTPUploadInput{ Bytes: map[string][]byte{ fileName: zipBytes, }, } return HTTPUpload(operatorConfig, endpoint, uploadInput, qParams...) } func operatorRequest(operatorConfig OperatorConfig, method string, endpoint string, body io.Reader, qParams ...map[string]string) (*http.Request, error) { req, err := http.NewRequest(method, operatorConfig.OperatorEndpoint+endpoint, body) if err != nil { return nil, errors.Wrap(err, _errStrCantMakeRequest) } values := req.URL.Query() for _, paramMap := range qParams { for key, value := range paramMap { values.Set(key, value) } } req.URL.RawQuery = values.Encode() return req, nil } func makeOperatorRequest(operatorConfig OperatorConfig, request *http.Request) ([]byte, error) { if operatorConfig.Telemetry { values := request.URL.Query() values.Set("clientID", operatorConfig.ClientID) request.URL.RawQuery = values.Encode() } request.Header.Set("CortexAPIVersion", consts.CortexVersion) awsClient, err := aws.New() if err != nil { return nil, err } authHeader, err := awsClient.IdentityRequestAsHeader() if err != nil { return nil, err } request.Header.Set(consts.AuthHeader, authHeader) timeout := 600 * time.Second if request.URL.Path == "/info" { timeout = 10 * time.Second } client := &http.Client{ Timeout: timeout, Transport: &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, }, } response, err := client.Do(request) if err != nil { return nil, ErrorFailedToConnectOperator(err, operatorConfig.EnvName, operatorConfig.OperatorEndpoint) } defer response.Body.Close() if response.StatusCode != 200 { bodyBytes, err := ioutil.ReadAll(response.Body) if err != nil { return nil, errors.Wrap(err, _errStrRead) } var output schema.ErrorResponse err = json.Unmarshal(bodyBytes, &output) if err != nil || output.Message == "" { return nil, ErrorOperatorResponseUnknown(string(bodyBytes), response.StatusCode) } return nil, errors.WithStack(&errors.Error{ Kind: output.Kind, Message: output.Message, NoTelemetry: true, }) } bodyBytes, err := ioutil.ReadAll(response.Body) if err != nil { return nil, errors.Wrap(err, _errStrRead) } return bodyBytes, nil } ================================================ FILE: cli/cluster/logs.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package cluster import ( "crypto/tls" "fmt" "io/ioutil" "net/http" "os" "os/signal" "strings" "github.com/cortexlabs/cortex/cli/lib/routines" "github.com/cortexlabs/cortex/pkg/consts" "github.com/cortexlabs/cortex/pkg/lib/aws" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/exit" "github.com/cortexlabs/cortex/pkg/lib/json" "github.com/cortexlabs/cortex/pkg/operator/schema" "github.com/gorilla/websocket" ) func GetLogs(operatorConfig OperatorConfig, apiName string) (schema.LogResponse, error) { httpRes, err := HTTPGet(operatorConfig, "/logs/"+apiName) if err != nil { return schema.LogResponse{}, err } var logResponse schema.LogResponse if err = json.Unmarshal(httpRes, &logResponse); err != nil { return schema.LogResponse{}, errors.Wrap(err, "/logs/"+apiName, string(httpRes)) } return logResponse, nil } func GetJobLogs(operatorConfig OperatorConfig, apiName string, jobID string) (schema.LogResponse, error) { httpRes, err := HTTPGet(operatorConfig, "/logs/"+apiName, map[string]string{"jobID": jobID}) if err != nil { return schema.LogResponse{}, err } var logResponse schema.LogResponse if err = json.Unmarshal(httpRes, &logResponse); err != nil { return schema.LogResponse{}, errors.Wrap(err, "/logs/"+apiName, string(httpRes)) } return logResponse, nil } func StreamLogs(operatorConfig OperatorConfig, apiName string) error { return streamLogs(operatorConfig, "/streamlogs/"+apiName) } func StreamJobLogs(operatorConfig OperatorConfig, apiName string, jobID string) error { return streamLogs(operatorConfig, "/streamlogs/"+apiName, map[string]string{"jobID": jobID}) } func streamLogs(operatorConfig OperatorConfig, path string, qParams ...map[string]string) error { interrupt := make(chan os.Signal, 1) signal.Notify(interrupt, os.Interrupt) req, err := operatorRequest(operatorConfig, "GET", path, nil, qParams...) if err != nil { return err } values := req.URL.Query() if operatorConfig.Telemetry { values.Set("clientID", operatorConfig.ClientID) } req.URL.RawQuery = values.Encode() wsURL := req.URL.String() wsURL = strings.Replace(wsURL, "http", "ws", 1) header := http.Header{} header.Set("CortexAPIVersion", consts.CortexVersion) awsClient, err := aws.New() if err != nil { return err } authHeader, err := awsClient.IdentityRequestAsHeader() if err != nil { return err } header.Set(consts.AuthHeader, authHeader) var dialer = websocket.Dialer{ TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, } connection, response, err := dialer.Dial(wsURL, header) if err != nil && response == nil { return ErrorFailedToConnectOperator(err, operatorConfig.EnvName, strings.Replace(operatorConfig.OperatorEndpoint, "http", "ws", 1)) } defer response.Body.Close() if err != nil { bodyBytes, err := ioutil.ReadAll(response.Body) if err != nil || bodyBytes == nil || string(bodyBytes) == "" { return ErrorFailedToConnectOperator(err, operatorConfig.EnvName, strings.Replace(operatorConfig.OperatorEndpoint, "http", "ws", 1)) } var output schema.ErrorResponse err = json.Unmarshal(bodyBytes, &output) if err != nil || output.Message == "" { return ErrorOperatorStreamResponseUnknown(string(bodyBytes), response.StatusCode) } return errors.WithStack(&errors.Error{ Kind: output.Kind, Message: output.Message, NoTelemetry: true, }) } defer connection.Close() done := make(chan struct{}) handleConnection(connection, done) closeConnection(connection, done, interrupt) return nil } func handleConnection(connection *websocket.Conn, done chan struct{}) { routines.RunWithPanicHandler(func() { defer close(done) for { _, message, err := connection.ReadMessage() if err != nil { exit.Error(ErrorOperatorSocketRead(err)) } fmt.Print(string(message)) } }, false) } func closeConnection(connection *websocket.Conn, done chan struct{}, interrupt chan os.Signal) { for { select { case <-done: return case <-interrupt: connection.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, "")) return } } } ================================================ FILE: cli/cluster/refresh.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package cluster import ( "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/json" s "github.com/cortexlabs/cortex/pkg/lib/strings" "github.com/cortexlabs/cortex/pkg/operator/schema" ) func Refresh(operatorConfig OperatorConfig, apiName string, force bool) (schema.RefreshResponse, error) { params := map[string]string{ "force": s.Bool(force), } httpRes, err := HTTPPostNoBody(operatorConfig, "/refresh/"+apiName, params) if err != nil { return schema.RefreshResponse{}, err } var refreshRes schema.RefreshResponse err = json.Unmarshal(httpRes, &refreshRes) if err != nil { return schema.RefreshResponse{}, errors.Wrap(err, "/refresh", string(httpRes)) } return refreshRes, nil } ================================================ FILE: cli/cmd/cluster.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package cmd import ( "encoding/base64" "fmt" "os" "path/filepath" "regexp" "strings" "time" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/autoscaling" "github.com/aws/aws-sdk-go/service/ec2" "github.com/aws/aws-sdk-go/service/eks" "github.com/aws/aws-sdk-go/service/elb" "github.com/aws/aws-sdk-go/service/elbv2" "github.com/aws/aws-sdk-go/service/s3" "github.com/cortexlabs/cortex/cli/cluster" "github.com/cortexlabs/cortex/cli/types/cliconfig" "github.com/cortexlabs/cortex/cli/types/flags" "github.com/cortexlabs/cortex/pkg/consts" "github.com/cortexlabs/cortex/pkg/health" awslib "github.com/cortexlabs/cortex/pkg/lib/aws" "github.com/cortexlabs/cortex/pkg/lib/console" "github.com/cortexlabs/cortex/pkg/lib/docker" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/exit" "github.com/cortexlabs/cortex/pkg/lib/files" libjson "github.com/cortexlabs/cortex/pkg/lib/json" "github.com/cortexlabs/cortex/pkg/lib/k8s" libmath "github.com/cortexlabs/cortex/pkg/lib/math" "github.com/cortexlabs/cortex/pkg/lib/pointer" "github.com/cortexlabs/cortex/pkg/lib/prompt" s "github.com/cortexlabs/cortex/pkg/lib/strings" "github.com/cortexlabs/cortex/pkg/lib/table" "github.com/cortexlabs/cortex/pkg/lib/telemetry" libtime "github.com/cortexlabs/cortex/pkg/lib/time" "github.com/cortexlabs/cortex/pkg/operator/schema" "github.com/cortexlabs/cortex/pkg/types/clusterconfig" "github.com/cortexlabs/cortex/pkg/types/clusterstate" "github.com/cortexlabs/yaml" "github.com/spf13/cobra" "k8s.io/apimachinery/pkg/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" "sigs.k8s.io/aws-iam-authenticator/pkg/token" ) var ( _flagClusterUpEnv string _flagClusterInfoEnv string _flagClusterConfig string _flagClusterName string _flagClusterRegion string _flagClusterInfoDebug bool _flagClusterInfoPrintConfig bool _flagClusterDisallowPrompt bool _flagClusterDownKeepAWSResources bool ) var _eksctlPrefixRegex = regexp.MustCompile(`^.*[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2} \[.+] {2}`) func clusterInit() { _clusterUpCmd.Flags().SortFlags = false _clusterUpCmd.Flags().StringVarP(&_flagClusterUpEnv, "configure-env", "e", "", "name of environment to configure (default: the name of your cluster)") _clusterUpCmd.Flags().BoolVarP(&_flagClusterDisallowPrompt, "yes", "y", false, "skip prompts") _clusterCmd.AddCommand(_clusterUpCmd) _clusterInfoCmd.Flags().SortFlags = false addClusterConfigFlag(_clusterInfoCmd) addClusterNameFlag(_clusterInfoCmd) addClusterRegionFlag(_clusterInfoCmd) _clusterInfoCmd.Flags().VarP(&_flagOutput, "output", "o", fmt.Sprintf("output format: one of %s", strings.Join(flags.OutputTypeStrings(), "|"))) _clusterInfoCmd.Flags().StringVarP(&_flagClusterInfoEnv, "configure-env", "e", "", "name of environment to configure") _clusterInfoCmd.Flags().BoolVarP(&_flagClusterInfoDebug, "debug", "d", false, "save the current cluster state to a file") _clusterInfoCmd.Flags().BoolVarP(&_flagClusterInfoPrintConfig, "print-config", "", false, "print the cluster config") _clusterInfoCmd.Flags().BoolVarP(&_flagClusterDisallowPrompt, "yes", "y", false, "skip prompts") _clusterCmd.AddCommand(_clusterInfoCmd) _clusterConfigureCmd.Flags().SortFlags = false _clusterConfigureCmd.Flags().BoolVarP(&_flagClusterDisallowPrompt, "yes", "y", false, "skip prompts") _clusterCmd.AddCommand(_clusterConfigureCmd) _clusterDownCmd.Flags().SortFlags = false addClusterConfigFlag(_clusterDownCmd) addClusterNameFlag(_clusterDownCmd) addClusterRegionFlag(_clusterDownCmd) _clusterDownCmd.Flags().BoolVarP(&_flagClusterDisallowPrompt, "yes", "y", false, "skip prompts") _clusterDownCmd.Flags().BoolVar(&_flagClusterDownKeepAWSResources, "keep-aws-resources", false, "skip deletion of resources that cortex provisioned on aws (bucket contents, ebs volumes, log group)") _clusterCmd.AddCommand(_clusterDownCmd) _clusterExportCmd.Flags().SortFlags = false addClusterConfigFlag(_clusterExportCmd) addClusterNameFlag(_clusterExportCmd) addClusterRegionFlag(_clusterExportCmd) _clusterCmd.AddCommand(_clusterExportCmd) _clusterHealthCmd.Flags().SortFlags = false addClusterConfigFlag(_clusterHealthCmd) addClusterNameFlag(_clusterHealthCmd) addClusterRegionFlag(_clusterHealthCmd) _clusterHealthCmd.Flags().VarP(&_flagOutput, "output", "o", fmt.Sprintf("output format: one of %s", strings.Join(flags.OutputTypeStringsExcluding(flags.YAMLOutputType), "|"))) _clusterCmd.AddCommand(_clusterHealthCmd) } func addClusterConfigFlag(cmd *cobra.Command) { cmd.Flags().StringVarP(&_flagClusterConfig, "config", "c", "", "path to a cluster configuration file") err := cmd.Flags().SetAnnotation("config", cobra.BashCompFilenameExt, _configFileExts) if err != nil { exit.Error(err) // should never happen } } func addClusterNameFlag(cmd *cobra.Command) { cmd.Flags().StringVarP(&_flagClusterName, "name", "n", "", "name of the cluster") } func addClusterRegionFlag(cmd *cobra.Command) { cmd.Flags().StringVarP(&_flagClusterRegion, "region", "r", "", "aws region of the cluster") } var _clusterCmd = &cobra.Command{ Use: "cluster", Short: "manage cortex clusters (contains subcommands)", } var _clusterUpCmd = &cobra.Command{ Use: "up CLUSTER_CONFIG_FILE", Short: "spin up a cluster on aws", Args: cobra.ExactArgs(1), Run: func(cmd *cobra.Command, args []string) { telemetry.EventNotify("cli.cluster.up") clusterConfigFile := args[0] if _, err := docker.GetDockerClient(); err != nil { exit.Error(err) } accessConfig, err := getNewClusterAccessConfig(clusterConfigFile) if err != nil { exit.Error(err) } envName := _flagClusterUpEnv if envName == "" { envName = accessConfig.ClusterName } envExists, err := isEnvConfigured(envName) if err != nil { exit.Error(err) } if envExists { if _flagClusterDisallowPrompt { fmt.Printf("found an existing environment named \"%s\", which will be overwritten to connect to this cluster once it's created\n\n", envName) } else { prompt.YesOrExit(fmt.Sprintf("found an existing environment named \"%s\"; would you like to overwrite it to connect to this cluster once it's created?", envName), "", "you can specify a different environment name to be configured to connect to this cluster by specifying the --configure-env flag (e.g. `cortex cluster up --configure-env prod`); or you can list your environments with `cortex env list` and delete an environment with `cortex env delete ENV_NAME`") } } awsClient, err := newAWSClient(accessConfig.Region, true) if err != nil { exit.Error(err) } stacks, err := clusterstate.GetClusterStacks(awsClient, accessConfig) if err != nil { exit.Error(err) } state := clusterstate.GetClusterState(stacks) if err := clusterstate.AssertClusterState(stacks, state, clusterstate.StateClusterDoesntExist); err != nil { exit.Error(err) } promptIfNotAdmin(awsClient, _flagClusterDisallowPrompt) clusterConfig, err := getInstallClusterConfig(awsClient, clusterConfigFile) if err != nil { exit.Error(err) } confirmInstallClusterConfig(clusterConfig, awsClient, _flagClusterDisallowPrompt) err = createS3BucketIfNotFound(awsClient, clusterConfig.Bucket, clusterConfig.Tags) if err != nil { exit.Error(err) } err = setLifecycleRulesOnClusterUp(awsClient, clusterConfig.Bucket, clusterConfig.ClusterUID) if err != nil { exit.Error(err) } err = createLogGroupIfNotFound(awsClient, clusterConfig.ClusterName, clusterConfig.Tags) if err != nil { exit.Error(err) } accountID, _, err := awsClient.GetCachedAccountID() if err != nil { exit.Error(err) } err = clusterconfig.CreateDefaultPolicy(awsClient, clusterconfig.CortexPolicyTemplateArgs{ ClusterName: clusterConfig.ClusterName, LogGroup: clusterConfig.ClusterName, Bucket: clusterConfig.Bucket, Region: clusterConfig.Region, AccountID: accountID, }) if err != nil { exit.Error(err) } out, exitCode, err := runManagerWithClusterConfig("/root/install.sh", clusterConfig, awsClient, nil, nil, nil) if err != nil { exit.Error(err) } if exitCode == nil || *exitCode != 0 { out = s.LastNChars(filterEKSCTLOutput(out), 8192) // get the last 8192 characters because that is the sentry message limit eksCluster, err := awsClient.EKSClusterOrNil(clusterConfig.ClusterName) if err != nil { helpStr := "\ndebugging tips (may or may not apply to this error):" helpStr += fmt.Sprintf("\n* if your cluster started spinning up but was unable to provision instances, additional error information may be found in the activity history of your cluster's autoscaling groups (select each autoscaling group and click the \"Activity\" or \"Activity History\" tab): https://console.aws.amazon.com/ec2/autoscaling/home?region=%s#AutoScalingGroups:", clusterConfig.Region) helpStr += "\n* if your cluster started spinning up, please run `cortex cluster down` to delete the cluster before trying to create this cluster again" fmt.Println(helpStr) exit.Error(ErrorClusterUp(out)) } // the cluster never started spinning up if eksCluster == nil { exit.Error(ErrorClusterUp(out)) } clusterTags := map[string]string{clusterconfig.ClusterNameTag: clusterConfig.ClusterName} asgs, err := awsClient.AutoscalingGroups(clusterTags) if err != nil { helpStr := "\ndebugging tips (may or may not apply to this error):" helpStr += fmt.Sprintf("\n* if your cluster was unable to provision instances, additional error information may be found in the activity history of your cluster's autoscaling groups (select each autoscaling group and click the \"Activity\" or \"Activity History\" tab): https://console.aws.amazon.com/ec2/autoscaling/home?region=%s#AutoScalingGroups:", clusterConfig.Region) helpStr += "\n* please run `cortex cluster down` to delete the cluster before trying to create this cluster again" fmt.Println(helpStr) exit.Error(ErrorClusterUp(out + helpStr)) } // no autoscaling groups were created if len(asgs) == 0 { helpStr := "\nplease run `cortex cluster down` to delete the cluster before trying to create this cluster again" fmt.Println(helpStr) exit.Error(ErrorClusterUp(out + helpStr)) } for _, asg := range asgs { activity, err := awsClient.MostRecentASGActivity(*asg.AutoScalingGroupName) if err != nil { helpStr := "\ndebugging tips (may or may not apply to this error):" helpStr += fmt.Sprintf("\n* if your cluster was unable to provision instances, additional error information may be found in the activity history of your cluster's autoscaling groups (select each autoscaling group and click the \"Activity\" or \"Activity History\" tab): https://console.aws.amazon.com/ec2/autoscaling/home?region=%s#AutoScalingGroups:", clusterConfig.Region) helpStr += "\n* please run `cortex cluster down` to delete the cluster before trying to create this cluster again" fmt.Println(helpStr) exit.Error(ErrorClusterUp(out + helpStr)) } if activity != nil && (activity.StatusCode == nil || *activity.StatusCode != autoscaling.ScalingActivityStatusCodeSuccessful) { status := "(none)" if activity.StatusCode != nil { status = *activity.StatusCode } description := "(none)" if activity.Description != nil { description = *activity.Description } helpStr := "\nyour cluster was unable to provision EC2 instances; here is one of the encountered errors:" helpStr += fmt.Sprintf("\n\n> status: %s\n> description: %s", status, description) helpStr += fmt.Sprintf("\n\nadditional error information might be found in the activity history of your cluster's autoscaling groups (select each autoscaling group and click the \"Activity\" or \"Activity History\" tab): https://console.aws.amazon.com/ec2/autoscaling/home?region=%s#AutoScalingGroups:", clusterConfig.Region) helpStr += "\n\nplease run `cortex cluster down` to delete the cluster before trying to create this cluster again" fmt.Println(helpStr) exit.Error(ErrorClusterUp(out + helpStr)) } } // No failed asg activities helpStr := "\nplease run `cortex cluster down` to delete the cluster before trying to create this cluster again" fmt.Println(helpStr) exit.Error(ErrorClusterUp(out + helpStr)) } loadBalancer, err := getNLBLoadBalancer(clusterConfig.ClusterName, OperatorLoadBalancer, awsClient) if err != nil { exit.Error(errors.Append(err, fmt.Sprintf("\n\nyou can attempt to resolve this issue and configure your cli environment by running `cortex cluster info --configure-env %s`", envName))) } newEnvironment := cliconfig.Environment{ Name: envName, OperatorEndpoint: "https://" + *loadBalancer.DNSName, } err = addEnvToCLIConfig(newEnvironment, true) if err != nil { exit.Error(errors.Append(err, fmt.Sprintf("\n\nyou can attempt to resolve this issue and configure your cli environment by running `cortex cluster info --configure-env %s`", envName))) } if envExists { fmt.Printf(console.Bold("\nthe environment named \"%s\" has been updated to point to this cluster (and was set as the default environment)\n"), envName) } else { fmt.Printf(console.Bold("\nan environment named \"%s\" has been configured to point to this cluster (and was set as the default environment)\n"), envName) } }, } var _clusterConfigureCmd = &cobra.Command{ Use: "configure CLUSTER_CONFIG_FILE", Short: "update the cluster's configuration", Args: cobra.ExactArgs(1), Run: func(cmd *cobra.Command, args []string) { telemetry.Event("cli.cluster.configure") clusterConfigFile := args[0] if _, err := docker.GetDockerClient(); err != nil { exit.Error(err) } accessConfig, err := getNewClusterAccessConfig(clusterConfigFile) if err != nil { exit.Error(err) } awsClient, err := newAWSClient(accessConfig.Region, true) if err != nil { exit.Error(err) } restConfig, err := getClusterRESTConfig(awsClient, accessConfig.ClusterName) if err != nil { exit.Error(err) } scheme := runtime.NewScheme() if err := clientgoscheme.AddToScheme(scheme); err != nil { exit.Error(err) } k8sClient, err := k8s.New(consts.DefaultNamespace, false, restConfig, scheme) if err != nil { exit.Error(err) } stacks, err := clusterstate.GetClusterStacks(awsClient, accessConfig) if err != nil { exit.Error(err) } state := clusterstate.GetClusterState(stacks) if err := clusterstate.AssertClusterState(stacks, state, clusterstate.StateClusterExists); err != nil { exit.Error(err) } oldClusterConfig := refreshCachedClusterConfig(awsClient, accessConfig, true) promptIfNotAdmin(awsClient, _flagClusterDisallowPrompt) newClusterConfig, configureChanges, err := getConfigureClusterConfig(awsClient, k8sClient, stacks, oldClusterConfig, clusterConfigFile) if err != nil { exit.Error(err) } if !configureChanges.HasChanges() { fmt.Println("your cluster is already up to date") exit.Ok() } confirmConfigureClusterConfig(configureChanges, oldClusterConfig, *newClusterConfig, _flagClusterDisallowPrompt) out, exitCode, err := runManagerWithClusterConfig("/root/install.sh --configure", newClusterConfig, awsClient, nil, nil, []string{ "CORTEX_NODEGROUP_NAMES_TO_UPDATE=" + strings.Join(configureChanges.NodeGroupsToUpdate, " "), // NodeGroupsToUpdate contain the cluster config node-group names "CORTEX_NODEGROUP_NAMES_TO_ADD=" + strings.Join(configureChanges.NodeGroupsToAdd, " "), // NodeGroupsToAdd contain the cluster config node-group names "CORTEX_EKS_NODEGROUP_NAMES_TO_REMOVE=" + strings.Join(configureChanges.EKSNodeGroupsToRemove, " "), // EKSNodeGroupsToRemove contain the EKS node-group names }) if err != nil { exit.Error(err) } if exitCode == nil || *exitCode != 0 { out = s.LastNChars(out, 8192) // get the last 8192 characters because that is the sentry message limit helpStr := "\ndebugging tips (may or may not apply to this error):" helpStr += fmt.Sprintf( "\n* if your cluster was unable to provision/remove/scale some nodegroups, additional error information may be found in the description of your cloudformation stack (https://console.aws.amazon.com/cloudformation/home?region=%s#/stacks)"+ " or in the activity history of your cluster's autoscaling groups (select each autoscaling group and click the \"Activity\" or \"Activity History\" tab) (https://console.aws.amazon.com/ec2/autoscaling/home?region=%s#AutoScalingGroups)", oldClusterConfig.Region, oldClusterConfig.Region, ) fmt.Println(helpStr) exit.Error(ErrorClusterConfigure(out + helpStr)) } }, } var _clusterInfoCmd = &cobra.Command{ Use: "info", Short: "get information about a cluster", Args: cobra.NoArgs, Run: func(cmd *cobra.Command, args []string) { telemetry.Event("cli.cluster.info") if _, err := docker.GetDockerClient(); err != nil { exit.Error(err) } accessConfig, err := getClusterAccessConfigWithCache(true) if err != nil { exit.Error(err) } if _flagClusterInfoPrintConfig && _flagOutput == flags.PrettyOutputType { _flagOutput = flags.YAMLOutputType } awsClient, err := newAWSClient(accessConfig.Region, _flagOutput == flags.PrettyOutputType) if err != nil { exit.Error(err) } if _flagClusterInfoPrintConfig && _flagClusterInfoDebug { exit.Error(ErrorMutuallyExclusiveFlags("--print-config", "--debug")) } if _flagClusterInfoDebug && _flagOutput != flags.PrettyOutputType { exit.Error(ErrorMutuallyExclusiveFlags("--debug", "--output")) } stacks, err := clusterstate.GetClusterStacks(awsClient, accessConfig) if err != nil { exit.Error(err) } state := clusterstate.GetClusterState(stacks) if err := clusterstate.AssertClusterState(stacks, state, clusterstate.StateClusterExists); err != nil { exit.Error(err) } if _flagClusterInfoDebug { cmdDebug(awsClient, accessConfig) } else if _flagClusterInfoPrintConfig { cmdPrintConfig(awsClient, accessConfig, _flagOutput) } else { cmdInfo(awsClient, accessConfig, stacks, _flagOutput, _flagClusterDisallowPrompt) } }, } var _clusterDownCmd = &cobra.Command{ Use: "down", Short: "spin down a cluster", Args: cobra.NoArgs, Run: func(cmd *cobra.Command, args []string) { telemetry.Event("cli.cluster.down") if _, err := docker.GetDockerClient(); err != nil { exit.Error(err) } accessConfig, err := getClusterAccessConfigWithCache(true) if err != nil { exit.Error(err) } // Check AWS access awsClient, err := newAWSClient(accessConfig.Region, true) if err != nil { exit.Error(err) } accountID, _, err := awsClient.GetCachedAccountID() if err != nil { exit.Error(err) } bucketName := clusterconfig.BucketName(accountID, accessConfig.ClusterName, accessConfig.Region) warnIfNotAdmin(awsClient) if _flagClusterDisallowPrompt { fmt.Printf("your cluster named \"%s\" in %s will be spun down and all apis will be deleted\n\n", accessConfig.ClusterName, accessConfig.Region) } else { prompt.YesOrExit(fmt.Sprintf("your cluster named \"%s\" in %s will be spun down and all apis will be deleted, are you sure you want to continue?", accessConfig.ClusterName, accessConfig.Region), "", "") } var clusterExists bool errorsList := []error{} fmt.Print("￮ retrieving cluster ... ") stacks, err := clusterstate.GetClusterStacks(awsClient, accessConfig) if err != nil { errorsList = append(errorsList, err) fmt.Print("failed ✗") fmt.Printf("\n\ncouldn't retrieve cluster state; check the cluster stacks in the cloudformation console: https://%s.console.aws.amazon.com/cloudformation\n", accessConfig.Region) errors.PrintError(err) fmt.Println() } else { if clusterstate.GetClusterState(stacks) == clusterstate.StateClusterDoesntExist { fmt.Println("cluster doesn't exist ✓") } else { fmt.Println("✓") clusterExists = true } } // updating CLI env is best-effort, so ignore errors loadBalancer, _ := getNLBLoadBalancer(accessConfig.ClusterName, OperatorLoadBalancer, awsClient) fmt.Print("￮ deleting sqs queues ... ") numDeleted, err := awsClient.DeleteQueuesWithPrefix(clusterconfig.SQSNamePrefix(accessConfig.ClusterName)) if err != nil { errorsList = append(errorsList, err) fmt.Print("failed ✗") fmt.Printf("\n\nfailed to delete all sqs queues; please delete queues starting with the name %s via the cloudwatch console: https://%s.console.aws.amazon.com/sqs/v2/home\n", clusterconfig.SQSNamePrefix(accessConfig.ClusterName), accessConfig.Region) errors.PrintError(err) fmt.Println() } else if numDeleted == 0 { fmt.Println("no sqs queues exist ✓") } else { fmt.Println("✓") } clusterDoesntExist := !clusterExists if clusterExists { fmt.Print("￮ spinning down the cluster ...") out, exitCode, err := runManagerAccessCommand("/root/uninstall.sh", *accessConfig, awsClient, nil, nil) if err != nil { errorsList = append(errorsList, err) fmt.Println() errors.PrintError(err) } else if exitCode == nil || *exitCode != 0 { template := "\nNote: if this error cannot be resolved, please ensure that all CloudFormation stacks for this cluster eventually become fully deleted (%s)." template += " If the stack deletion process has failed, please delete the stacks directly from the AWS console (this may require manually deleting particular AWS resources that are blocking the stack deletion)." template += " In addition to deleting the stacks manually from the AWS console, also make sure to empty and remove the %s bucket" helpStr := fmt.Sprintf(template, clusterstate.CloudFormationURL(accessConfig.ClusterName, accessConfig.Region), bucketName) fmt.Println(helpStr) errorsList = append(errorsList, ErrorClusterDown(filterEKSCTLOutput(out)+helpStr)) } else { clusterDoesntExist = true } fmt.Println() } // set lifecycle policy to clean the bucket var bucketExists bool if !_flagClusterDownKeepAWSResources { fmt.Printf("￮ setting lifecycle policy to empty the %s bucket ... ", bucketName) bucketExists, err := awsClient.DoesBucketExist(bucketName) if err != nil { errorsList = append(errorsList, err) fmt.Print("failed ✗") fmt.Printf("\n\nfailed to set lifecycle policy to empty the %s bucket; you can remove the bucket manually via the s3 console: https://s3.console.aws.amazon.com/s3/management/%s\n", bucketName, bucketName) errors.PrintError(err) fmt.Println() } else if !bucketExists { fmt.Println("bucket doesn't exist ✗") } else { err = setLifecycleRulesOnClusterDown(awsClient, bucketName) if err != nil { errorsList = append(errorsList, err) fmt.Print("failed ✗") fmt.Printf("\n\nfailed to set lifecycle policy to empty the %s bucket; you can remove the bucket manually via the s3 console: https://s3.console.aws.amazon.com/s3/management/%s\n", bucketName, bucketName) errors.PrintError(err) fmt.Println() } else { fmt.Println("✓") } } } // delete policy after spinning down the cluster (which deletes the roles) because policies can't be deleted if they are attached to roles if clusterDoesntExist { policyARN := clusterconfig.DefaultPolicyARN(accountID, accessConfig.ClusterName, accessConfig.Region) fmt.Printf("￮ deleting auto-generated iam policy %s ... ", policyARN) if policy, err := awsClient.GetPolicyOrNil(policyARN); err != nil { errorsList = append(errorsList, err) fmt.Print("failed ✗") fmt.Printf("\n\nfailed to delete auto-generated cortex policy %s; please delete the policy via the iam console: https://console.aws.amazon.com/iam/home#/policies\n", policyARN) errors.PrintError(err) fmt.Println() } else if policy == nil { fmt.Println("policy doesn't exist ✓") } else { err = awsClient.DeletePolicy(policyARN) if err != nil { errorsList = append(errorsList, err) fmt.Print("failed ✗") fmt.Printf("\n\nfailed to delete auto-generated cortex policy %s; please delete the policy via the iam console: https://console.aws.amazon.com/iam/home#/policies\n", policyARN) errors.PrintError(err) fmt.Println() } else { fmt.Println("✓") } } } if !_flagClusterDownKeepAWSResources { fmt.Print("￮ deleting ebs volumes ... ") volumes, err := listPVCVolumesForCluster(awsClient, accessConfig.ClusterName) if err != nil { errorsList = append(errorsList, err) fmt.Println("\n\nfailed to list volumes for deletion; please delete any volumes associated with your cluster via the ec2 console: https://console.aws.amazon.com/ec2/v2/home?#Volumes") errors.PrintError(err) fmt.Println() } else { var failedToDeleteVolumes []string var lastErr error for _, volume := range volumes { err := awsClient.DeleteVolume(*volume.VolumeId) if err != nil { failedToDeleteVolumes = append(failedToDeleteVolumes, *volume.VolumeId) lastErr = err } } if len(volumes) == 0 { fmt.Println("no ebs volumes exist ✓") } else if lastErr != nil { errorsList = append(errorsList, lastErr) fmt.Printf("\n\nfailed to delete %s %s; please delete %s via the ec2 console: https://console.aws.amazon.com/ec2/v2/home?#Volumes\n", s.PluralS("volume", len(failedToDeleteVolumes)), s.UserStrsAnd(failedToDeleteVolumes), s.PluralCustom("it", "them", len(failedToDeleteVolumes))) errors.PrintError(lastErr) fmt.Println() } else { fmt.Println("✓") } } fmt.Printf("￮ deleting log group %s ... ", accessConfig.ClusterName) logGroupExists, err := awsClient.DoesLogGroupExist(accessConfig.ClusterName) if err != nil { errorsList = append(errorsList, err) fmt.Print("failed ✗") fmt.Printf("\n\nfailed to list log group for deletion; please delete the log group associated with your cluster via the ec2 console: https://%s.console.aws.amazon.com/cloudwatch/home?#logsV2:log-groups\n", accessConfig.Region) errors.PrintError(err) fmt.Println() } else { if !logGroupExists { fmt.Println("log group doesn't exist ✓") } else { err = awsClient.DeleteLogGroup(accessConfig.ClusterName) if err != nil { errorsList = append(errorsList, err) fmt.Print("failed ✗") fmt.Printf("\n\nfailed to delete log group %s; please delete the log group associated with your cluster via the ec2 console: https://%s.console.aws.amazon.com/cloudwatch/home?#logsV2:log-groups\n", accessConfig.ClusterName, accessConfig.Region) errors.PrintError(err) fmt.Println() } else { fmt.Println("✓") } } } } // best-effort deletion of cached config cachedClusterConfigPath := getCachedClusterConfigPath(accessConfig.ClusterName, accessConfig.Region) _ = os.Remove(cachedClusterConfigPath) if len(errorsList) > 0 { exit.Error(errors.ListOfErrors(ErrClusterDown, false, errorsList...)) } fmt.Printf("\nplease check CloudFormation to ensure that all resources for the %s cluster eventually become successfully deleted: %s\n", accessConfig.ClusterName, clusterstate.CloudFormationURL(accessConfig.ClusterName, accessConfig.Region)) if !_flagClusterDownKeepAWSResources && bucketExists { fmt.Printf("\na lifecycle rule has been applied to the cluster's %s bucket to empty its contents within the next 24 hours; you can delete the %s bucket via the s3 console once it has been emptied (or you can empty and delete it now): https://s3.console.aws.amazon.com/s3/management/%s\n", bucketName, bucketName, bucketName) } fmt.Println() // best-effort deletion of cli environment(s) if loadBalancer != nil { envNames, isDefaultEnv, _ := getEnvNamesByOperatorEndpoint(*loadBalancer.DNSName) if len(envNames) > 0 { for _, envName := range envNames { err := removeEnvFromCLIConfig(envName) if err != nil { exit.Error(err) } } fmt.Printf("deleted the %s environment configuration%s\n", s.StrsAnd(envNames), s.SIfPlural(len(envNames))) if isDefaultEnv { newDefaultEnv, err := getDefaultEnv() if err != nil { exit.Error(err) } if newDefaultEnv != nil { fmt.Println(fmt.Sprintf("set the default environment to %s", *newDefaultEnv)) } } } } }, } var _clusterExportCmd = &cobra.Command{ Use: "export", Short: "download the configurations for all APIs", Args: cobra.NoArgs, Run: func(cmd *cobra.Command, args []string) { telemetry.Event("cli.cluster.export") accessConfig, err := getClusterAccessConfigWithCache(true) if err != nil { exit.Error(err) } // Check AWS access awsClient, err := newAWSClient(accessConfig.Region, true) if err != nil { exit.Error(err) } warnIfNotAdmin(awsClient) stacks, err := clusterstate.GetClusterStacks(awsClient, accessConfig) if err != nil { exit.Error(err) } state := clusterstate.GetClusterState(stacks) if err := clusterstate.AssertClusterState(stacks, state, clusterstate.StateClusterExists); err != nil { exit.Error(err) } loadBalancer, err := getNLBLoadBalancer(accessConfig.ClusterName, OperatorLoadBalancer, awsClient) if err != nil { exit.Error(err) } operatorConfig := cluster.OperatorConfig{ Telemetry: isTelemetryEnabled(), ClientID: clientID(), OperatorEndpoint: "https://" + *loadBalancer.DNSName, } var apisResponse []schema.APIResponse apisResponse, err = cluster.GetAPIs(operatorConfig) if err != nil { exit.Error(err) } if len(apisResponse) == 0 { fmt.Println(fmt.Sprintf("no apis found in your cluster named %s in %s", accessConfig.ClusterName, accessConfig.Region)) exit.Ok() } exportPath := fmt.Sprintf("export-%s-%s", accessConfig.Region, accessConfig.ClusterName) err = files.CreateDir(exportPath) if err != nil { exit.Error(err) } for _, api := range apisResponse { apisWithSpec, err := cluster.GetAPI(operatorConfig, api.Metadata.Name) if err != nil { exit.Error(err) } specFilePath := filepath.Join(exportPath, api.Metadata.Name+".yaml") fmt.Println(fmt.Sprintf("exporting %s to %s", api.Metadata.Name, specFilePath)) yamlBytes, err := yaml.Marshal(apisWithSpec[0].Spec.API.SubmittedAPISpec) if err != nil { exit.Error(err) } err = files.WriteFile(yamlBytes, specFilePath) if err != nil { exit.Error(err) } } }, } var _clusterHealthCmd = &cobra.Command{ Use: "health", Short: "inspect the health of components in the cluster", Args: cobra.NoArgs, Run: func(cmd *cobra.Command, args []string) { accessConfig, err := getClusterAccessConfigWithCache(true) if err != nil { exit.Error(err) } awsClient, err := awslib.NewForRegion(accessConfig.Region) if err != nil { exit.Error(err) } restConfig, err := getClusterRESTConfig(awsClient, accessConfig.ClusterName) if err != nil { exit.Error(err) } scheme := runtime.NewScheme() if err := clientgoscheme.AddToScheme(scheme); err != nil { exit.Error(err) } k8sClient, err := k8s.New(consts.DefaultNamespace, false, restConfig, scheme) if err != nil { exit.Error(err) } clusterHealth, err := health.Check(awsClient, k8sClient, accessConfig.ClusterName) if err != nil { exit.Error(err) } clusterWarnings, err := health.GetWarnings(k8sClient) if err != nil { exit.Error(err) } if _flagOutput == flags.JSONOutputType { fmt.Println(clusterHealth) return } healthTable := table.Table{ Headers: []table.Header{ {Title: ""}, {Title: "live"}, {Title: "warning", Hidden: !clusterWarnings.HasWarnings()}, }, Rows: [][]interface{}{ {"operator", console.BoolColor(clusterHealth.Operator), ""}, {"prometheus", console.BoolColor(clusterHealth.Prometheus), clusterWarnings.Prometheus}, {"autoscaler", console.BoolColor(clusterHealth.Autoscaler), ""}, {"activator", console.BoolColor(clusterHealth.Activator), ""}, {"async gateway", console.BoolColor(clusterHealth.AsyncGateway), ""}, {"grafana", console.BoolColor(clusterHealth.Grafana), ""}, {"controller manager", console.BoolColor(clusterHealth.ControllerManager), ""}, {"apis gateway", console.BoolColor(clusterHealth.APIsGateway), ""}, {"operator gateway", console.BoolColor(clusterHealth.APIsGateway), ""}, {"cluster autoscaler", console.BoolColor(clusterHealth.ClusterAutoscaler), ""}, {"operator load balancer", console.BoolColor(clusterHealth.OperatorLoadBalancer), ""}, {"apis load balancer", console.BoolColor(clusterHealth.APIsLoadBalancer), ""}, {"fluent bit", console.BoolColor(clusterHealth.FluentBit), ""}, {"node exporter", console.BoolColor(clusterHealth.NodeExporter), ""}, {"dcgm exporter", console.BoolColor(clusterHealth.DCGMExporter), ""}, {"statsd exporter", console.BoolColor(clusterHealth.StatsDExporter), ""}, {"event exporter", console.BoolColor(clusterHealth.EventExporter), ""}, {"kube state metrics", console.BoolColor(clusterHealth.KubeStateMetrics), ""}, }, } fmt.Println(healthTable.MustFormat()) }, } func cmdPrintConfig(awsClient *awslib.Client, accessConfig *clusterconfig.AccessConfig, outputType flags.OutputType) { clusterConfig := refreshCachedClusterConfig(awsClient, accessConfig, outputType == flags.PrettyOutputType) infoInterface := clusterConfig.CoreConfig if outputType == flags.JSONOutputType { outputBytes, err := libjson.Marshal(infoInterface) if err != nil { exit.Error(err) } fmt.Println(string(outputBytes)) } else { outputBytes, err := yaml.Marshal(infoInterface) if err != nil { exit.Error(err) } fmt.Println(string(outputBytes)) } } func cmdInfo(awsClient *awslib.Client, accessConfig *clusterconfig.AccessConfig, stacks clusterstate.ClusterStacks, outputType flags.OutputType, disallowPrompt bool) { clusterConfig := refreshCachedClusterConfig(awsClient, accessConfig, outputType == flags.PrettyOutputType) operatorLoadBalancer, err := getNLBLoadBalancer(accessConfig.ClusterName, OperatorLoadBalancer, awsClient) if err != nil { exit.Error(err) } operatorEndpoint := s.EnsurePrefix(*operatorLoadBalancer.DNSName, "https://") var apiEndpoint string if clusterConfig.APILoadBalancerType == clusterconfig.NLBLoadBalancerType { apiLoadBalancer, err := getNLBLoadBalancer(accessConfig.ClusterName, APILoadBalancer, awsClient) if err != nil { exit.Error(err) } apiEndpoint = *apiLoadBalancer.DNSName } if clusterConfig.APILoadBalancerType == clusterconfig.ELBLoadBalancerType { apiLoadBalancer, err := getELBLoadBalancer(accessConfig.ClusterName, APILoadBalancer, awsClient) if err != nil { exit.Error(err) } apiEndpoint = *apiLoadBalancer.DNSName } if outputType == flags.JSONOutputType || outputType == flags.YAMLOutputType { infoResponse, err := getInfoOperatorResponse(operatorEndpoint) if err != nil { exit.Error(err) } infoResponse.ClusterConfig.Config = clusterConfig infoInterface := map[string]interface{}{ "cluster_config": infoResponse.ClusterConfig.Config, "cluster_metadata": infoResponse.ClusterConfig.OperatorMetadata, "worker_node_infos": infoResponse.WorkerNodeInfos, "operator_node_infos": infoResponse.OperatorNodeInfos, "endpoint_operator": operatorEndpoint, "endpoint_api": apiEndpoint, } var outputBytes []byte if outputType == flags.JSONOutputType { outputBytes, err = libjson.Marshal(infoInterface) } else { outputBytes, err = yaml.Marshal(infoInterface) } if err != nil { exit.Error(err) } fmt.Println(string(outputBytes)) } if outputType == flags.PrettyOutputType { fmt.Println(console.Bold("endpoints:")) fmt.Println("operator: ", operatorEndpoint) fmt.Println("api load balancer:", apiEndpoint) fmt.Println() if err := printInfoOperatorResponse(clusterConfig, stacks, operatorEndpoint); err != nil { exit.Error(err) } } if _flagClusterInfoEnv != "" { if err := updateCLIEnv(_flagClusterInfoEnv, operatorEndpoint, disallowPrompt, outputType == flags.PrettyOutputType); err != nil { exit.Error(err) } } } func printInfoOperatorResponse(clusterConfig clusterconfig.Config, stacks clusterstate.ClusterStacks, operatorEndpoint string) error { fmt.Print("fetching cluster status ...\n\n") fmt.Println(stacks.TableString()) yamlBytes, err := yaml.Marshal(clusterConfig) if err != nil { return err } yamlString := string(yamlBytes) infoResponse, err := getInfoOperatorResponse(operatorEndpoint) if err != nil { fmt.Println(yamlString) return err } infoResponse.ClusterConfig.Config = clusterConfig fmt.Println(console.Bold("cluster config:")) fmt.Println(fmt.Sprintf("cluster version: %s", infoResponse.ClusterConfig.APIVersion)) fmt.Print(yamlString) printInfoPricing(infoResponse, clusterConfig) printInfoNodes(infoResponse) return nil } func getInfoOperatorResponse(operatorEndpoint string) (*schema.InfoResponse, error) { operatorConfig := cluster.OperatorConfig{ Telemetry: isTelemetryEnabled(), ClientID: clientID(), OperatorEndpoint: operatorEndpoint, } return cluster.Info(operatorConfig) } func printInfoPricing(infoResponse *schema.InfoResponse, clusterConfig clusterconfig.Config) { eksPrice := awslib.EKSPrices[clusterConfig.Region] operatorInstancePrice := awslib.InstanceMetadatas[clusterConfig.Region]["t3.medium"].Price operatorEBSPrice := awslib.EBSMetadatas[clusterConfig.Region]["gp3"].PriceGB * 20 / 30 / 24 prometheusInstancePrice := awslib.InstanceMetadatas[clusterConfig.Region][clusterConfig.PrometheusInstanceType].Price prometheusEBSPrice := awslib.EBSMetadatas[clusterConfig.Region]["gp3"].PriceGB * 20 / 30 / 24 metricsEBSPrice := awslib.EBSMetadatas[clusterConfig.Region]["gp2"].PriceGB * (40 + 2) / 30 / 24 nlbPrice := awslib.NLBMetadatas[clusterConfig.Region].Price elbPrice := awslib.ELBMetadatas[clusterConfig.Region].Price natUnitPrice := awslib.NATMetadatas[clusterConfig.Region].Price var loadBalancersPrice float64 usesELBForAPILoadBalancer := clusterConfig.APILoadBalancerType == clusterconfig.ELBLoadBalancerType if usesELBForAPILoadBalancer { loadBalancersPrice = nlbPrice + elbPrice } else { loadBalancersPrice = 2 * nlbPrice } headers := []table.Header{ {Title: "aws resource"}, {Title: "cost per hour"}, } var rows [][]interface{} rows = append(rows, []interface{}{"1 eks cluster", s.DollarsMaxPrecision(eksPrice)}) var totalNodeGroupsPrice float64 for _, ng := range clusterConfig.NodeGroups { var ngNamePrefix string if ng.Spot { ngNamePrefix = "cx-ws-" } else { ngNamePrefix = "cx-wd-" } nodesInfo := infoResponse.GetNodesWithNodeGroupName(ngNamePrefix + ng.Name) numInstances := len(nodesInfo) ebsPrice := awslib.EBSMetadatas[clusterConfig.Region][ng.InstanceVolumeType.String()].PriceGB * float64(ng.InstanceVolumeSize) / 30 / 24 if ng.InstanceVolumeType == clusterconfig.IO1VolumeType && ng.InstanceVolumeIOPS != nil { ebsPrice += awslib.EBSMetadatas[clusterConfig.Region][ng.InstanceVolumeType.String()].PriceIOPS * float64(*ng.InstanceVolumeIOPS) / 30 / 24 } if ng.InstanceVolumeType == clusterconfig.GP3VolumeType && ng.InstanceVolumeIOPS != nil && ng.InstanceVolumeThroughput != nil { ebsPrice += libmath.MaxFloat64(0, (awslib.EBSMetadatas[clusterConfig.Region][ng.InstanceVolumeType.String()].PriceIOPS-3000)*float64(*ng.InstanceVolumeIOPS)/30/24) ebsPrice += libmath.MaxFloat64(0, (awslib.EBSMetadatas[clusterConfig.Region][ng.InstanceVolumeType.String()].PriceThroughput-125)*float64(*ng.InstanceVolumeThroughput)/30/24) } totalEBSPrice := ebsPrice * float64(numInstances) totalInstancePrice := float64(0) for _, nodeInfo := range nodesInfo { totalInstancePrice += nodeInfo.Price } rows = append(rows, []interface{}{fmt.Sprintf("nodegroup %s: %d (out of %d) %s", ng.Name, numInstances, ng.MaxInstances, s.PluralS("instance", numInstances)), s.DollarsAndTenthsOfCents(totalInstancePrice+totalEBSPrice) + " total"}) totalNodeGroupsPrice += totalEBSPrice + totalInstancePrice } operatorNodeGroupPrice := float64(len(infoResponse.OperatorNodeInfos)) * (operatorInstancePrice + operatorEBSPrice) prometheusNodeGroupPrice := prometheusInstancePrice + prometheusEBSPrice + metricsEBSPrice var natTotalPrice float64 if clusterConfig.NATGateway == clusterconfig.SingleNATGateway { natTotalPrice = natUnitPrice } else if clusterConfig.NATGateway == clusterconfig.HighlyAvailableNATGateway { natTotalPrice = natUnitPrice * float64(len(clusterConfig.AvailabilityZones)) } totalPrice := eksPrice + totalNodeGroupsPrice + operatorNodeGroupPrice + prometheusNodeGroupPrice + loadBalancersPrice + natTotalPrice fmt.Printf(console.Bold("\nyour cluster currently costs %s per hour\n\n"), s.DollarsAndCents(totalPrice)) rows = append(rows, []interface{}{fmt.Sprintf("%d t3.medium %s (cortex system)", len(infoResponse.OperatorNodeInfos), s.PluralS("instance", len(infoResponse.OperatorNodeInfos))), s.DollarsAndTenthsOfCents(operatorNodeGroupPrice) + " total"}) rows = append(rows, []interface{}{fmt.Sprintf("1 %s instance (prometheus)", clusterConfig.PrometheusInstanceType), s.DollarsAndTenthsOfCents(prometheusNodeGroupPrice)}) if usesELBForAPILoadBalancer { rows = append(rows, []interface{}{"1 network load balancer", s.DollarsMaxPrecision(nlbPrice)}) rows = append(rows, []interface{}{"1 classic load balancer", s.DollarsMaxPrecision(elbPrice)}) } else { rows = append(rows, []interface{}{"2 network load balancers", s.DollarsMaxPrecision(loadBalancersPrice) + " total"}) } if clusterConfig.NATGateway == clusterconfig.SingleNATGateway { rows = append(rows, []interface{}{"1 nat gateway", s.DollarsMaxPrecision(natUnitPrice)}) } else if clusterConfig.NATGateway == clusterconfig.HighlyAvailableNATGateway { numNATs := len(clusterConfig.AvailabilityZones) rows = append(rows, []interface{}{fmt.Sprintf("%d nat gateways", numNATs), s.DollarsMaxPrecision(natUnitPrice*float64(numNATs)) + " total"}) } t := table.Table{ Headers: headers, Rows: rows, } t.MustPrint(&table.Opts{Sort: pointer.Bool(false)}) } func printInfoNodes(infoResponse *schema.InfoResponse) { numAPIInstances := len(infoResponse.WorkerNodeInfos) var totalReplicas int var doesClusterHaveGPUs, doesClusterHaveInfs, doesClusterHaveEnqueuers bool for _, nodeInfo := range infoResponse.WorkerNodeInfos { totalReplicas += nodeInfo.NumReplicas if nodeInfo.ComputeUserCapacity.GPU > 0 { doesClusterHaveGPUs = true } if nodeInfo.ComputeUserCapacity.Inf > 0 { doesClusterHaveInfs = true } if nodeInfo.NumEnqueuerReplicas > 0 { doesClusterHaveEnqueuers = true } } var pendingReplicasStr string if infoResponse.NumPendingReplicas > 0 { pendingReplicasStr = fmt.Sprintf(", and %d unscheduled %s", infoResponse.NumPendingReplicas, s.PluralS("replica", infoResponse.NumPendingReplicas)) } fmt.Printf(console.Bold("\nyour cluster has %d API %s running across %d %s%s\n"), totalReplicas, s.PluralS("replica", totalReplicas), numAPIInstances, s.PluralS("instance", numAPIInstances), pendingReplicasStr) if len(infoResponse.WorkerNodeInfos) == 0 { return } headers := []table.Header{ {Title: "instance type"}, {Title: "lifecycle"}, {Title: "replicas"}, {Title: "batch enqueuer replicas", Hidden: !doesClusterHaveEnqueuers}, {Title: "CPU (requested / total allocatable)"}, {Title: "memory (requested / total allocatable)"}, {Title: "GPU (requested / total allocatable)", Hidden: !doesClusterHaveGPUs}, {Title: "Inf (requested / total allocatable)", Hidden: !doesClusterHaveInfs}, } var rows [][]interface{} for _, nodeInfo := range infoResponse.WorkerNodeInfos { lifecycle := "on-demand" if nodeInfo.IsSpot { lifecycle = "spot" } cpuStr := nodeInfo.ComputeUserRequested.CPU.MilliString() + " / " + nodeInfo.ComputeUserCapacity.CPU.MilliString() memStr := nodeInfo.ComputeUserRequested.Mem.String() + " / " + nodeInfo.ComputeUserCapacity.Mem.String() gpuStr := s.Int64(nodeInfo.ComputeUserRequested.GPU) + " / " + s.Int64(nodeInfo.ComputeUserCapacity.GPU) infStr := s.Int64(nodeInfo.ComputeUserRequested.Inf) + " / " + s.Int64(nodeInfo.ComputeUserCapacity.Inf) rows = append(rows, []interface{}{nodeInfo.InstanceType, lifecycle, nodeInfo.NumReplicas, nodeInfo.NumEnqueuerReplicas, cpuStr, memStr, gpuStr, infStr}) } t := table.Table{ Headers: headers, Rows: rows, } fmt.Println() t.MustPrint(&table.Opts{Sort: pointer.Bool(false)}) } func updateCLIEnv(envName string, operatorEndpoint string, disallowPrompt bool, printToStdout bool) error { prevEnv, err := readEnv(envName) if err != nil { return err } newEnvironment := cliconfig.Environment{ Name: envName, OperatorEndpoint: operatorEndpoint, } shouldWriteEnv := false envWasUpdated := false if prevEnv == nil { shouldWriteEnv = true if printToStdout { fmt.Println() } } else if prevEnv.OperatorEndpoint != operatorEndpoint { envWasUpdated = true if printToStdout { if disallowPrompt { shouldWriteEnv = true fmt.Println() } else { shouldWriteEnv = prompt.YesOrNo(fmt.Sprintf("\nfound an existing environment named \"%s\"; would you like to overwrite it to connect to this cluster?", envName), "", "") } } else { shouldWriteEnv = true } } if shouldWriteEnv { err := addEnvToCLIConfig(newEnvironment, true) if err != nil { return err } if printToStdout { if envWasUpdated { fmt.Printf(console.Bold("the environment named \"%s\" has been updated to point to this cluster (and was set as the default environment)\n"), envName) } else { fmt.Printf(console.Bold("an environment named \"%s\" has been configured to point to this cluster (and was set as the default environment)\n"), envName) } } } return nil } func cmdDebug(awsClient *awslib.Client, accessConfig *clusterconfig.AccessConfig) { // note: if modifying this string, also change it in files.IgnoreCortexDebug() debugFileName := fmt.Sprintf("cortex-debug-%s.tgz", time.Now().UTC().Format("2006-01-02-15-04-05")) containerDebugPath := "/out/" + debugFileName copyFromPaths := []dockerCopyFromPath{ { containerPath: containerDebugPath, localDir: _cwd, }, } out, exitCode, err := runManagerAccessCommand("/root/debug.sh "+containerDebugPath, *accessConfig, awsClient, nil, copyFromPaths) if err != nil { exit.Error(err) } if exitCode == nil || *exitCode != 0 { exit.Error(ErrorClusterDebug(out)) } fmt.Println("saved cluster info to ./" + debugFileName) return } func refreshCachedClusterConfig(awsClient *awslib.Client, accessConfig *clusterconfig.AccessConfig, printToStdout bool) clusterconfig.Config { // add empty file if cached cluster doesn't exist so that the file output by manager container maintains current user permissions cachedClusterConfigPath := getCachedClusterConfigPath(accessConfig.ClusterName, accessConfig.Region) containerConfigPath := fmt.Sprintf("/out/%s", filepath.Base(cachedClusterConfigPath)) copyFromPaths := []dockerCopyFromPath{ { containerPath: containerConfigPath, localDir: files.Dir(cachedClusterConfigPath), }, } if printToStdout { fmt.Print("syncing cluster configuration ...\n\n") } out, exitCode, err := runManagerAccessCommand("/root/refresh.sh "+containerConfigPath, *accessConfig, awsClient, nil, copyFromPaths) if err != nil { exit.Error(err) } if exitCode == nil || *exitCode != 0 { exit.Error(ErrorClusterRefresh(out)) } refreshedClusterConfig := &clusterconfig.Config{} err = readCachedClusterConfigFile(refreshedClusterConfig, cachedClusterConfigPath) if err != nil { exit.Error(err) } return *refreshedClusterConfig } func createS3BucketIfNotFound(awsClient *awslib.Client, bucket string, tags map[string]string) error { bucketFound, err := awsClient.DoesBucketExist(bucket) if err != nil { return err } if !bucketFound { fmt.Print("￮ creating a new s3 bucket: ", bucket) err = awsClient.CreateBucket(bucket) if err != nil { fmt.Print("\n\n") return err } err = awsClient.EnableBucketEncryption(bucket) if err != nil { fmt.Print("\n\n") return err } } else { fmt.Print("￮ using existing s3 bucket: ", bucket) } // retry since it's possible that it takes some time for the new bucket to be registered by AWS for i := 0; i < 10; i++ { err = awsClient.TagBucket(bucket, tags) if err == nil { fmt.Println(" ✓") return nil } if !awslib.IsNoSuchBucketErr(err) { break } time.Sleep(1 * time.Second) } fmt.Print("\n\n") return err } func setLifecycleRulesOnClusterUp(awsClient *awslib.Client, bucket, newClusterUID string) error { err := awsClient.DeleteLifecycleRules(bucket) if err != nil { return err } clusterUIDs, err := awsClient.ListS3TopLevelDirs(bucket) if err != nil { return err } if len(clusterUIDs)+1 > consts.MaxBucketLifecycleRules { return ErrorClusterUIDsLimitInBucket(bucket) } expirationDate := libtime.GetCurrentUTCDate().Add(-24 * time.Hour) rules := []s3.LifecycleRule{} for _, clusterUID := range clusterUIDs { rules = append(rules, s3.LifecycleRule{ Expiration: &s3.LifecycleExpiration{ Date: &expirationDate, }, ID: pointer.String("cluster-remove-" + clusterUID), Filter: &s3.LifecycleRuleFilter{ Prefix: pointer.String(s.EnsureSuffix(clusterUID, "/")), }, Status: pointer.String("Enabled"), }) } rules = append(rules, s3.LifecycleRule{ Expiration: &s3.LifecycleExpiration{ Days: pointer.Int64(consts.AsyncWorkloadsExpirationDays), }, ID: pointer.String("async-workloads-expiry-policy"), Filter: &s3.LifecycleRuleFilter{ Prefix: pointer.String(s.EnsureSuffix(filepath.Join(newClusterUID, "workloads"), "/")), }, Status: pointer.String("Enabled"), }) return awsClient.SetLifecycleRules(bucket, rules) } func setLifecycleRulesOnClusterDown(awsClient *awslib.Client, bucket string) error { err := awsClient.DeleteLifecycleRules(bucket) if err != nil { return err } expirationDate := libtime.GetCurrentUTCDate().Add(-24 * time.Hour) return awsClient.SetLifecycleRules(bucket, []s3.LifecycleRule{ { Expiration: &s3.LifecycleExpiration{ Date: &expirationDate, }, ID: pointer.String("bucket-cleaner"), Filter: &s3.LifecycleRuleFilter{ Prefix: pointer.String(""), }, Status: pointer.String("Enabled"), }, }) } func createLogGroupIfNotFound(awsClient *awslib.Client, logGroup string, tags map[string]string) error { logGroupFound, err := awsClient.DoesLogGroupExist(logGroup) if err != nil { return err } if !logGroupFound { fmt.Print("￮ creating a new cloudwatch log group: ", logGroup) err = awsClient.CreateLogGroup(logGroup, tags) if err != nil { fmt.Print("\n\n") return err } fmt.Println(" ✓") return nil } fmt.Print("￮ using existing cloudwatch log group: ", logGroup) // retry since it's possible that it takes some time for the new log group to be registered by AWS err = awsClient.TagLogGroup(logGroup, tags) if err != nil { fmt.Print("\n\n") return err } fmt.Println(" ✓") return nil } type LoadBalancer string var ( OperatorLoadBalancer LoadBalancer = "operator" APILoadBalancer LoadBalancer = "api" ) func (lb LoadBalancer) String() string { return string(lb) } // Will return error if the load balancer can't be found func getNLBLoadBalancer(clusterName string, whichLB LoadBalancer, awsClient *awslib.Client) (*elbv2.LoadBalancer, error) { loadBalancer, err := awsClient.FindLoadBalancerV2(map[string]string{ clusterconfig.ClusterNameTag: clusterName, "cortex.dev/load-balancer": whichLB.String(), }) if err != nil { return nil, errors.Wrap(err, fmt.Sprintf("unable to locate %s load balancer", whichLB.String())) } if loadBalancer == nil { return nil, ErrorNoOperatorLoadBalancer(whichLB.String()) } return loadBalancer, nil } // Will return error if the load balancer can't be found func getELBLoadBalancer(clusterName string, whichLB LoadBalancer, awsClient *awslib.Client) (*elb.LoadBalancerDescription, error) { loadBalancer, err := awsClient.FindLoadBalancer(map[string]string{ clusterconfig.ClusterNameTag: clusterName, "cortex.dev/load-balancer": whichLB.String(), }) if err != nil { return nil, errors.Wrap(err, fmt.Sprintf("unable to locate %s load balancer", whichLB.String())) } if loadBalancer == nil { return nil, ErrorNoOperatorLoadBalancer(whichLB.String()) } return loadBalancer, nil } func listPVCVolumesForCluster(awsClient *awslib.Client, clusterName string) ([]ec2.Volume, error) { return awsClient.ListVolumes(ec2.Tag{ Key: pointer.String(fmt.Sprintf("kubernetes.io/cluster/%s", clusterName)), Value: nil, // any value should be ok as long as the key is present }) } func filterEKSCTLOutput(out string) string { return strings.Join(s.RemoveDuplicates(strings.Split(out, "\n"), _eksctlPrefixRegex), "\n") } func getClusterRESTConfig(awsClient *awslib.Client, clusterName string) (*rest.Config, error) { clusterOutput, err := awsClient.EKS().DescribeCluster( &eks.DescribeClusterInput{ Name: aws.String(clusterName), }, ) if err != nil { return nil, err } gen, err := token.NewGenerator(true, false) if err != nil { return nil, err } opts := &token.GetTokenOptions{ ClusterID: aws.StringValue(clusterOutput.Cluster.Name), } tok, err := gen.GetWithOptions(opts) if err != nil { return nil, err } ca, err := base64.StdEncoding.DecodeString(aws.StringValue(clusterOutput.Cluster.CertificateAuthority.Data)) if err != nil { return nil, err } return &rest.Config{ Host: aws.StringValue(clusterOutput.Cluster.Endpoint), BearerToken: tok.Token, TLSClientConfig: rest.TLSClientConfig{ CAData: ca, }, }, nil } ================================================ FILE: cli/cmd/completion.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package cmd import ( "fmt" "os" "github.com/cortexlabs/cortex/pkg/lib/exit" "github.com/spf13/cobra" ) func completionInit() { _completionCmd.Flags().SortFlags = false } var _bashAliasText = ` # alias alias cx='cortex' if [[ $(type -t compopt) = "builtin" ]]; then complete -o default -F __start_cortex cx else complete -o default -o nospace -F __start_cortex cx fi ` var _completionCmd = &cobra.Command{ Use: "completion SHELL", Short: "generate shell completion scripts", Long: `generate shell completion scripts to enable cortex shell completion: bash: add this to ~/.bash_profile (mac) or ~/.bashrc (linux): source <(cortex completion bash) note: bash-completion must be installed on your system; example installation instructions: mac: 1) install bash completion: brew install bash-completion 2) add this to your ~/.bash_profile: source $(brew --prefix)/etc/bash_completion 3) log out and back in, or close your terminal window and reopen it ubuntu: 1) install bash completion: apt update && apt install -y bash-completion # you may need sudo 2) open ~/.bashrc and uncomment the bash completion section, or add this: if [ -f /etc/bash_completion ] && ! shopt -oq posix; then . /etc/bash_completion; fi 3) log out and back in, or close your terminal window and reopen it zsh: option 1: add this to ~/.zshrc: source <(cortex completion zsh) if that failed, you can try adding this line (above the source command you just added): autoload -Uz compinit && compinit option 2: create a _cortex file in your fpath, for example: cortex completion zsh > /usr/local/share/zsh/site-functions/_cortex Note: this will also add the "cx" alias for cortex for convenience `, Args: cobra.ExactArgs(1), ValidArgs: []string{"bash", "zsh"}, Run: func(cmd *cobra.Command, args []string) { switch args[0] { case "bash": _rootCmd.GenBashCompletion(os.Stdout) fmt.Print(_bashAliasText) case "zsh": _rootCmd.GenZshCompletion(os.Stdout) fmt.Print("alias cx='cortex'\n\n") // https://github.com/spf13/cobra/pull/887 // https://github.com/corneliusweig/rakkess/blob/master/cmd/completion.go // https://github.com/GoogleContainerTools/skaffold/blob/master/cmd/skaffold/app/cmd/completion.go // https://github.com/spf13/cobra/issues/881 // https://github.com/asdf-vm/asdf/issues/266 fmt.Println("if compquote '' 2>/dev/null; then _cortex; else compdef _cortex cortex; fi") default: fmt.Println() exit.Error(ErrorShellCompletionNotSupported(args[0])) } }, } ================================================ FILE: cli/cmd/const.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package cmd const ( _timeFormat = "02 Jan 06 15:04:05 MST" ) ================================================ FILE: cli/cmd/delete.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package cmd import ( "fmt" "strings" "github.com/cortexlabs/cortex/cli/cluster" "github.com/cortexlabs/cortex/cli/types/flags" "github.com/cortexlabs/cortex/pkg/lib/exit" libjson "github.com/cortexlabs/cortex/pkg/lib/json" "github.com/cortexlabs/cortex/pkg/lib/print" "github.com/cortexlabs/cortex/pkg/lib/telemetry" "github.com/cortexlabs/cortex/pkg/operator/schema" "github.com/spf13/cobra" ) var ( _flagDeleteEnv string _flagDeleteKeepCache bool _flagDeleteForce bool ) func deleteInit() { _deleteCmd.Flags().SortFlags = false _deleteCmd.Flags().StringVarP(&_flagDeleteEnv, "env", "e", "", "environment to use") _deleteCmd.Flags().BoolVarP(&_flagDeleteForce, "force", "f", false, "delete the api without confirmation") _deleteCmd.Flags().BoolVarP(&_flagDeleteKeepCache, "keep-cache", "c", false, "keep cached data for the api") _deleteCmd.Flags().VarP(&_flagOutput, "output", "o", fmt.Sprintf("output format: one of %s", strings.Join(flags.OutputTypeStringsExcluding(flags.YAMLOutputType), "|"))) } var _deleteCmd = &cobra.Command{ Use: "delete API_NAME [JOB_ID]", Short: "delete an api or stop a job", Args: cobra.RangeArgs(1, 2), Run: func(cmd *cobra.Command, args []string) { envName, err := getEnvFromFlag(_flagDeleteEnv) if err != nil { telemetry.Event("cli.delete") exit.Error(err) } env, err := ReadOrConfigureEnv(envName) if err != nil { telemetry.Event("cli.delete") exit.Error(err) } telemetry.Event("cli.delete", map[string]interface{}{"env_name": env.Name}) err = printEnvIfNotSpecified(env.Name, cmd) if err != nil { exit.Error(err) } var deleteResponse schema.DeleteResponse if len(args) == 2 { apisRes, err := cluster.GetAPI(MustGetOperatorConfig(env.Name), args[0]) if err != nil { exit.Error(err) } deleteResponse, err = cluster.StopJob(MustGetOperatorConfig(env.Name), apisRes[0].Spec.Kind, args[0], args[1]) if err != nil { exit.Error(err) } } else { deleteResponse, err = cluster.Delete(MustGetOperatorConfig(env.Name), args[0], _flagDeleteKeepCache, _flagDeleteForce) if err != nil { exit.Error(err) } } if _flagOutput == flags.JSONOutputType { bytes, err := libjson.Marshal(deleteResponse) if err != nil { exit.Error(err) } fmt.Print(string(bytes)) return } print.BoldFirstLine(deleteResponse.Message) }, } ================================================ FILE: cli/cmd/deploy.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package cmd import ( "fmt" "strings" "github.com/cortexlabs/cortex/cli/cluster" "github.com/cortexlabs/cortex/cli/types/flags" "github.com/cortexlabs/cortex/pkg/lib/exit" "github.com/cortexlabs/cortex/pkg/lib/files" libjson "github.com/cortexlabs/cortex/pkg/lib/json" "github.com/cortexlabs/cortex/pkg/lib/print" "github.com/cortexlabs/cortex/pkg/lib/telemetry" "github.com/cortexlabs/cortex/pkg/operator/schema" "github.com/spf13/cobra" ) var ( _warningFileBytes = 1024 * 1024 * 10 _warningProjectBytes = 1024 * 1024 * 10 _warningFileCount = 1000 _maxFileSizeBytes int64 = 1024 * 1024 * 32 // 32mb _maxProjectSizeBytes int64 = 1024 * 1024 * 32 // 32mb _flagDeployEnv string _flagDeployForce bool _flagDeployDisallowPrompt bool ) func deployInit() { _deployCmd.Flags().SortFlags = false _deployCmd.Flags().StringVarP(&_flagDeployEnv, "env", "e", "", "environment to use") _deployCmd.Flags().BoolVarP(&_flagDeployForce, "force", "f", false, "override the in-progress api update") _deployCmd.Flags().BoolVarP(&_flagDeployDisallowPrompt, "yes", "y", false, "skip prompts") _deployCmd.Flags().VarP(&_flagOutput, "output", "o", fmt.Sprintf("output format: one of %s", strings.Join(flags.OutputTypeStringsExcluding(flags.YAMLOutputType), "|"))) } var _deployCmd = &cobra.Command{ Use: "deploy [CONFIG_FILE]", Short: "create or update apis", Args: cobra.RangeArgs(0, 1), Run: func(cmd *cobra.Command, args []string) { envName, err := getEnvFromFlag(_flagDeployEnv) if err != nil { telemetry.Event("cli.deploy") exit.Error(err) } env, err := ReadOrConfigureEnv(envName) if err != nil { telemetry.Event("cli.deploy") exit.Error(err) } telemetry.Event("cli.deploy", map[string]interface{}{"env_name": env.Name}) err = printEnvIfNotSpecified(env.Name, cmd) if err != nil { exit.Error(err) } configPath := getConfigPath(args) projectRoot := files.Dir(configPath) if projectRoot == _homeDir { exit.Error(ErrorDeployFromTopLevelDir("home")) } if projectRoot == "/" { exit.Error(ErrorDeployFromTopLevelDir("root")) } deploymentBytes, err := getDeploymentBytes(configPath) if err != nil { exit.Error(err) } deployResults, err := cluster.Deploy(MustGetOperatorConfig(env.Name), configPath, deploymentBytes, _flagDeployForce) if err != nil { exit.Error(err) } switch _flagOutput { case flags.JSONOutputType: bytes, err := libjson.Marshal(deployResults) if err != nil { exit.Error(err) } fmt.Print(string(bytes)) case flags.PrettyOutputType: message := mergeResultMessages(deployResults) if didAnyResultsError(deployResults) { print.StderrBoldFirstBlock(message) } else { print.BoldFirstBlock(message) } } if didAnyResultsError(deployResults) { exit.Error(nil) } }, } // Returns absolute path func getConfigPath(args []string) string { var configPath string if len(args) == 0 { configPath = "cortex.yaml" if !files.IsFile(configPath) { exit.Error(ErrorCortexYAMLNotFound()) } } else { configPath = args[0] if err := files.CheckFile(configPath); err != nil { exit.Error(err) } } return files.RelToAbsPath(configPath, _cwd) } func getDeploymentBytes(configPath string) (map[string][]byte, error) { configBytes, err := files.ReadFileBytes(configPath) if err != nil { return nil, err } uploadBytes := map[string][]byte{ "config": configBytes, } return uploadBytes, nil } func mergeResultMessages(results []schema.DeployResult) string { var okMessages []string var errMessages []string for _, result := range results { if result.Error != "" { errMessages = append(errMessages, result.Error) } else { okMessages = append(okMessages, result.Message) } } output := "" if len(okMessages) > 0 { output += strings.Join(okMessages, "\n") if len(errMessages) > 0 { output += "\n\n" } } if len(errMessages) > 0 { output += strings.Join(errMessages, "\n") } return output } func didAllResultsError(results []schema.DeployResult) bool { for _, result := range results { if result.Error == "" { return false } } return true } func didAnyResultsError(results []schema.DeployResult) bool { for _, result := range results { if result.Error != "" { return true } } return false } ================================================ FILE: cli/cmd/describe.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package cmd import ( "fmt" "github.com/cortexlabs/cortex/cli/cluster" "github.com/cortexlabs/cortex/cli/types/cliconfig" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/exit" "github.com/cortexlabs/cortex/pkg/lib/telemetry" "github.com/cortexlabs/cortex/pkg/types/userconfig" "github.com/spf13/cobra" ) const ( _titleReplicaStatus = "replica status" _titleReplicaCount = "replica count" ) var ( _flagDescribeEnv string _flagDescribeWatch bool ) func describeInit() { _describeCmd.Flags().SortFlags = false _describeCmd.Flags().StringVarP(&_flagDescribeEnv, "env", "e", "", "environment to use") _describeCmd.Flags().BoolVarP(&_flagDescribeWatch, "watch", "w", false, "re-run the command every 2 seconds") } var _describeCmd = &cobra.Command{ Use: "describe [API_NAME]", Short: "describe an api", Args: cobra.ExactArgs(1), Run: func(cmd *cobra.Command, args []string) { apiName := args[0] var envName string if wasFlagProvided(cmd, "env") { envName = _flagDescribeEnv } else { var err error envName, err = getEnvFromFlag("") if err != nil { telemetry.Event("cli.describe") exit.Error(err) } } env, err := ReadOrConfigureEnv(envName) if err != nil { telemetry.Event("cli.describe") exit.Error(err) } telemetry.Event("cli.describe", map[string]interface{}{"env_name": env.Name}) rerun(_flagDescribeWatch, func() (string, error) { env, err := ReadOrConfigureEnv(envName) if err != nil { exit.Error(err) } out, err := envStringIfNotSpecified(envName, cmd) if err != nil { return "", err } apiTable, err := describeAPI(env, apiName) if err != nil { return "", err } return out + apiTable, nil }) }, } func describeAPI(env cliconfig.Environment, apiName string) (string, error) { apisRes, err := cluster.DescribeAPI(MustGetOperatorConfig(env.Name), apiName) if err != nil { return "", err } if len(apisRes) == 0 { exit.Error(errors.ErrorUnexpected(fmt.Sprintf("unable to find api %s", apiName))) } apiRes := apisRes[0] switch apiRes.Metadata.Kind { case userconfig.RealtimeAPIKind: return realtimeDescribeAPITable(apiRes, env) case userconfig.AsyncAPIKind: return asyncDescribeAPITable(apiRes, env) default: return "", errors.ErrorUnexpected(fmt.Sprintf("encountered unexpected kind %s for api %s", apiRes.Metadata.Kind, apiRes.Metadata.Name)) } } ================================================ FILE: cli/cmd/env.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package cmd import ( "fmt" "strings" "github.com/cortexlabs/cortex/cli/types/cliconfig" "github.com/cortexlabs/cortex/cli/types/flags" "github.com/cortexlabs/cortex/pkg/lib/exit" libjson "github.com/cortexlabs/cortex/pkg/lib/json" "github.com/cortexlabs/cortex/pkg/lib/print" "github.com/cortexlabs/cortex/pkg/lib/telemetry" "github.com/spf13/cobra" ) var ( _flagEnvOperatorEndpoint string ) func envInit() { _envConfigureCmd.Flags().SortFlags = false _envConfigureCmd.Flags().StringVarP(&_flagEnvOperatorEndpoint, "operator-endpoint", "o", "", "set the operator endpoint without prompting") _envCmd.AddCommand(_envConfigureCmd) _envListCmd.Flags().SortFlags = false _envListCmd.Flags().VarP(&_flagOutput, "output", "o", fmt.Sprintf("output format: one of %s", strings.Join(flags.OutputTypeStringsExcluding(flags.YAMLOutputType), "|"))) _envCmd.AddCommand(_envListCmd) _envDefaultCmd.Flags().SortFlags = false _envCmd.AddCommand(_envDefaultCmd) _envRenameCmd.Flags().SortFlags = false _envCmd.AddCommand(_envRenameCmd) _envDeleteCmd.Flags().SortFlags = false _envCmd.AddCommand(_envDeleteCmd) } var _envCmd = &cobra.Command{ Use: "env", Short: "manage cli environments (contains subcommands)", } var _envConfigureCmd = &cobra.Command{ Use: "configure [ENVIRONMENT_NAME]", Short: "configure an environment", Args: cobra.RangeArgs(0, 1), Run: func(cmd *cobra.Command, args []string) { telemetry.Event("cli.env.configure") var envName string if len(args) == 1 { envName = args[0] } fieldsToSkipPrompt := cliconfig.Environment{} if _flagEnvOperatorEndpoint != "" { operatorEndpoint, err := validateOperatorEndpoint(_flagEnvOperatorEndpoint) if err != nil { exit.Error(err) } fieldsToSkipPrompt.OperatorEndpoint = operatorEndpoint } if _, err := configureEnv(envName, fieldsToSkipPrompt); err != nil { exit.Error(err) } }, } var _envListCmd = &cobra.Command{ Use: "list", Short: "list all configured environments", Args: cobra.NoArgs, Run: func(cmd *cobra.Command, args []string) { telemetry.Event("cli.env.list") cliConfig, err := readCLIConfig() if err != nil { exit.Error(err) } if _flagOutput == flags.JSONOutputType { bytes, err := libjson.Marshal(cliConfig.ConvertToUserFacingCLIConfig()) if err != nil { exit.Error(err) } fmt.Print(string(bytes)) return } if len(cliConfig.Environments) == 0 { fmt.Println("no environments are configured") return } defaultEnv, err := getDefaultEnv() if err != nil { exit.Error(err) } for i, env := range cliConfig.Environments { fmt.Print(env.String(defaultEnv != nil && *defaultEnv == env.Name)) if i+1 < len(cliConfig.Environments) { fmt.Println() } } }, } var _envDefaultCmd = &cobra.Command{ Use: "default [ENVIRONMENT_NAME]", Short: "set the default environment", Args: cobra.RangeArgs(0, 1), Run: func(cmd *cobra.Command, args []string) { telemetry.Event("cli.env.default") defaultEnv, err := getDefaultEnv() if err != nil { exit.Error(err) } var envName string if len(args) == 0 { if defaultEnv != nil { fmt.Printf("current default environment: %s\n\n", *defaultEnv) } envName = promptForExistingEnvName("name of environment to set as default") } else { envName = args[0] } if defaultEnv != nil && *defaultEnv == envName { print.BoldFirstLine(fmt.Sprintf("%s is already the default environment", envName)) exit.Ok() } if err := setDefaultEnv(envName); err != nil { exit.Error(err) } print.BoldFirstLine(fmt.Sprintf("set %s as the default environment", envName)) }, } var _envRenameCmd = &cobra.Command{ Use: "rename EXISTING_NAME NEW_NAME", Short: "rename an environment", Args: cobra.ExactArgs(2), Run: func(cmd *cobra.Command, args []string) { telemetry.Event("cli.env.rename") oldEnvName := args[0] newEnvName := args[1] if err := renameEnv(oldEnvName, newEnvName); err != nil { exit.Error(err) } print.BoldFirstLine(fmt.Sprintf("renamed the %s environment to %s", oldEnvName, newEnvName)) }, } var _envDeleteCmd = &cobra.Command{ Use: "delete [ENVIRONMENT_NAME]", Short: "delete an environment configuration", Args: cobra.RangeArgs(0, 1), Run: func(cmd *cobra.Command, args []string) { telemetry.Event("cli.env.delete") var envName string if len(args) == 1 { envName = args[0] } else { envName = promptForExistingEnvName("name of environment to delete") } prevDefault, err := getDefaultEnv() if err != nil { exit.Error(err) } if err := removeEnvFromCLIConfig(envName); err != nil { exit.Error(err) } newDefault, err := getDefaultEnv() if err != nil { exit.Error(err) } print.BoldFirstLine(fmt.Sprintf("deleted the %s environment configuration", envName)) if prevDefault != nil && newDefault == nil { print.BoldFirstLine("unset the default environment") } else if newDefault != nil && (prevDefault == nil || *prevDefault != *newDefault) { print.BoldFirstLine(fmt.Sprintf("set the default environment to %s", *newDefault)) } }, } ================================================ FILE: cli/cmd/errors.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package cmd import ( "fmt" "net/url" "strings" "github.com/cortexlabs/cortex/pkg/consts" "github.com/cortexlabs/cortex/pkg/lib/errors" s "github.com/cortexlabs/cortex/pkg/lib/strings" "github.com/cortexlabs/cortex/pkg/lib/urls" "github.com/cortexlabs/cortex/pkg/types/clusterconfig" ) const ( _errStrCantMakeRequest = "unable to make request" _errStrRead = "unable to read" ) func errStrFailedToConnect(u url.URL) string { return "failed to connect to " + urls.TrimQueryParamsURL(u) } const ( ErrInvalidProvider = "cli.invalid_provider" ErrInvalidLegacyProvider = "cli.invalid_legacy_provider" ErrNoAvailableEnvironment = "cli.no_available_environment" ErrEnvironmentNotSet = "cli.environment_not_set" ErrEnvironmentNotFound = "cli.environment_not_found" ErrFieldNotFoundInEnvironment = "cli.field_not_found_in_environment" ErrInvalidOperatorEndpoint = "cli.invalid_operator_endpoint" ErrNoOperatorLoadBalancer = "cli.no_operator_load_balancer" ErrCortexYAMLNotFound = "cli.cortex_yaml_not_found" ErrDockerCtrlC = "cli.docker_ctrl_c" ErrResponseUnknown = "cli.response_unknown" ErrMissingAWSCredentials = "cli.missing_aws_credentials" ErrCredentialsInClusterConfig = "cli.credentials_in_cluster_config" ErrClusterUp = "cli.cluster_up" ErrClusterConfigure = "cli.cluster_configure" ErrClusterDebug = "cli.cluster_debug" ErrClusterRefresh = "cli.cluster_refresh" ErrClusterDown = "cli.cluster_down" ErrSpecifyAtLeastOneFlag = "cli.specify_at_least_one_flag" ErrMinInstancesLowerThan = "cli.min_instances_lower_than" ErrMaxInstancesLowerThan = "cli.max_instances_lower_than" ErrMinInstancesGreaterThanMaxInstances = "cli.min_instances_greater_than_max_instances" ErrNodeGroupNotFound = "cli.nodegroup_not_found" ErrMutuallyExclusiveFlags = "cli.mutually_exclusive_flags" ErrClusterAccessConfigRequired = "cli.cluster_access_config_or_prompts_required" ErrShellCompletionNotSupported = "cli.shell_completion_not_supported" ErrNoTerminalWidth = "cli.no_terminal_width" ErrDeployFromTopLevelDir = "cli.deploy_from_top_level_dir" ErrAPINameMustBeProvided = "cli.api_name_must_be_provided" ErrAPINotFoundInConfig = "cli.api_not_found_in_config" ErrClusterUIDsLimitInBucket = "cli.cluster_uids_limit_in_bucket" ) func ErrorInvalidProvider(providerStr, cliConfigPath string) error { return errors.WithStack(&errors.Error{ Kind: ErrInvalidProvider, Message: fmt.Sprintf("\"%s\" is not a supported provider (only aws is supported); remove the environment(s) which use the %s provider from %s, or delete %s (it will be recreated on subsequent CLI commands)", providerStr, providerStr, cliConfigPath, cliConfigPath), }) } func ErrorInvalidLegacyProvider(providerStr, cliConfigPath string) error { return errors.WithStack(&errors.Error{ Kind: ErrInvalidLegacyProvider, Message: fmt.Sprintf("the %s provider is no longer supported on cortex v%s; remove the environment(s) which use the %s provider from %s, or delete %s (it will be recreated on subsequent CLI commands)", providerStr, consts.CortexVersionMinor, providerStr, cliConfigPath, cliConfigPath), }) } func ErrorNoAvailableEnvironment() error { return errors.WithStack(&errors.Error{ Kind: ErrNoAvailableEnvironment, Message: "no environments are configured; run `cortex cluster up` to create a cluster, or run `cortex env configure` to connect to an existing cluster", }) } func ErrorEnvironmentNotSet() error { return errors.WithStack(&errors.Error{ Kind: ErrEnvironmentNotSet, Message: "no environment was provided and the default environment is not set; specify the environment to use via the `-e/--env` flag, or run `cortex env default` to set the default environment", }) } func ErrorEnvironmentNotFound(envName string) error { return errors.WithStack(&errors.Error{ Kind: ErrEnvironmentNotFound, Message: fmt.Sprintf("unable to find environment named \"%s\"", envName), }) } func ErrorFieldNotFoundInEnvironment(fieldName string, envName string) error { return errors.WithStack(&errors.Error{ Kind: ErrFieldNotFoundInEnvironment, Message: fmt.Sprintf("%s was not found in %s environment", fieldName, envName), }) } func ErrorInvalidOperatorEndpoint(endpoint string) error { return errors.WithStack(&errors.Error{ Kind: ErrInvalidOperatorEndpoint, Message: fmt.Sprintf("%s is not a cortex operator endpoint; run `cortex cluster info` to show your operator endpoint or run `cortex cluster up` to spin up a new cluster", endpoint), }) } func ErrorNoOperatorLoadBalancer(whichLB string) error { return errors.WithStack(&errors.Error{ Kind: ErrNoOperatorLoadBalancer, Message: fmt.Sprintf("unable to locate %s load balancer", whichLB), }) } func ErrorCortexYAMLNotFound() error { return errors.WithStack(&errors.Error{ Kind: ErrCortexYAMLNotFound, Message: "no api config file was specified, and ./cortex.yaml does not exist; create cortex.yaml, or reference an existing config file by running `cortex deploy `", }) } func ErrorDockerCtrlC() error { return errors.WithStack(&errors.Error{ Kind: ErrDockerCtrlC, NoPrint: true, NoTelemetry: true, }) } func ErrorResponseUnknown(body string, statusCode int) error { msg := body if strings.TrimSpace(body) == "" { msg = fmt.Sprintf("empty response (status code %d)", statusCode) } return errors.WithStack(&errors.Error{ Kind: ErrResponseUnknown, Message: msg, }) } func ErrorClusterUp(out string) error { return errors.WithStack(&errors.Error{ Kind: ErrClusterUp, Message: out, NoPrint: true, }) } func ErrorClusterConfigure(out string) error { return errors.WithStack(&errors.Error{ Kind: ErrClusterConfigure, Message: out, NoPrint: true, }) } func ErrorClusterDebug(out string) error { return errors.WithStack(&errors.Error{ Kind: ErrClusterDebug, Message: out, NoPrint: true, }) } func ErrorClusterRefresh(out string) error { return errors.WithStack(&errors.Error{ Kind: ErrClusterRefresh, Message: out, NoPrint: true, }) } func ErrorClusterDown(out string) error { return errors.WithStack(&errors.Error{ Kind: ErrClusterDown, Message: out, NoPrint: true, }) } func ErrorSpecifyAtLeastOneFlag(flagsToSpecify ...string) error { return errors.WithStack(&errors.Error{ Kind: ErrSpecifyAtLeastOneFlag, Message: fmt.Sprintf("must specify at least one of the following flags: %s", s.StrsOr(flagsToSpecify)), }) } func ErrorMinInstancesLowerThan(minValue int64) error { return errors.WithStack(&errors.Error{ Kind: ErrMinInstancesLowerThan, Message: fmt.Sprintf("min instances cannot be set to a value lower than %d", minValue), }) } func ErrorMaxInstancesLowerThan(minValue int64) error { return errors.WithStack(&errors.Error{ Kind: ErrMaxInstancesLowerThan, Message: fmt.Sprintf("max instances cannot be set to a value lower than %d", minValue), }) } func ErrorMinInstancesGreaterThanMaxInstances(minInstances, maxInstances int64) error { return errors.WithStack(&errors.Error{ Kind: ErrMinInstancesGreaterThanMaxInstances, Message: fmt.Sprintf("min instances (%d) cannot be set to a value higher than max instances (%d)", minInstances, maxInstances), }) } func ErrorNodeGroupNotFound(scalingNodeGroupName, clusterName, clusterRegion string, availableNodeGroups []string) error { return errors.WithStack(&errors.Error{ Kind: ErrNodeGroupNotFound, Message: fmt.Sprintf("nodegroup %s couldn't be found in the cluster named %s in region %s; the available nodegroups for this cluster are: %s", scalingNodeGroupName, clusterName, clusterRegion, s.StrsAnd(availableNodeGroups)), }) } func ErrorMutuallyExclusiveFlags(flagA, flagB string) error { return errors.WithStack(&errors.Error{ Kind: ErrMutuallyExclusiveFlags, Message: fmt.Sprintf("flags %s and %s cannot be used at the same time", flagA, flagB), }) } func ErrorClusterAccessConfigRequired(cliFlagsOnly bool) error { message := "" if cliFlagsOnly { message = "please provide the name and region of the cluster using the CLI flags (e.g. via `--name` and `--region`)" } else { message = fmt.Sprintf("please provide a cluster configuration file which specifies `%s` and `%s` (e.g. via `--config cluster.yaml`) or use the CLI flags to specify the cluster (e.g. via `--name` and `--region`)", clusterconfig.ClusterNameKey, clusterconfig.RegionKey) } return errors.WithStack(&errors.Error{ Kind: ErrClusterAccessConfigRequired, Message: message, }) } func ErrorShellCompletionNotSupported(shell string) error { return errors.WithStack(&errors.Error{ Kind: ErrShellCompletionNotSupported, Message: fmt.Sprintf("shell completion for %s is not supported (only bash and zsh are supported)", shell), }) } func ErrorNoTerminalWidth() error { return errors.WithStack(&errors.Error{ Kind: ErrNoTerminalWidth, Message: "unable to determine terminal width; please re-run the command without the `--watch` flag", }) } func ErrorDeployFromTopLevelDir(genericDirName string) error { return errors.WithStack(&errors.Error{ Kind: ErrDeployFromTopLevelDir, Message: fmt.Sprintf("cannot deploy from your %s directory - when deploying your API, cortex sends all files in your project directory (i.e. the directory which contains cortex.yaml) to your cluster (see https://docs.cortexlabs.com/v/%s/); therefore it is recommended to create a subdirectory for your project files", genericDirName, consts.CortexVersionMinor), }) } func ErrorAPINameMustBeProvided() error { return errors.WithStack(&errors.Error{ Kind: ErrAPINameMustBeProvided, Message: fmt.Sprintf("multiple apis listed; please specify the name of an api"), }) } func ErrorAPINotFoundInConfig(apiName string) error { return errors.WithStack(&errors.Error{ Kind: ErrAPINotFoundInConfig, Message: fmt.Sprintf("api '%s' not found in config", apiName), }) } func ErrorClusterUIDsLimitInBucket(bucket string) error { return errors.WithStack(&errors.Error{ Kind: ErrClusterUIDsLimitInBucket, Message: fmt.Sprintf("detected too many top level folders in %s bucket; please empty your bucket and try again", bucket), }) } ================================================ FILE: cli/cmd/get.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package cmd import ( "fmt" "strings" "time" "github.com/cortexlabs/cortex/cli/cluster" "github.com/cortexlabs/cortex/cli/types/cliconfig" "github.com/cortexlabs/cortex/cli/types/flags" "github.com/cortexlabs/cortex/pkg/lib/console" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/exit" libjson "github.com/cortexlabs/cortex/pkg/lib/json" "github.com/cortexlabs/cortex/pkg/lib/pointer" s "github.com/cortexlabs/cortex/pkg/lib/strings" "github.com/cortexlabs/cortex/pkg/lib/table" "github.com/cortexlabs/cortex/pkg/lib/telemetry" libtime "github.com/cortexlabs/cortex/pkg/lib/time" "github.com/cortexlabs/cortex/pkg/operator/schema" "github.com/cortexlabs/cortex/pkg/types/userconfig" "github.com/cortexlabs/yaml" "github.com/spf13/cobra" ) const ( _titleEnvironment = "env" _titleRealtimeAPI = "realtime api" _titleAsyncAPI = "async api" _titleLive = "live" _titleUpToDate = "up-to-date" _titleLastUpdated = "last update" ) var ( _flagGetEnv string _flagGetWatch bool ) func getInit() { _getCmd.Flags().SortFlags = false _getCmd.Flags().StringVarP(&_flagGetEnv, "env", "e", "", "environment to use") _getCmd.Flags().BoolVarP(&_flagGetWatch, "watch", "w", false, "re-run the command every 2 seconds") _getCmd.Flags().VarP(&_flagOutput, "output", "o", fmt.Sprintf("output format: one of %s", strings.Join(flags.OutputTypeStringsExcluding(flags.YAMLOutputType), "|"))) addVerboseFlag(_getCmd) } var _getCmd = &cobra.Command{ Use: "get [API_NAME] [JOB_ID]", Short: "get information about apis or jobs", Args: cobra.RangeArgs(0, 2), Run: func(cmd *cobra.Command, args []string) { var envName string if wasFlagProvided(cmd, "env") { envName = _flagGetEnv } else if len(args) > 0 { var err error envName, err = getEnvFromFlag("") if err != nil { telemetry.Event("cli.get") exit.Error(err) } } if len(args) == 1 || wasFlagProvided(cmd, "env") { env, err := ReadOrConfigureEnv(envName) if err != nil { telemetry.Event("cli.get") exit.Error(err) } telemetry.Event("cli.get", map[string]interface{}{"env_name": env.Name}) } else { telemetry.Event("cli.get") } rerun(_flagGetWatch, func() (string, error) { if len(args) == 1 { env, err := ReadOrConfigureEnv(envName) if err != nil { exit.Error(err) } out, err := envStringIfNotSpecified(envName, cmd) if err != nil { return "", err } apiTable, err := getAPI(env, args[0]) if err != nil { return "", err } if _flagOutput == flags.JSONOutputType || _flagOutput == flags.YAMLOutputType { return apiTable, nil } return out + apiTable, nil } else if len(args) == 2 { env, err := ReadOrConfigureEnv(envName) if err != nil { exit.Error(err) } out, err := envStringIfNotSpecified(envName, cmd) if err != nil { return "", err } apisRes, err := cluster.GetAPI(MustGetOperatorConfig(envName), args[0]) if err != nil { return "", err } var jobTable string if apisRes[0].Metadata.Kind == userconfig.BatchAPIKind { jobTable, err = getBatchJob(env, args[0], args[1]) } else { jobTable, err = getTaskJob(env, args[0], args[1]) } if err != nil { return "", err } if _flagOutput == flags.JSONOutputType || _flagOutput == flags.YAMLOutputType { return jobTable, nil } return out + jobTable, nil } else { envs, err := listConfiguredEnvs() if err != nil { return "", err } if len(envs) == 0 { return "", ErrorNoAvailableEnvironment() } if wasFlagProvided(cmd, "env") { env, err := ReadOrConfigureEnv(envName) if err != nil { exit.Error(err) } out, err := envStringIfNotSpecified(envName, cmd) if err != nil { return "", err } apiTable, err := getAPIsByEnv(env) if err != nil { return "", err } if _flagOutput == flags.JSONOutputType || _flagOutput == flags.YAMLOutputType { return apiTable, nil } return out + apiTable, nil } out, err := getAPIsInAllEnvironments() if err != nil { return "", err } return out, nil } }) }, } func getAPIsInAllEnvironments() (string, error) { cliConfig, err := readCLIConfig() if err != nil { return "", err } var allRealtimeAPIs []schema.APIResponse var allRealtimeAPIEnvs []string var allAsyncAPIs []schema.APIResponse var allAsyncAPIEnvs []string var allBatchAPIs []schema.APIResponse var allBatchAPIEnvs []string var allTaskAPIs []schema.APIResponse var allTaskAPIEnvs []string var allTrafficSplitters []schema.APIResponse var allTrafficSplitterEnvs []string type getAPIsOutput struct { EnvName string `json:"env_name"` APIs []schema.APIResponse `json:"apis"` Error string `json:"error"` } allAPIsOutput := []getAPIsOutput{} errorsMap := map[string]error{} // get apis from both environments for _, env := range cliConfig.Environments { apisRes, err := cluster.GetAPIs(MustGetOperatorConfig(env.Name)) apisOutput := getAPIsOutput{ EnvName: env.Name, APIs: apisRes, } if err == nil { for _, api := range apisRes { switch api.Metadata.Kind { case userconfig.BatchAPIKind: allBatchAPIEnvs = append(allBatchAPIEnvs, env.Name) allBatchAPIs = append(allBatchAPIs, api) case userconfig.RealtimeAPIKind: allRealtimeAPIEnvs = append(allRealtimeAPIEnvs, env.Name) allRealtimeAPIs = append(allRealtimeAPIs, api) case userconfig.AsyncAPIKind: allAsyncAPIEnvs = append(allAsyncAPIEnvs, env.Name) allAsyncAPIs = append(allAsyncAPIs, api) case userconfig.TaskAPIKind: allTaskAPIEnvs = append(allTaskAPIEnvs, env.Name) allTaskAPIs = append(allTaskAPIs, api) case userconfig.TrafficSplitterKind: allTrafficSplitterEnvs = append(allTrafficSplitterEnvs, env.Name) allTrafficSplitters = append(allTrafficSplitters, api) } } } else { apisOutput.Error = err.Error() errorsMap[env.Name] = err } allAPIsOutput = append(allAPIsOutput, apisOutput) } var bytes []byte if _flagOutput == flags.JSONOutputType { bytes, err = libjson.Marshal(allAPIsOutput) } else if _flagOutput == flags.YAMLOutputType { bytes, err = yaml.Marshal(allAPIsOutput) } if err != nil { return "", err } if _flagOutput == flags.JSONOutputType || _flagOutput == flags.YAMLOutputType { return string(bytes), nil } out := "" if len(allRealtimeAPIs) == 0 && len(allAsyncAPIs) == 0 && len(allBatchAPIs) == 0 && len(allTrafficSplitters) == 0 && len(allTaskAPIs) == 0 { // check if any environments errorred if len(errorsMap) != len(cliConfig.Environments) { if len(errorsMap) == 0 { return console.Bold("no apis are deployed"), nil } var successfulEnvs []string for _, env := range cliConfig.Environments { if _, ok := errorsMap[env.Name]; !ok { successfulEnvs = append(successfulEnvs, env.Name) } } fmt.Println(console.Bold(fmt.Sprintf("no apis are deployed in %s: %s", s.PluralS("environment", len(successfulEnvs)), s.StrsAnd(successfulEnvs))) + "\n") } // Print the first error for name, err := range errorsMap { if err != nil { exit.Error(errors.Wrap(err, "env "+name)) } } } else { if len(allBatchAPIs) > 0 { t := batchAPIsTable(allBatchAPIs, allBatchAPIEnvs) out += t.MustFormat() } if len(allTaskAPIs) > 0 { t := taskAPIsTable(allTaskAPIs, allTaskAPIEnvs) if len(allBatchAPIs) > 0 { out += "\n" } out += t.MustFormat() } if len(allRealtimeAPIs) > 0 { t := realtimeAPIsTable(allRealtimeAPIs, allRealtimeAPIEnvs) if len(allBatchAPIs) > 0 || len(allTaskAPIs) > 0 { out += "\n" } out += t.MustFormat() } if len(allAsyncAPIs) > 0 { t := asyncAPIsTable(allAsyncAPIs, allAsyncAPIEnvs) if len(allBatchAPIs) > 0 || len(allTaskAPIs) > 0 || len(allRealtimeAPIs) > 0 { out += "\n" } out += t.MustFormat() } if len(allTrafficSplitters) > 0 { t := trafficSplitterListTable(allTrafficSplitters, allTrafficSplitterEnvs) if len(allBatchAPIs) > 0 || len(allTaskAPIs) > 0 || len(allRealtimeAPIs) > 0 || len(allAsyncAPIs) > 0 { out += "\n" } out += t.MustFormat() } } if len(errorsMap) == 1 { out = s.EnsureBlankLineIfNotEmpty(out) out += fmt.Sprintf("unable to detect apis from the %s environment; run `cortex get --env %s` if this is unexpected\n", errors.FirstKeyInErrorMap(errorsMap), errors.FirstKeyInErrorMap(errorsMap)) } else if len(errorsMap) > 1 { out = s.EnsureBlankLineIfNotEmpty(out) out += fmt.Sprintf("unable to detect apis from the %s environments; run `cortex get --env ENV_NAME` if this is unexpected\n", s.StrsAnd(errors.NonNilErrorMapKeys(errorsMap))) } return out, nil } func getAPIsByEnv(env cliconfig.Environment) (string, error) { apisRes, err := cluster.GetAPIs(MustGetOperatorConfig(env.Name)) if err != nil { return "", err } var bytes []byte if _flagOutput == flags.JSONOutputType { bytes, err = libjson.Marshal(apisRes) } else if _flagOutput == flags.YAMLOutputType { bytes, err = yaml.Marshal(apisRes) } if err != nil { return "", err } if _flagOutput == flags.JSONOutputType || _flagOutput == flags.YAMLOutputType { return string(bytes), nil } var allRealtimeAPIs []schema.APIResponse var allAsyncAPIs []schema.APIResponse var allBatchAPIs []schema.APIResponse var allTaskAPIs []schema.APIResponse var allTrafficSplitters []schema.APIResponse for _, api := range apisRes { switch api.Metadata.Kind { case userconfig.BatchAPIKind: allBatchAPIs = append(allBatchAPIs, api) case userconfig.TaskAPIKind: allTaskAPIs = append(allTaskAPIs, api) case userconfig.RealtimeAPIKind: allRealtimeAPIs = append(allRealtimeAPIs, api) case userconfig.AsyncAPIKind: allAsyncAPIs = append(allAsyncAPIs, api) case userconfig.TrafficSplitterKind: allTrafficSplitters = append(allTrafficSplitters, api) } } if len(allRealtimeAPIs) == 0 && len(allAsyncAPIs) == 0 && len(allBatchAPIs) == 0 && len(allTaskAPIs) == 0 && len(allTrafficSplitters) == 0 { return console.Bold("no apis are deployed"), nil } out := "" if len(allBatchAPIs) > 0 { envNames := []string{} for range allBatchAPIs { envNames = append(envNames, env.Name) } t := batchAPIsTable(allBatchAPIs, envNames) t.FindHeaderByTitle(_titleEnvironment).Hidden = true out += t.MustFormat() } if len(allTaskAPIs) > 0 { envNames := []string{} for range allTaskAPIs { envNames = append(envNames, env.Name) } t := taskAPIsTable(allTaskAPIs, envNames) t.FindHeaderByTitle(_titleEnvironment).Hidden = true if len(allBatchAPIs) > 0 { out += "\n" } out += t.MustFormat() } if len(allRealtimeAPIs) > 0 { envNames := []string{} for range allRealtimeAPIs { envNames = append(envNames, env.Name) } t := realtimeAPIsTable(allRealtimeAPIs, envNames) t.FindHeaderByTitle(_titleEnvironment).Hidden = true if len(allBatchAPIs) > 0 || len(allTaskAPIs) > 0 { out += "\n" } out += t.MustFormat() } if len(allAsyncAPIs) > 0 { envNames := []string{} for range allAsyncAPIs { envNames = append(envNames, env.Name) } t := asyncAPIsTable(allAsyncAPIs, envNames) t.FindHeaderByTitle(_titleEnvironment).Hidden = true if len(allBatchAPIs) > 0 || len(allTaskAPIs) > 0 || len(allRealtimeAPIs) > 0 { out += "\n" } out += t.MustFormat() } if len(allTrafficSplitters) > 0 { envNames := []string{} for range allTrafficSplitters { envNames = append(envNames, env.Name) } t := trafficSplitterListTable(allTrafficSplitters, envNames) t.FindHeaderByTitle(_titleEnvironment).Hidden = true if len(allBatchAPIs) > 0 || len(allTaskAPIs) > 0 || len(allRealtimeAPIs) > 0 || len(allAsyncAPIs) > 0 { out += "\n" } out += t.MustFormat() } return out, nil } func getAPI(env cliconfig.Environment, apiName string) (string, error) { apisRes, err := cluster.GetAPI(MustGetOperatorConfig(env.Name), apiName) if err != nil { return "", err } var bytes []byte if _flagOutput == flags.JSONOutputType { bytes, err = libjson.Marshal(apisRes) } else if _flagOutput == flags.YAMLOutputType { bytes, err = yaml.Marshal(apisRes) } if err != nil { return "", err } if _flagOutput == flags.JSONOutputType || _flagOutput == flags.YAMLOutputType { return string(bytes), nil } if len(apisRes) == 0 { exit.Error(errors.ErrorUnexpected(fmt.Sprintf("unable to find api %s", apiName))) } apiRes := apisRes[0] switch apiRes.Metadata.Kind { case userconfig.RealtimeAPIKind: return realtimeAPITable(apiRes, env) case userconfig.AsyncAPIKind: return asyncAPITable(apiRes, env) case userconfig.TrafficSplitterKind: return trafficSplitterTable(apiRes, env) case userconfig.BatchAPIKind: return batchAPITable(apiRes), nil case userconfig.TaskAPIKind: return taskAPITable(apiRes), nil default: return "", errors.ErrorUnexpected(fmt.Sprintf("encountered unexpected kind %s for api %s", apiRes.Metadata.Kind, apiRes.Metadata.Name)) } } func apiHistoryTable(apiVersions []schema.APIVersion) string { t := table.Table{ Headers: []table.Header{ {Title: "api id"}, {Title: "last deployed"}, }, } t.Rows = make([][]interface{}, len(apiVersions)) for i, apiVersion := range apiVersions { lastUpdated := time.Unix(apiVersion.LastUpdated, 0) t.Rows[i] = []interface{}{apiVersion.APIID, libtime.SinceStr(&lastUpdated)} } return t.MustFormat(&table.Opts{Sort: pointer.Bool(false)}) } func titleStr(title string) string { return "\n" + console.Bold(title) + "\n" } ================================================ FILE: cli/cmd/lib_apis.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package cmd import ( "github.com/cortexlabs/cortex/pkg/lib/table" "github.com/cortexlabs/cortex/pkg/types/status" ) func replicaCountTable(counts *status.ReplicaCounts) table.Table { var rows [][]interface{} for _, replicaCountType := range status.ReplicaCountTypes { count := counts.GetCountBy(replicaCountType) canBeHiddenIfZero := false switch replicaCountType { case status.ReplicaCountFailed: canBeHiddenIfZero = true case status.ReplicaCountKilled: canBeHiddenIfZero = true case status.ReplicaCountKilledOOM: canBeHiddenIfZero = true case status.ReplicaCountErrImagePull: canBeHiddenIfZero = true case status.ReplicaCountUnknown: canBeHiddenIfZero = true case status.ReplicaCountStalled: canBeHiddenIfZero = true } if count == 0 && canBeHiddenIfZero { continue } rows = append(rows, []interface{}{ replicaCountType, count, }) } return table.Table{ Headers: []table.Header{ {Title: _titleReplicaStatus, MinWidth: 32, MaxWidth: 32}, {Title: _titleReplicaCount}, }, Rows: rows, } } ================================================ FILE: cli/cmd/lib_async_apis.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package cmd import ( "fmt" "strings" "time" "github.com/cortexlabs/cortex/cli/types/cliconfig" "github.com/cortexlabs/cortex/pkg/lib/console" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/table" libtime "github.com/cortexlabs/cortex/pkg/lib/time" "github.com/cortexlabs/cortex/pkg/operator/schema" ) func asyncAPITable(asyncAPI schema.APIResponse, env cliconfig.Environment) (string, error) { var out string t := asyncAPIsTable([]schema.APIResponse{asyncAPI}, []string{env.Name}) out += t.MustFormat() if asyncAPI.DashboardURL != nil && *asyncAPI.DashboardURL != "" { out += "\n" + console.Bold("metrics dashboard: ") + *asyncAPI.DashboardURL + "\n" } if asyncAPI.Endpoint != nil { out += "\n" + console.Bold("endpoint: ") + *asyncAPI.Endpoint + "\n" } out += "\n" + apiHistoryTable(asyncAPI.APIVersions) if !_flagVerbose { return out, nil } out += titleStr("configuration") + strings.TrimSpace(asyncAPI.Spec.UserStr()) return out, nil } func asyncDescribeAPITable(asyncAPI schema.APIResponse, env cliconfig.Environment) (string, error) { if asyncAPI.Metadata == nil { return "", errors.ErrorUnexpected("missing metadata from operator response") } if asyncAPI.Status == nil { return "", errors.ErrorUnexpected(fmt.Sprintf("missing status for %s api", asyncAPI.Metadata.Name)) } t := asyncAPIsTable([]schema.APIResponse{asyncAPI}, []string{env.Name}) out := t.MustFormat() if asyncAPI.DashboardURL != nil && *asyncAPI.DashboardURL != "" { out += "\n" + console.Bold("metrics dashboard: ") + *asyncAPI.DashboardURL + "\n" } if asyncAPI.Endpoint != nil { out += "\n" + console.Bold("endpoint: ") + *asyncAPI.Endpoint + "\n" } t = replicaCountTable(asyncAPI.Status.ReplicaCounts) out += "\n" + t.MustFormat() return out, nil } func asyncAPIsTable(asyncAPIs []schema.APIResponse, envNames []string) table.Table { rows := make([][]interface{}, 0, len(asyncAPIs)) for i, asyncAPI := range asyncAPIs { if asyncAPI.Metadata == nil || asyncAPI.Status == nil { continue } lastUpdated := time.Unix(asyncAPI.Metadata.LastUpdated, 0) rows = append(rows, []interface{}{ envNames[i], asyncAPI.Metadata.Name, fmt.Sprintf("%d/%d", asyncAPI.Status.Ready, asyncAPI.Status.Requested), asyncAPI.Status.UpToDate, libtime.SinceStr(&lastUpdated), }) } return table.Table{ Headers: []table.Header{ {Title: _titleEnvironment}, {Title: _titleAsyncAPI}, {Title: _titleLive}, {Title: _titleUpToDate}, {Title: _titleLastUpdated}, }, Rows: rows, } } ================================================ FILE: cli/cmd/lib_aws_creds.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package cmd import ( "fmt" "github.com/cortexlabs/cortex/pkg/consts" "github.com/cortexlabs/cortex/pkg/lib/aws" "github.com/cortexlabs/cortex/pkg/lib/prompt" "github.com/cortexlabs/cortex/pkg/types/clusterconfig" ) func newAWSClient(region string, printToStdout bool) (*aws.Client, error) { if err := clusterconfig.ValidateRegion(region); err != nil { return nil, err } awsClient, err := aws.NewForRegion(region) if err != nil { return nil, err } if _, _, err := awsClient.CheckCredentials(); err != nil { return nil, err } if printToStdout { fmt.Println("using aws credentials with access key " + *awsClient.AccessKeyID() + "\n") } return awsClient, nil } func promptIfNotAdmin(awsClient *aws.Client, disallowPrompt bool) { accessKeyMsg := "" if accessKey := awsClient.AccessKeyID(); accessKey != nil { accessKeyMsg = fmt.Sprintf(" (with access key %s)", *accessKey) } if !awsClient.IsAdmin() { warningStr := fmt.Sprintf("warning: your IAM user%s does not have administrator access. Please attach the AdministratorAccess policy to your IAM user (or to a group that your IAM user belongs to), or visit https://docs.cortexlabs.com/v/%s/ to view the minimum permissions required to run `cortex cluster` commands.\n\n", accessKeyMsg, consts.CortexVersionMinor) if disallowPrompt { fmt.Print(warningStr) } else { prompt.YesOrExit(warningStr+"are you sure you want to continue without administrator access?", "", "") } } } func warnIfNotAdmin(awsClient *aws.Client) { accessKeyMsg := "" if accessKey := awsClient.AccessKeyID(); accessKey != nil { accessKeyMsg = fmt.Sprintf(" (with access key %s)", *accessKey) } if !awsClient.IsAdmin() { fmt.Println(fmt.Sprintf("warning: your IAM user or assumed role%s does not have administrator access. This may prevent this command from executing correctly, so it is recommended to attach the AdministratorAccess policy to your IAM user or role.\n", accessKeyMsg), "", "") } } ================================================ FILE: cli/cmd/lib_batch_apis.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package cmd import ( "fmt" "time" "github.com/cortexlabs/cortex/cli/cluster" "github.com/cortexlabs/cortex/cli/types/cliconfig" "github.com/cortexlabs/cortex/cli/types/flags" "github.com/cortexlabs/cortex/pkg/lib/console" libjson "github.com/cortexlabs/cortex/pkg/lib/json" "github.com/cortexlabs/cortex/pkg/lib/pointer" s "github.com/cortexlabs/cortex/pkg/lib/strings" "github.com/cortexlabs/cortex/pkg/lib/table" libtime "github.com/cortexlabs/cortex/pkg/lib/time" "github.com/cortexlabs/cortex/pkg/operator/schema" "github.com/cortexlabs/cortex/pkg/types/status" "github.com/cortexlabs/yaml" ) const ( _titleBatchAPI = "batch api" _titleJobCount = "running jobs" _titleLatestJobID = "latest job id" ) func batchAPIsTable(batchAPIs []schema.APIResponse, envNames []string) table.Table { rows := make([][]interface{}, 0, len(batchAPIs)) for i, batchAPI := range batchAPIs { if batchAPI.Metadata == nil { continue } lastAPIUpdated := time.Unix(batchAPI.Metadata.LastUpdated, 0) latestStartTime := time.Time{} latestJobID := "-" runningJobs := 0 for _, job := range batchAPI.BatchJobStatuses { if job.StartTime.After(latestStartTime) { latestStartTime = job.StartTime latestJobID = job.ID + fmt.Sprintf(" (submitted %s ago)", libtime.SinceStr(&latestStartTime)) } if job.Status.IsInProgress() { runningJobs++ } } rows = append(rows, []interface{}{ envNames[i], batchAPI.Metadata.Name, runningJobs, latestJobID, libtime.SinceStr(&lastAPIUpdated), }) } return table.Table{ Headers: []table.Header{ {Title: _titleEnvironment}, {Title: _titleBatchAPI}, {Title: _titleJobCount}, {Title: _titleLatestJobID}, {Title: _titleLastUpdated}, }, Rows: rows, } } func batchAPITable(batchAPI schema.APIResponse) string { jobRows := make([][]interface{}, 0, len(batchAPI.BatchJobStatuses)) out := "" if len(batchAPI.BatchJobStatuses) == 0 { out = console.Bold("no submitted batch jobs\n") } else { for _, job := range batchAPI.BatchJobStatuses { jobEndTime := time.Now() if job.EndTime != nil { jobEndTime = *job.EndTime } duration := jobEndTime.Sub(job.StartTime).Truncate(time.Second).String() jobRows = append(jobRows, []interface{}{ job.ID, job.Status.Message(), job.TotalBatchCount, job.StartTime.Format(_timeFormat), duration, }) } t := table.Table{ Headers: []table.Header{ {Title: "job id"}, {Title: "status"}, {Title: "total batches"}, {Title: "start time"}, {Title: "duration"}, }, Rows: jobRows, } out += t.MustFormat() } if batchAPI.DashboardURL != nil && *batchAPI.DashboardURL != "" { out += "\n" + console.Bold("metrics dashboard: ") + *batchAPI.DashboardURL + "\n" } if batchAPI.Endpoint != nil { out += "\n" + console.Bold("endpoint: ") + *batchAPI.Endpoint + "\n" } out += "\n" + apiHistoryTable(batchAPI.APIVersions) if !_flagVerbose { return out } out += titleStr("batch api configuration") + batchAPI.Spec.UserStr() return out } func getBatchJob(env cliconfig.Environment, apiName string, jobID string) (string, error) { resp, err := cluster.GetBatchJob(MustGetOperatorConfig(env.Name), apiName, jobID) if err != nil { return "", err } var bytes []byte if _flagOutput == flags.JSONOutputType { bytes, err = libjson.Marshal(resp) } else if _flagOutput == flags.YAMLOutputType { bytes, err = yaml.Marshal(resp) } if err != nil { return "", err } if _flagOutput == flags.JSONOutputType || _flagOutput == flags.YAMLOutputType { return string(bytes), nil } job := resp.JobStatus out := "" jobIntroTable := table.KeyValuePairs{} jobIntroTable.Add("job id", job.ID) jobIntroTable.Add("status", job.Status.Message()) out += jobIntroTable.String(&table.KeyValuePairOpts{BoldKeys: pointer.Bool(true)}) jobTimingTable := table.KeyValuePairs{} jobTimingTable.Add("start time", job.StartTime.Format(_timeFormat)) jobEndTime := time.Now() if job.EndTime != nil { jobTimingTable.Add("end time", job.EndTime.Format(_timeFormat)) jobEndTime = *job.EndTime } else { jobTimingTable.Add("end time", "-") } duration := jobEndTime.Sub(job.StartTime).Truncate(time.Second).String() jobTimingTable.Add("duration", duration) out += "\n" + jobTimingTable.String(&table.KeyValuePairOpts{BoldKeys: pointer.Bool(true)}) succeeded := "-" failed := "-" avgTimePerBatch := "-" if resp.Metrics != nil { if resp.Metrics.AverageTimePerBatch != nil { batchMetricsDuration := time.Duration(*resp.Metrics.AverageTimePerBatch*1000000000) * time.Nanosecond avgTimePerBatch = batchMetricsDuration.Truncate(time.Millisecond).String() } succeeded = s.Int(resp.Metrics.Succeeded) failed = s.Int(resp.Metrics.Failed) } t := table.Table{ Headers: []table.Header{ {Title: "total"}, {Title: "succeeded"}, {Title: "failed attempts"}, {Title: "avg time per batch"}, }, Rows: [][]interface{}{ { job.TotalBatchCount, succeeded, failed, avgTimePerBatch, }, }, } out += titleStr("batch stats") + t.MustFormat(&table.Opts{BoldHeader: pointer.Bool(false)}) if job.Status == status.JobEnqueuing { out += "\n" + "still enqueuing, workers have not been allocated for this job yet\n" } else if job.Status.IsCompleted() { out += "\n" + "worker stats are not available because this job is not currently running\n" } else { out += titleStr("worker stats") if job.WorkerCounts != nil { t := table.Table{ Headers: []table.Header{ {Title: "Requested"}, {Title: "Pending"}, {Title: "Creating"}, {Title: "Ready"}, {Title: "NotReady"}, {Title: "ErrImagePull", Hidden: job.WorkerCounts.ErrImagePull == 0}, {Title: "Terminating", Hidden: job.WorkerCounts.Terminating == 0}, {Title: "Failed", Hidden: job.WorkerCounts.Failed == 0}, {Title: "Killed", Hidden: job.WorkerCounts.Killed == 0}, {Title: "KilledOOM", Hidden: job.WorkerCounts.KilledOOM == 0}, {Title: "Stalled", Hidden: job.WorkerCounts.Stalled == 0}, {Title: "Unknown", Hidden: job.WorkerCounts.Unknown == 0}, {Title: "Succeeded"}, }, Rows: [][]interface{}{ { job.Workers, job.WorkerCounts.Pending, job.WorkerCounts.Creating, job.WorkerCounts.Ready, job.WorkerCounts.NotReady, job.WorkerCounts.ErrImagePull, job.WorkerCounts.Terminating, job.WorkerCounts.Failed, job.WorkerCounts.Killed, job.WorkerCounts.KilledOOM, job.WorkerCounts.Stalled, job.WorkerCounts.Unknown, job.WorkerCounts.Succeeded, }, }, } out += t.MustFormat(&table.Opts{BoldHeader: pointer.Bool(false)}) } else { out += "unable to get worker stats\n" } } out += "\n" + console.Bold("job endpoint: ") + resp.Endpoint + "\n" jobSpecStr, err := libjson.Pretty(job.BatchJob) if err != nil { return "", err } out += titleStr("job configuration") + jobSpecStr return out, nil } ================================================ FILE: cli/cmd/lib_cli_config.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package cmd import ( "crypto/tls" "fmt" "io/ioutil" "net/http" "os" "strings" "github.com/cortexlabs/cortex/cli/cluster" "github.com/cortexlabs/cortex/cli/types/cliconfig" cr "github.com/cortexlabs/cortex/pkg/lib/configreader" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/exit" "github.com/cortexlabs/cortex/pkg/lib/files" "github.com/cortexlabs/cortex/pkg/lib/json" "github.com/cortexlabs/cortex/pkg/lib/print" "github.com/cortexlabs/cortex/pkg/lib/prompt" s "github.com/cortexlabs/cortex/pkg/lib/strings" "github.com/cortexlabs/cortex/pkg/lib/urls" "github.com/cortexlabs/cortex/pkg/operator/schema" "github.com/cortexlabs/yaml" ) var _cliConfigValidation = &cr.StructValidation{ TreatNullAsEmpty: true, StructFieldValidations: []*cr.StructFieldValidation{ { StructField: "Telemetry", BoolPtrValidation: &cr.BoolPtrValidation{ Required: false, }, }, { StructField: "DefaultEnvironment", StringPtrValidation: &cr.StringPtrValidation{ Required: false, AllowExplicitNull: true, }, }, { StructField: "Environments", StructListValidation: &cr.StructListValidation{ AllowExplicitNull: true, StructValidation: &cr.StructValidation{ AllowExtraFields: true, // backwards compatibility with previous version cli.yaml that require aws creds StructFieldValidations: []*cr.StructFieldValidation{ { StructField: "Name", StringValidation: &cr.StringValidation{ Required: true, MaxLength: 63, }, }, { Key: "provider", StringValidation: &cr.StringValidation{ AllowEmpty: true, Validator: func(provider string) (string, error) { if provider == "" || provider == "aws" { return "", nil } if provider == "gcp" || provider == "local" { return "", ErrorInvalidLegacyProvider(provider, _cliConfigPath) } return "", ErrorInvalidProvider(provider, _cliConfigPath) }, }, }, { StructField: "OperatorEndpoint", StringValidation: &cr.StringValidation{ Required: true, Validator: cliconfig.CortexEndpointValidator, }, }, }, }, }, }, }, } func getEnvFromFlag(envFlag string) (string, error) { if envFlag != "" { return envFlag, nil } defaultEnv, err := getDefaultEnv() if err != nil { return "", err } if defaultEnv != nil { return *defaultEnv, nil } envs, err := listConfiguredEnvs() if err != nil { return "", err } if len(envs) == 0 { return "", ErrorNoAvailableEnvironment() } return "", ErrorEnvironmentNotSet() } func promptForExistingEnvName(promptMsg string) string { configuredEnvNames, err := listConfiguredEnvNames() if err != nil { exit.Error(err) } fmt.Printf("your currently configured environments are: %s\n\n", strings.Join(configuredEnvNames, ", ")) envNameContainer := &struct { EnvironmentName string }{} err = cr.ReadPrompt(envNameContainer, &cr.PromptValidation{ PromptItemValidations: []*cr.PromptItemValidation{ { StructField: "EnvironmentName", PromptOpts: &prompt.Options{ Prompt: promptMsg, }, StringValidation: &cr.StringValidation{ Required: true, AllowedValues: configuredEnvNames, }, }, }, }) if err != nil { exit.Error(err) } return envNameContainer.EnvironmentName } func promptEnv(env *cliconfig.Environment, defaults cliconfig.Environment) error { if env.OperatorEndpoint == "" { fmt.Print("you can get your cortex operator endpoint using `cortex cluster info` if you already have a cortex cluster running, otherwise run `cortex cluster up` to create a cortex cluster\n\n") } validator := func(endpoint string) (string, error) { operatorURL, err := validateOperatorEndpoint(endpoint) if err != nil { return "", err } return operatorURL, nil } for true { err := cr.ReadPrompt(env, &cr.PromptValidation{ SkipNonEmptyFields: true, PromptItemValidations: []*cr.PromptItemValidation{ { StructField: "Name", PromptOpts: &prompt.Options{ Prompt: "name of environment to create or update", }, StringValidation: &cr.StringValidation{ Required: true, }, }, { StructField: "OperatorEndpoint", PromptOpts: &prompt.Options{ Prompt: "cortex operator endpoint", }, StringValidation: &cr.StringValidation{ Required: true, Default: defaults.OperatorEndpoint, Validator: validator, }, }, }, }) if err != nil { return err } return nil } return nil } // Only validate this during prompt, not when reading from file func validateOperatorEndpoint(endpoint string) (string, error) { url, err := cliconfig.CortexEndpointValidator(endpoint) if err != nil { return "", err } parsedURL, err := urls.Parse(url) if err != nil { return "", err } url = parsedURL.String() req, err := http.NewRequest("GET", urls.Join(url, "/verifycortex"), nil) if err != nil { return "", errors.Wrap(err, "verifying operator endpoint", url) } client := http.Client{ Transport: &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, }, } response, err := client.Do(req) if err != nil { return "", ErrorInvalidOperatorEndpoint(url) } defer response.Body.Close() if response.StatusCode != 200 { return "", ErrorInvalidOperatorEndpoint(url) } bodyBytes, err := ioutil.ReadAll(response.Body) if err != nil { return "", errors.Wrap(err, _errStrRead) } var verifyCortex schema.VerifyCortexResponse if err = json.Unmarshal(bodyBytes, &verifyCortex); err != nil { return "", errors.Wrap(err, endpoint, string(bodyBytes)) } return url, nil } func getDefaultEnv() (*string, error) { cliConfig, err := readCLIConfig() if err != nil { return nil, err } if cliConfig.DefaultEnvironment != nil { return cliConfig.DefaultEnvironment, nil } if len(cliConfig.Environments) == 1 { defaultEnv := cliConfig.Environments[0].Name err := setDefaultEnv(defaultEnv) if err != nil { return nil, err } return &defaultEnv, nil } return nil, nil } func setDefaultEnv(envName string) error { cliConfig, err := readCLIConfig() if err != nil { return err } envExists, err := isEnvConfigured(envName) if err != nil { return err } if !envExists { return cliconfig.ErrorEnvironmentNotConfigured(envName) } cliConfig.DefaultEnvironment = &envName if err := writeCLIConfig(cliConfig); err != nil { return err } return nil } func renameEnv(oldEnvName string, newEnvName string) error { cliConfig, err := readCLIConfig() if err != nil { return err } renamedEnv := false for _, env := range cliConfig.Environments { if env.Name == newEnvName { return cliconfig.ErrorEnvironmentAlreadyConfigured(newEnvName) } if env.Name == oldEnvName { env.Name = newEnvName renamedEnv = true } } if !renamedEnv { return cliconfig.ErrorEnvironmentNotConfigured(oldEnvName) } if cliConfig.DefaultEnvironment != nil && *cliConfig.DefaultEnvironment == oldEnvName { cliConfig.DefaultEnvironment = &newEnvName } if err := writeCLIConfig(cliConfig); err != nil { return err } return nil } func readTelemetryConfig() (bool, error) { cliConfig, err := readCLIConfig() if err != nil { return false, err } if cliConfig.Telemetry != nil && *cliConfig.Telemetry == false { return false, nil } return true, nil } // Returns false if there is an error reading the CLI config func isTelemetryEnabled() bool { enabled, err := readTelemetryConfig() if err != nil { return false } return enabled } // Will return nil if not configured func readEnv(envName string) (*cliconfig.Environment, error) { cliConfig, err := readCLIConfig() if err != nil { return nil, err } for _, env := range cliConfig.Environments { if env.Name == envName { return env, nil } } return nil, nil } func ReadOrConfigureEnv(envName string) (cliconfig.Environment, error) { existingEnv, err := readEnv(envName) if err != nil { return cliconfig.Environment{}, err } if existingEnv != nil { return *existingEnv, nil } promptStr := fmt.Sprintf("the %s environment is not configured; do you already have a Cortex cluster running?", envName) yesMsg := fmt.Sprintf("please configure the %s environment to point to your running cluster:\n", envName) noMsg := "you can create a cluster by running the `cortex cluster up` command" prompt.YesOrExit(promptStr, yesMsg, noMsg) env, err := configureEnv(envName, cliconfig.Environment{}) if err != nil { return cliconfig.Environment{}, err } return env, nil } func getEnvConfigDefaults(envName string) cliconfig.Environment { defaults := cliconfig.Environment{} prevEnv, err := readEnv(envName) if err == nil && prevEnv != nil { defaults = *prevEnv } if defaults.OperatorEndpoint == "" && os.Getenv("CORTEX_OPERATOR_ENDPOINT") != "" { defaults.OperatorEndpoint = os.Getenv("CORTEX_OPERATOR_ENDPOINT") } return defaults } // If envName is "", this will prompt for the environment name to configure func configureEnv(envName string, fieldsToSkipPrompt cliconfig.Environment) (cliconfig.Environment, error) { env := cliconfig.Environment{ Name: envName, OperatorEndpoint: fieldsToSkipPrompt.OperatorEndpoint, } defaults := getEnvConfigDefaults(env.Name) err := promptEnv(&env, defaults) if err != nil { return cliconfig.Environment{}, err } if err := env.Validate(); err != nil { return cliconfig.Environment{}, err } if err := addEnvToCLIConfig(env, false); err != nil { return cliconfig.Environment{}, err } print.BoldFirstLine(fmt.Sprintf("configured %s environment", env.Name)) return env, nil } func MustGetOperatorConfig(envName string) cluster.OperatorConfig { clientID := clientID() env, err := readEnv(envName) if err != nil { exit.Error(err) } if env == nil { exit.Error(ErrorEnvironmentNotFound(envName)) } operatorConfig := cluster.OperatorConfig{ Telemetry: isTelemetryEnabled(), ClientID: clientID, EnvName: env.Name, } if env.OperatorEndpoint == "" { exit.Error(ErrorFieldNotFoundInEnvironment(cliconfig.OperatorEndpointKey, env.Name)) } operatorConfig.OperatorEndpoint = env.OperatorEndpoint return operatorConfig } func listConfiguredEnvs() ([]*cliconfig.Environment, error) { cliConfig, err := readCLIConfig() if err != nil { return nil, err } return cliConfig.Environments, nil } func listConfiguredEnvNames() ([]string, error) { envList, err := listConfiguredEnvs() if err != nil { return nil, err } envNames := make([]string, len(envList)) for i, env := range envList { envNames[i] = env.Name } return envNames, nil } func isEnvConfigured(envName string) (bool, error) { envList, err := listConfiguredEnvs() if err != nil { return false, err } for _, env := range envList { if env.Name == envName { return true, nil } } return false, nil } func addEnvToCLIConfig(newEnv cliconfig.Environment, setAsDefault bool) error { cliConfig, err := readCLIConfig() if err != nil { return errors.Wrap(err, "unable to configure cli environment") } replaced := false for i, prevEnv := range cliConfig.Environments { if prevEnv.Name == newEnv.Name { cliConfig.Environments[i] = &newEnv replaced = true break } } if !replaced { cliConfig.Environments = append(cliConfig.Environments, &newEnv) } if setAsDefault { cliConfig.DefaultEnvironment = &newEnv.Name } if err := writeCLIConfig(cliConfig); err != nil { return errors.Wrap(err, "unable to configure cli environment") } return nil } func removeEnvFromCLIConfig(envName string) error { cliConfig, err := readCLIConfig() if err != nil { return err } prevDefault, err := getDefaultEnv() if err != nil { return err } var updatedEnvs []*cliconfig.Environment deleted := false for _, env := range cliConfig.Environments { if env.Name == envName { deleted = true continue } updatedEnvs = append(updatedEnvs, env) } if !deleted { return cliconfig.ErrorEnvironmentNotConfigured(envName) } cliConfig.Environments = updatedEnvs if prevDefault != nil && envName == *prevDefault { cliConfig.DefaultEnvironment = nil } if len(cliConfig.Environments) == 1 { cliConfig.DefaultEnvironment = &cliConfig.Environments[0].Name } if err := writeCLIConfig(cliConfig); err != nil { return err } return nil } // returns the list of environment names, and whether any of them were the default func getEnvNamesByOperatorEndpoint(operatorEndpoint string) ([]string, bool, error) { cliConfig, err := readCLIConfig() if err != nil { return nil, false, err } var envNames []string isDefaultEnv := false for _, env := range cliConfig.Environments { if env.OperatorEndpoint != "" && s.LastSplit(env.OperatorEndpoint, "//") == s.LastSplit(operatorEndpoint, "//") { envNames = append(envNames, env.Name) if cliConfig.DefaultEnvironment != nil && env.Name == *cliConfig.DefaultEnvironment { isDefaultEnv = true } } } return envNames, isDefaultEnv, nil } func readCLIConfig() (cliconfig.CLIConfig, error) { if !files.IsFile(_cliConfigPath) { cliConfig := cliconfig.CLIConfig{} if err := cliConfig.Validate(); err != nil { return cliconfig.CLIConfig{}, err // unexpected } // create file so that the file created by the manager container maintains current user permissions if err := writeCLIConfig(cliConfig); err != nil { return cliconfig.CLIConfig{}, errors.Wrap(err, "unable to save CLI configuration file") } return cliConfig, nil } cliConfig := cliconfig.CLIConfig{} errs := cr.ParseYAMLFile(&cliConfig, _cliConfigValidation, _cliConfigPath) if errors.HasError(errs) { return cliconfig.CLIConfig{}, errors.FirstError(errs...) } if err := cliConfig.Validate(); err != nil { return cliconfig.CLIConfig{}, errors.Wrap(err, _cliConfigPath) } return cliConfig, nil } func writeCLIConfig(cliConfig cliconfig.CLIConfig) error { if err := cliConfig.Validate(); err != nil { return err } cliConfigBytes, err := yaml.Marshal(cliConfig) if err != nil { return errors.WithStack(err) } if err := files.WriteFile(cliConfigBytes, _cliConfigPath); err != nil { return err } return nil } ================================================ FILE: cli/cmd/lib_client_id.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package cmd import ( "github.com/cortexlabs/cortex/pkg/lib/files" "github.com/google/uuid" ) var _cachedClientID string func clientID() string { if _cachedClientID != "" { return _cachedClientID } var err error _cachedClientID, err = files.ReadFile(_clientIDPath) if err != nil || _cachedClientID == "" { _cachedClientID = uuid.New().String() files.WriteFile([]byte(_cachedClientID), _clientIDPath) } return _cachedClientID } ================================================ FILE: cli/cmd/lib_cluster_config.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package cmd import ( "fmt" "path" "path/filepath" "regexp" "github.com/cortexlabs/cortex/pkg/consts" "github.com/cortexlabs/cortex/pkg/lib/aws" cr "github.com/cortexlabs/cortex/pkg/lib/configreader" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/files" "github.com/cortexlabs/cortex/pkg/lib/k8s" "github.com/cortexlabs/cortex/pkg/lib/maps" libmath "github.com/cortexlabs/cortex/pkg/lib/math" "github.com/cortexlabs/cortex/pkg/lib/pointer" "github.com/cortexlabs/cortex/pkg/lib/prompt" s "github.com/cortexlabs/cortex/pkg/lib/strings" "github.com/cortexlabs/cortex/pkg/lib/table" "github.com/cortexlabs/cortex/pkg/types/clusterconfig" "github.com/cortexlabs/cortex/pkg/types/clusterstate" ) var _cachedClusterConfigRegex = regexp.MustCompile(`^cluster_\S+\.yaml$`) func getCachedClusterConfigPath(clusterName string, region string) string { return filepath.Join(_localDir, fmt.Sprintf("cluster_%s_%s.yaml", clusterName, region)) } func existingCachedClusterConfigPaths() []string { paths, err := files.ListDir(_localDir, false) if err != nil { return nil } var matches []string for _, p := range paths { if _cachedClusterConfigRegex.MatchString(path.Base(p)) { matches = append(matches, p) } } return matches } func readCachedClusterConfigFile(clusterConfig *clusterconfig.Config, filePath string) error { errs := cr.ParseYAMLFile(clusterConfig, clusterconfig.FullConfigValidation, filePath) if errors.HasError(errs) { return errors.FirstError(errs...) } return nil } func readUserClusterConfigFile(clusterConfig *clusterconfig.Config, filePath string) error { errs := cr.ParseYAMLFile(clusterConfig, clusterconfig.FullConfigValidation, filePath) if errors.HasError(errs) { return errors.Append(errors.FirstError(errs...), fmt.Sprintf("\n\ncluster configuration schema can be found at https://docs.cortexlabs.com/v/%s/", consts.CortexVersionMinor)) } return nil } func getNewClusterAccessConfig(clusterConfigFile string) (*clusterconfig.AccessConfig, error) { accessConfig := &clusterconfig.AccessConfig{} errs := cr.ParseYAMLFile(accessConfig, clusterconfig.AccessValidation, clusterConfigFile) if errors.HasError(errs) { return nil, errors.Append(errors.FirstError(errs...), fmt.Sprintf("\n\ncluster configuration schema can be found at https://docs.cortexlabs.com/v/%s/", consts.CortexVersionMinor)) } return accessConfig, nil } func getClusterAccessConfigWithCache(hasClusterFlags bool) (*clusterconfig.AccessConfig, error) { accessConfig := &clusterconfig.AccessConfig{ ImageManager: consts.DefaultRegistry() + "/manager:" + consts.CortexVersion, } cachedPaths := existingCachedClusterConfigPaths() if len(cachedPaths) == 1 { cachedAccessConfig := &clusterconfig.AccessConfig{} cr.ParseYAMLFile(cachedAccessConfig, clusterconfig.AccessValidation, cachedPaths[0]) accessConfig.ClusterName = cachedAccessConfig.ClusterName accessConfig.Region = cachedAccessConfig.Region } if _flagClusterConfig != "" { errs := cr.ParseYAMLFile(accessConfig, clusterconfig.AccessValidation, _flagClusterConfig) if errors.HasError(errs) { return nil, errors.Append(errors.FirstError(errs...), fmt.Sprintf("\n\ncluster configuration schema can be found at https://docs.cortexlabs.com/v/%s/", consts.CortexVersionMinor)) } } if _flagClusterName != "" { accessConfig.ClusterName = _flagClusterName } if _flagClusterRegion != "" { accessConfig.Region = _flagClusterRegion } if accessConfig.ClusterName == "" || accessConfig.Region == "" { return nil, ErrorClusterAccessConfigRequired(hasClusterFlags) } return accessConfig, nil } func getInstallClusterConfig(awsClient *aws.Client, clusterConfigFile string) (*clusterconfig.Config, error) { clusterConfig := &clusterconfig.Config{} err := readUserClusterConfigFile(clusterConfig, clusterConfigFile) if err != nil { return nil, err } clusterConfig.Telemetry = isTelemetryEnabled() err = clusterConfig.ValidateOnInstall(awsClient) if err != nil { err = errors.Append(err, fmt.Sprintf("\n\ncluster configuration schema can be found at https://docs.cortexlabs.com/v/%s/", consts.CortexVersionMinor)) return nil, errors.Wrap(err, clusterConfigFile) } return clusterConfig, nil } func getConfigureClusterConfig(awsClient *aws.Client, k8sClient *k8s.Client, stacks clusterstate.ClusterStacks, cachedClusterConfig clusterconfig.Config, newClusterConfigFile string) (*clusterconfig.Config, clusterconfig.ConfigureChanges, error) { newUserClusterConfig := &clusterconfig.Config{} err := readUserClusterConfigFile(newUserClusterConfig, newClusterConfigFile) if err != nil { return nil, clusterconfig.ConfigureChanges{}, err } newUserClusterConfig.Telemetry = isTelemetryEnabled() cachedClusterConfig.Telemetry = newUserClusterConfig.Telemetry configureChanges, err := newUserClusterConfig.ValidateOnConfigure(awsClient, k8sClient, cachedClusterConfig, stacks.NodeGroupsStacks) if err != nil { err = errors.Append(err, fmt.Sprintf("\n\ncluster configuration schema can be found at https://docs.cortexlabs.com/v/%s/", consts.CortexVersionMinor)) return nil, clusterconfig.ConfigureChanges{}, errors.Wrap(err, newClusterConfigFile) } return newUserClusterConfig, configureChanges, nil } func confirmInstallClusterConfig(clusterConfig *clusterconfig.Config, awsClient *aws.Client, disallowPrompt bool) { eksPrice := aws.EKSPrices[clusterConfig.Region] operatorInstancePrice := aws.InstanceMetadatas[clusterConfig.Region]["t3.medium"].Price prometheusInstancePrice := aws.InstanceMetadatas[clusterConfig.Region][clusterConfig.PrometheusInstanceType].Price operatorEBSPrice := aws.EBSMetadatas[clusterConfig.Region]["gp3"].PriceGB * 20 / 30 / 24 prometheusEBSPrice := aws.EBSMetadatas[clusterConfig.Region]["gp3"].PriceGB * 20 / 30 / 24 metricsEBSPrice := aws.EBSMetadatas[clusterConfig.Region]["gp2"].PriceGB * (40 + 2) / 30 / 24 nlbPrice := aws.NLBMetadatas[clusterConfig.Region].Price elbPrice := aws.ELBMetadatas[clusterConfig.Region].Price natUnitPrice := aws.NATMetadatas[clusterConfig.Region].Price var loadBalancersPrice float64 usesELBForAPILoadBalancer := clusterConfig.APILoadBalancerType == clusterconfig.ELBLoadBalancerType if usesELBForAPILoadBalancer { loadBalancersPrice = nlbPrice + elbPrice } else { loadBalancersPrice = 2 * nlbPrice } var natTotalPrice float64 if clusterConfig.NATGateway == clusterconfig.SingleNATGateway { natTotalPrice = natUnitPrice } else if clusterConfig.NATGateway == clusterconfig.HighlyAvailableNATGateway { natTotalPrice = natUnitPrice * float64(len(clusterConfig.AvailabilityZones)) } headers := []table.Header{ {Title: "aws resource"}, {Title: "cost per hour"}, } var rows [][]interface{} rows = append(rows, []interface{}{"1 eks cluster", s.DollarsMaxPrecision(eksPrice)}) ngNameToSpotInstancesUsed := map[string]int{} fixedPrice := eksPrice + 2*(operatorInstancePrice+operatorEBSPrice) + prometheusInstancePrice + prometheusEBSPrice + metricsEBSPrice + loadBalancersPrice + natTotalPrice totalMinPrice := fixedPrice totalMaxPrice := fixedPrice for _, ng := range clusterConfig.NodeGroups { apiInstancePrice := aws.InstanceMetadatas[clusterConfig.Region][ng.InstanceType].Price apiEBSPrice := aws.EBSMetadatas[clusterConfig.Region][ng.InstanceVolumeType.String()].PriceGB * float64(ng.InstanceVolumeSize) / 30 / 24 if ng.InstanceVolumeType == clusterconfig.IO1VolumeType && ng.InstanceVolumeIOPS != nil { apiEBSPrice += aws.EBSMetadatas[clusterConfig.Region][ng.InstanceVolumeType.String()].PriceIOPS * float64(*ng.InstanceVolumeIOPS) / 30 / 24 } if ng.InstanceVolumeType == clusterconfig.GP3VolumeType && ng.InstanceVolumeIOPS != nil && ng.InstanceVolumeThroughput != nil { apiEBSPrice += libmath.MaxFloat64(0, (aws.EBSMetadatas[clusterConfig.Region][ng.InstanceVolumeType.String()].PriceIOPS-3000)*float64(*ng.InstanceVolumeIOPS)/30/24) apiEBSPrice += libmath.MaxFloat64(0, (aws.EBSMetadatas[clusterConfig.Region][ng.InstanceVolumeType.String()].PriceThroughput-125)*float64(*ng.InstanceVolumeThroughput)/30/24) } totalMaxPrice += float64(ng.MaxInstances) * (apiInstancePrice + apiEBSPrice) workerInstanceStr := fmt.Sprintf("nodegroup %s: %d-%d %s instances", ng.Name, ng.MinInstances, ng.MaxInstances, ng.InstanceType) if ng.MinInstances == ng.MaxInstances { workerInstanceStr = fmt.Sprintf("nodegroup %s: %d %s %s", ng.Name, ng.MinInstances, ng.InstanceType, s.PluralS("instance", ng.MinInstances)) } workerPriceStr := s.DollarsAndTenthsOfCents(apiInstancePrice+apiEBSPrice) + " each" if ng.Spot { ngNameToSpotInstancesUsed[ng.Name]++ spotPrice, err := awsClient.SpotInstancePrice(ng.InstanceType) workerPriceStr += " (spot pricing unavailable)" if err == nil && spotPrice != 0 { workerPriceStr = fmt.Sprintf("%s - %s each (varies based on spot price)", s.DollarsAndTenthsOfCents(spotPrice+apiEBSPrice), s.DollarsAndTenthsOfCents(apiInstancePrice+apiEBSPrice)) if ng.MinInstances > *ng.SpotConfig.OnDemandBaseCapacity { totalMinPrice += float64(ng.MinInstances-*ng.SpotConfig.OnDemandBaseCapacity)*(spotPrice+apiEBSPrice)*float64(100-*ng.SpotConfig.OnDemandPercentageAboveBaseCapacity)/100 + float64(ng.MinInstances-*ng.SpotConfig.OnDemandBaseCapacity)*(apiInstancePrice+apiEBSPrice)*float64(*ng.SpotConfig.OnDemandPercentageAboveBaseCapacity)/100 + float64(*ng.SpotConfig.OnDemandBaseCapacity)*(apiInstancePrice+apiEBSPrice) } else { totalMinPrice += float64(ng.MinInstances) * (apiInstancePrice + apiEBSPrice) } } else { totalMinPrice += float64(ng.MinInstances) * (apiInstancePrice + apiEBSPrice) } } else { totalMinPrice += float64(ng.MinInstances) * (apiInstancePrice + apiEBSPrice) } rows = append(rows, []interface{}{workerInstanceStr, workerPriceStr}) } operatorNodeGroupPrice := 2 * (operatorInstancePrice + operatorEBSPrice) prometheusNodeGroupPrice := prometheusInstancePrice + prometheusEBSPrice + metricsEBSPrice rows = append(rows, []interface{}{"2 t3.medium instances (cortex system)", s.DollarsAndTenthsOfCents(operatorNodeGroupPrice) + " total"}) rows = append(rows, []interface{}{fmt.Sprintf("1 %s instance (prometheus)", clusterConfig.PrometheusInstanceType), s.DollarsAndTenthsOfCents(prometheusNodeGroupPrice)}) if usesELBForAPILoadBalancer { rows = append(rows, []interface{}{"1 network load balancer", s.DollarsMaxPrecision(nlbPrice)}) rows = append(rows, []interface{}{"1 classic load balancer", s.DollarsMaxPrecision(elbPrice)}) } else { rows = append(rows, []interface{}{"2 network load balancers", s.DollarsMaxPrecision(loadBalancersPrice) + " total"}) } if clusterConfig.NATGateway == clusterconfig.SingleNATGateway { rows = append(rows, []interface{}{"1 nat gateway", s.DollarsMaxPrecision(natUnitPrice)}) } else if clusterConfig.NATGateway == clusterconfig.HighlyAvailableNATGateway { rows = append(rows, []interface{}{fmt.Sprintf("%d nat gateways", len(clusterConfig.AvailabilityZones)), s.DollarsMaxPrecision(natUnitPrice) + " each"}) } items := table.Table{ Headers: headers, Rows: rows, } fmt.Println(items.MustFormat(&table.Opts{Sort: pointer.Bool(false)})) priceStr := s.DollarsAndCents(totalMaxPrice) suffix := "" if totalMinPrice != totalMaxPrice { priceStr = fmt.Sprintf("%s - %s", s.DollarsAndCents(totalMinPrice), s.DollarsAndCents(totalMaxPrice)) if len(ngNameToSpotInstancesUsed) > 0 && len(ngNameToSpotInstancesUsed) < len(clusterConfig.NodeGroups) { suffix = " based on cluster size and spot instance pricing/availability" } else if len(ngNameToSpotInstancesUsed) == len(clusterConfig.NodeGroups) { suffix = " based on spot instance pricing/availability" } else if len(ngNameToSpotInstancesUsed) == 0 { suffix = " based on cluster size" } } fmt.Printf("your cluster will cost %s per hour%s\n\n", priceStr, suffix) privateSubnetMsg := "" if clusterConfig.SubnetVisibility == clusterconfig.PrivateSubnetVisibility { privateSubnetMsg = ", and will use private subnets for all EC2 instances" } fmt.Printf("cortex will also create an s3 bucket (%s) and a cloudwatch log group (%s)%s\n\n", clusterConfig.Bucket, clusterConfig.ClusterName, privateSubnetMsg) if clusterConfig.OperatorLoadBalancerScheme == clusterconfig.InternalLoadBalancerScheme { fmt.Print(fmt.Sprintf("warning: you've configured the operator load balancer to be internal; you must configure VPC Peering to connect your CLI to your cluster operator (see https://docs.cortexlabs.com/v/%s/)\n\n", consts.CortexVersionMinor)) } if len(clusterConfig.Subnets) > 0 { fmt.Print("warning: you've configured your cluster to be installed in an existing VPC; if your cluster doesn't spin up or function as expected, please double-check your VPC configuration (here are the requirements: https://eksctl.io/usage/vpc-networking/#use-existing-vpc-other-custom-configuration)\n\n") } if len(clusterConfig.NodeGroups) > 1 && len(ngNameToSpotInstancesUsed) > 0 { fmt.Printf("warning: you've enabled spot instances for %s %s; spot instances are not guaranteed to be available so please take that into account for production clusters; see https://docs.cortexlabs.com/v/%s/ for more information\n\n", s.PluralS("nodegroup", len(ngNameToSpotInstancesUsed)), s.StrsAnd(maps.StrMapKeysInt(ngNameToSpotInstancesUsed)), consts.CortexVersionMinor) } if !disallowPrompt { exitMessage := fmt.Sprintf("cluster configuration can be modified via the cluster config file; see https://docs.cortexlabs.com/v/%s/ for more information", consts.CortexVersionMinor) prompt.YesOrExit("would you like to continue?", "", exitMessage) } } func confirmConfigureClusterConfig(configureChanges clusterconfig.ConfigureChanges, oldCc, newCc clusterconfig.Config, disallowPrompt bool) { fmt.Printf("your %s cluster in region %s will be updated as follows:\n\n", newCc.ClusterName, newCc.Region) for _, fieldToUpdate := range configureChanges.FieldsToUpdate { fmt.Printf("￮ %s will be updated\n", fieldToUpdate) } for _, ngName := range configureChanges.NodeGroupsToUpdate { ngOld := oldCc.GetNodeGroupByName(ngName) ngNew := newCc.GetNodeGroupByName(ngName) fmt.Printf("￮ %s\n", ngNew.UpdatePlan(ngOld)) } for _, ngName := range configureChanges.NodeGroupsToAdd { fmt.Printf("￮ nodegroup %s will be added\n", ngName) } for _, ngName := range configureChanges.NodeGroupsToRemove { fmt.Printf("￮ nodegroup %s will be removed\n", ngName) } // EKS node groups that don't appear in the old/new cluster config; this is unlikely but can happen for _, ngName := range configureChanges.GetGhostEKSNodeGroups() { fmt.Printf("￮ EKS nodegroup %s will be removed\n", ngName) } fmt.Println() if !disallowPrompt { exitMessage := fmt.Sprintf("cluster configuration can be modified via the cluster config file; see https://docs.cortexlabs.com/v/%s/ for more information", consts.CortexVersionMinor) prompt.YesOrExit(fmt.Sprintf("your cluster named \"%s\" in %s will be updated according to the configuration above, are you sure you want to continue?", newCc.ClusterName, newCc.Region), "", exitMessage) } } ================================================ FILE: cli/cmd/lib_manager.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package cmd import ( "bytes" "context" "fmt" "io" "os" "os/signal" "path/filepath" "strings" "syscall" "time" "github.com/cortexlabs/cortex/cli/lib/routines" "github.com/cortexlabs/cortex/pkg/consts" "github.com/cortexlabs/cortex/pkg/lib/archive" "github.com/cortexlabs/cortex/pkg/lib/aws" "github.com/cortexlabs/cortex/pkg/lib/docker" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/exit" "github.com/cortexlabs/cortex/pkg/lib/files" "github.com/cortexlabs/cortex/pkg/types/clusterconfig" "github.com/cortexlabs/yaml" dockertypes "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" ) type dockerCopyFromPath struct { containerPath string localDir string } type dockerCopyToPath struct { input *archive.Input containerPath string } func runManager(containerConfig *container.Config, addNewLineAfterPull bool, copyToPaths []dockerCopyToPath, copyFromPaths []dockerCopyFromPath) (string, *int, error) { containerConfig.Env = append(containerConfig.Env, "CORTEX_CLI_VERSION="+consts.CortexVersion) // Add a slight delay before running the command to ensure logs don't start until after the container is attached containerConfig.Cmd[0] = "sleep 0.1 && /root/check_cortex_version.sh && " + containerConfig.Cmd[0] dockerClient, err := docker.GetDockerClient() if err != nil { return "", nil, err } pulledImage, err := docker.PullImage(containerConfig.Image, docker.NoAuth, docker.PrintDots) if err != nil { if strings.Contains(err.Error(), "auth") { err = errors.Append(err, fmt.Sprintf("\n\nif your manager image is stored in a private repository: run `docker login` (if you haven't already), download your image with `docker pull %s`, and try this command again)", containerConfig.Image)) } return "", nil, err } if pulledImage && addNewLineAfterPull { fmt.Println() } containerInfo, err := dockerClient.ContainerCreate(context.Background(), containerConfig, nil, nil, "") if err != nil { return "", nil, docker.WrapDockerError(err) } removeContainer := func() { _ = dockerClient.ContainerRemove(context.Background(), containerInfo.ID, dockertypes.ContainerRemoveOptions{ RemoveVolumes: true, Force: true, }) } defer removeContainer() // Make sure to remove container immediately on ctrl+c c := make(chan os.Signal, 1) signal.Notify(c, os.Interrupt, syscall.SIGTERM) caughtCtrlC := false routines.RunWithPanicHandler(func() { <-c caughtCtrlC = true removeContainer() exit.Error(ErrorDockerCtrlC()) }, false) for _, copyPath := range copyToPaths { err = docker.CopyToContainer(containerInfo.ID, copyPath.input, copyPath.containerPath) if err != nil { return "", nil, err } } err = dockerClient.ContainerStart(context.Background(), containerInfo.ID, dockertypes.ContainerStartOptions{}) if err != nil { return "", nil, docker.WrapDockerError(err) } // Use ContainerAttach() since that allows logs to be streamed even if they don't end in new lines logsOutput, err := dockerClient.ContainerAttach(context.Background(), containerInfo.ID, dockertypes.ContainerAttachOptions{ Stream: true, Stdout: true, Stderr: true, }) if err != nil { return "", nil, docker.WrapDockerError(err) } defer logsOutput.Close() var outputBuffer bytes.Buffer tee := io.TeeReader(logsOutput.Reader, &outputBuffer) _, err = io.Copy(os.Stdout, tee) if err != nil && err != io.EOF { return "", nil, errors.WithStack(err) } output := strings.ReplaceAll(outputBuffer.String(), "\r\n", "\n") // Let the ctrl+c handler run its course if caughtCtrlC { time.Sleep(5 * time.Second) } info, err := dockerClient.ContainerInspect(context.Background(), containerInfo.ID) if err != nil { return "", nil, errors.WithStack(err) } if info.State.ExitCode == 0 { for _, copyPath := range copyFromPaths { err = docker.CopyFromContainer(containerInfo.ID, copyPath.containerPath, copyPath.localDir) if err != nil { return "", nil, err } } } if info.State.Running { return output, nil, nil } return output, &info.State.ExitCode, nil } func runManagerWithClusterConfig(entrypoint string, clusterConfig *clusterconfig.Config, awsClient *aws.Client, copyToPaths []dockerCopyToPath, copyFromPaths []dockerCopyFromPath, extraEnvs []string) (string, *int, error) { clusterConfigBytes, err := yaml.Marshal(clusterConfig) if err != nil { return "", nil, errors.WithStack(err) } cachedClusterConfigPath := getCachedClusterConfigPath(clusterConfig.ClusterName, clusterConfig.Region) if err := files.WriteFile(clusterConfigBytes, cachedClusterConfigPath); err != nil { return "", nil, err } containerClusterConfigPath := "/in/" + filepath.Base(cachedClusterConfigPath) copyToPaths = append(copyToPaths, dockerCopyToPath{ input: &archive.Input{ Files: []archive.FileInput{ { Source: cachedClusterConfigPath, Dest: containerClusterConfigPath, }, }, }, containerPath: "/", }) envs := []string{ "AWS_ACCESS_KEY_ID=" + *awsClient.AccessKeyID(), "AWS_SECRET_ACCESS_KEY=" + *awsClient.SecretAccessKey(), "CORTEX_TELEMETRY_DISABLE=" + os.Getenv("CORTEX_TELEMETRY_DISABLE"), "CORTEX_TELEMETRY_SENTRY_DSN=" + os.Getenv("CORTEX_TELEMETRY_SENTRY_DSN"), "CORTEX_TELEMETRY_SEGMENT_WRITE_KEY=" + os.Getenv("CORTEX_TELEMETRY_SEGMENT_WRITE_KEY"), "CORTEX_DEV_DEFAULT_IMAGE_REGISTRY=" + os.Getenv("CORTEX_DEV_DEFAULT_IMAGE_REGISTRY"), "CORTEX_DEV_ADD_CONTROL_PLANE_DASHBOARD=" + os.Getenv("CORTEX_DEV_ADD_CONTROL_PLANE_DASHBOARD"), "CORTEX_CLUSTER_CONFIG_FILE=" + containerClusterConfigPath, } envs = append(envs, extraEnvs...) containerConfig := &container.Config{ Image: clusterConfig.ImageManager, Entrypoint: []string{"/bin/bash", "-c"}, Cmd: []string{fmt.Sprintf("eval $(python /root/cluster_config_env.py %s) && %s", containerClusterConfigPath, entrypoint)}, Tty: true, AttachStdout: true, AttachStderr: true, Env: envs, } if sessionToken := awsClient.SessionToken(); sessionToken != nil { containerConfig.Env = append(containerConfig.Env, "AWS_SESSION_TOKEN="+*sessionToken) } output, exitCode, err := runManager(containerConfig, false, copyToPaths, copyFromPaths) if err != nil { return "", nil, err } return output, exitCode, nil } func runManagerAccessCommand(entrypoint string, accessConfig clusterconfig.AccessConfig, awsClient *aws.Client, copyToPaths []dockerCopyToPath, copyFromPaths []dockerCopyFromPath) (string, *int, error) { containerConfig := &container.Config{ Image: accessConfig.ImageManager, Entrypoint: []string{"/bin/bash", "-c"}, Cmd: []string{entrypoint}, Tty: true, AttachStdout: true, AttachStderr: true, Env: []string{ "AWS_ACCESS_KEY_ID=" + *awsClient.AccessKeyID(), "AWS_SECRET_ACCESS_KEY=" + *awsClient.SecretAccessKey(), "CORTEX_CLUSTER_NAME=" + accessConfig.ClusterName, "CORTEX_REGION=" + accessConfig.Region, "CORTEX_TELEMETRY_DISABLE=" + os.Getenv("CORTEX_TELEMETRY_DISABLE"), "CORTEX_TELEMETRY_SENTRY_DSN=" + os.Getenv("CORTEX_TELEMETRY_SENTRY_DSN"), "CORTEX_TELEMETRY_SEGMENT_WRITE_KEY=" + os.Getenv("CORTEX_TELEMETRY_SEGMENT_WRITE_KEY"), }, } if sessionToken := awsClient.SessionToken(); sessionToken != nil { containerConfig.Env = append(containerConfig.Env, "AWS_SESSION_TOKEN="+*sessionToken) } output, exitCode, err := runManager(containerConfig, true, copyToPaths, copyFromPaths) if err != nil { return "", nil, err } return output, exitCode, nil } ================================================ FILE: cli/cmd/lib_realtime_apis.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package cmd import ( "fmt" "strings" "time" "github.com/cortexlabs/cortex/cli/types/cliconfig" "github.com/cortexlabs/cortex/pkg/lib/console" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/table" libtime "github.com/cortexlabs/cortex/pkg/lib/time" "github.com/cortexlabs/cortex/pkg/operator/schema" ) func realtimeAPITable(realtimeAPI schema.APIResponse, env cliconfig.Environment) (string, error) { var out string t := realtimeAPIsTable([]schema.APIResponse{realtimeAPI}, []string{env.Name}) out += t.MustFormat() if realtimeAPI.DashboardURL != nil && *realtimeAPI.DashboardURL != "" { out += "\n" + console.Bold("metrics dashboard: ") + *realtimeAPI.DashboardURL + "\n" } if realtimeAPI.Endpoint != nil { out += "\n" + console.Bold("endpoint: ") + *realtimeAPI.Endpoint + "\n" } out += "\n" + apiHistoryTable(realtimeAPI.APIVersions) if !_flagVerbose { return out, nil } out += titleStr("configuration") + strings.TrimSpace(realtimeAPI.Spec.UserStr()) return out, nil } func realtimeDescribeAPITable(realtimeAPI schema.APIResponse, env cliconfig.Environment) (string, error) { if realtimeAPI.Metadata == nil { return "", errors.ErrorUnexpected("missing metadata from operator response") } if realtimeAPI.Status == nil { return "", errors.ErrorUnexpected(fmt.Sprintf("missing status for %s api", realtimeAPI.Metadata.Name)) } t := realtimeAPIsTable([]schema.APIResponse{realtimeAPI}, []string{env.Name}) out := t.MustFormat() if realtimeAPI.DashboardURL != nil && *realtimeAPI.DashboardURL != "" { out += "\n" + console.Bold("metrics dashboard: ") + *realtimeAPI.DashboardURL + "\n" } if realtimeAPI.Endpoint != nil { out += "\n" + console.Bold("endpoint: ") + *realtimeAPI.Endpoint + "\n" } t = replicaCountTable(realtimeAPI.Status.ReplicaCounts) out += "\n" + t.MustFormat() return out, nil } func realtimeAPIsTable(realtimeAPIs []schema.APIResponse, envNames []string) table.Table { rows := make([][]interface{}, 0, len(realtimeAPIs)) for i, realtimeAPI := range realtimeAPIs { if realtimeAPI.Metadata == nil || realtimeAPI.Status == nil { continue } lastUpdated := time.Unix(realtimeAPI.Metadata.LastUpdated, 0) rows = append(rows, []interface{}{ envNames[i], realtimeAPI.Metadata.Name, fmt.Sprintf("%d/%d", realtimeAPI.Status.Ready, realtimeAPI.Status.Requested), realtimeAPI.Status.UpToDate, libtime.SinceStr(&lastUpdated), }) } return table.Table{ Headers: []table.Header{ {Title: _titleEnvironment}, {Title: _titleRealtimeAPI}, {Title: _titleLive}, {Title: _titleUpToDate}, {Title: _titleLastUpdated}, }, Rows: rows, } } ================================================ FILE: cli/cmd/lib_task_apis.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package cmd import ( "fmt" "time" "github.com/cortexlabs/cortex/cli/cluster" "github.com/cortexlabs/cortex/cli/types/cliconfig" "github.com/cortexlabs/cortex/cli/types/flags" "github.com/cortexlabs/cortex/pkg/lib/console" libjson "github.com/cortexlabs/cortex/pkg/lib/json" "github.com/cortexlabs/cortex/pkg/lib/pointer" "github.com/cortexlabs/cortex/pkg/lib/table" libtime "github.com/cortexlabs/cortex/pkg/lib/time" "github.com/cortexlabs/cortex/pkg/operator/schema" "github.com/cortexlabs/yaml" ) const ( _titleTaskAPI = "task api" _titleTaskJobCount = "running jobs" _titleLatestTaskJobID = "latest job id" ) func taskAPIsTable(taskAPIs []schema.APIResponse, envNames []string) table.Table { rows := make([][]interface{}, 0, len(taskAPIs)) for i, taskAPI := range taskAPIs { if taskAPI.Metadata == nil { continue } lastAPIUpdated := time.Unix(taskAPI.Metadata.LastUpdated, 0) latestStartTime := time.Time{} latestJobID := "-" runningJobs := 0 for _, job := range taskAPI.TaskJobStatuses { if job.StartTime.After(latestStartTime) { latestStartTime = job.StartTime latestJobID = job.ID + fmt.Sprintf(" (submitted %s ago)", libtime.SinceStr(&latestStartTime)) } if job.Status.IsInProgress() { runningJobs++ } } rows = append(rows, []interface{}{ envNames[i], taskAPI.Metadata.Name, runningJobs, latestJobID, libtime.SinceStr(&lastAPIUpdated), }) } return table.Table{ Headers: []table.Header{ {Title: _titleEnvironment}, {Title: _titleTaskAPI}, {Title: _titleTaskJobCount}, {Title: _titleLatestTaskJobID}, {Title: _titleLastUpdated}, }, Rows: rows, } } func taskAPITable(taskAPI schema.APIResponse) string { jobRows := make([][]interface{}, 0, len(taskAPI.TaskJobStatuses)) out := "" if len(taskAPI.TaskJobStatuses) == 0 { out = console.Bold("no submitted task jobs\n") } else { for _, job := range taskAPI.TaskJobStatuses { jobEndTime := time.Now() if job.EndTime != nil { jobEndTime = *job.EndTime } duration := jobEndTime.Sub(job.StartTime).Truncate(time.Second).String() jobRows = append(jobRows, []interface{}{ job.ID, job.Status.Message(), job.StartTime.Format(_timeFormat), duration, }) } t := table.Table{ Headers: []table.Header{ {Title: "task job id"}, {Title: "status"}, {Title: "start time"}, {Title: "duration"}, }, Rows: jobRows, } out += t.MustFormat() } if taskAPI.DashboardURL != nil && *taskAPI.DashboardURL != "" { out += "\n" + console.Bold("metrics dashboard: ") + *taskAPI.DashboardURL + "\n" } if taskAPI.Endpoint != nil { out += "\n" + console.Bold("endpoint: ") + *taskAPI.Endpoint + "\n" } out += "\n" + apiHistoryTable(taskAPI.APIVersions) if !_flagVerbose { return out } out += titleStr("task api configuration") + taskAPI.Spec.UserStr() return out } func getTaskJob(env cliconfig.Environment, apiName string, jobID string) (string, error) { resp, err := cluster.GetTaskJob(MustGetOperatorConfig(env.Name), apiName, jobID) if err != nil { return "", err } var bytes []byte if _flagOutput == flags.JSONOutputType { bytes, err = libjson.Marshal(resp) } else if _flagOutput == flags.YAMLOutputType { bytes, err = yaml.Marshal(resp) } if err != nil { return "", err } if _flagOutput == flags.JSONOutputType || _flagOutput == flags.YAMLOutputType { return string(bytes), nil } job := resp.JobStatus out := "" jobIntroTable := table.KeyValuePairs{} jobIntroTable.Add("job id", job.ID) jobIntroTable.Add("status", job.Status.Message()) out += jobIntroTable.String(&table.KeyValuePairOpts{BoldKeys: pointer.Bool(true)}) jobTimingTable := table.KeyValuePairs{} jobTimingTable.Add("start time", job.StartTime.Format(_timeFormat)) jobEndTime := time.Now() if job.EndTime != nil { jobTimingTable.Add("end time", job.EndTime.Format(_timeFormat)) jobEndTime = *job.EndTime } else { jobTimingTable.Add("end time", "-") } duration := jobEndTime.Sub(job.StartTime).Truncate(time.Second).String() jobTimingTable.Add("duration", duration) out += "\n" + jobTimingTable.String(&table.KeyValuePairOpts{BoldKeys: pointer.Bool(true)}) if job.Status.IsCompleted() { out += "\n" + "worker stats are not available because this job is not currently running\n" } else { out += titleStr("worker stats") if job.WorkerCounts != nil { t := table.Table{ Headers: []table.Header{ {Title: "Requested"}, {Title: "Pending"}, {Title: "Creating"}, {Title: "Ready"}, {Title: "NotReady"}, {Title: "ErrImagePull", Hidden: job.WorkerCounts.ErrImagePull == 0}, {Title: "Terminating", Hidden: job.WorkerCounts.Terminating == 0}, {Title: "Failed", Hidden: job.WorkerCounts.Failed == 0}, {Title: "Killed", Hidden: job.WorkerCounts.Killed == 0}, {Title: "KilledOOM", Hidden: job.WorkerCounts.KilledOOM == 0}, {Title: "Stalled", Hidden: job.WorkerCounts.Stalled == 0}, {Title: "Unknown", Hidden: job.WorkerCounts.Unknown == 0}, {Title: "Succeeded"}, }, Rows: [][]interface{}{ { job.Workers, job.WorkerCounts.Pending, job.WorkerCounts.Creating, job.WorkerCounts.Ready, job.WorkerCounts.NotReady, job.WorkerCounts.ErrImagePull, job.WorkerCounts.Terminating, job.WorkerCounts.Failed, job.WorkerCounts.Killed, job.WorkerCounts.KilledOOM, job.WorkerCounts.Stalled, job.WorkerCounts.Unknown, job.WorkerCounts.Succeeded, }, }, } out += t.MustFormat(&table.Opts{BoldHeader: pointer.Bool(false)}) } else { out += "unable to get worker stats\n" } } out += "\n" + console.Bold("job endpoint: ") + resp.Endpoint + "\n" jobSpecStr, err := libjson.Pretty(job.TaskJob) if err != nil { return "", err } out += titleStr("job configuration") + jobSpecStr return out, nil } ================================================ FILE: cli/cmd/lib_traffic_splitters.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package cmd import ( "fmt" "strings" "time" "github.com/cortexlabs/cortex/cli/cluster" "github.com/cortexlabs/cortex/cli/types/cliconfig" "github.com/cortexlabs/cortex/pkg/lib/console" s "github.com/cortexlabs/cortex/pkg/lib/strings" "github.com/cortexlabs/cortex/pkg/lib/table" libtime "github.com/cortexlabs/cortex/pkg/lib/time" "github.com/cortexlabs/cortex/pkg/operator/schema" ) const ( _titleTrafficSplitter = "traffic splitter" _trafficSplitterWeights = "weights" _titleAPIs = "apis" ) func trafficSplitterTable(trafficSplitter schema.APIResponse, env cliconfig.Environment) (string, error) { var out string lastUpdated := time.Unix(trafficSplitter.Spec.LastUpdated, 0) t, err := trafficSplitTable(trafficSplitter, env) if err != nil { return "", err } out += t.MustFormat() out += "\n" + console.Bold("last updated: ") + libtime.SinceStr(&lastUpdated) if trafficSplitter.Endpoint != nil { out += "\n" + console.Bold("endpoint: ") + *trafficSplitter.Endpoint + "\n" } out += "\n" + apiHistoryTable(trafficSplitter.APIVersions) if !_flagVerbose { return out, nil } out += titleStr("configuration") + strings.TrimSpace(trafficSplitter.Spec.UserStr()) return out, nil } func trafficSplitTable(trafficSplitter schema.APIResponse, env cliconfig.Environment) (table.Table, error) { rows := make([][]interface{}, 0, len(trafficSplitter.Spec.APIs)) for _, api := range trafficSplitter.Spec.APIs { apisRes, err := cluster.GetAPI(MustGetOperatorConfig(env.Name), api.Name) if err != nil { return table.Table{}, err } apiRes := apisRes[0] if apiRes.Metadata == nil || apiRes.Status == nil { continue } lastUpdated := time.Unix(apiRes.Metadata.LastUpdated, 0) apiName := apiRes.Metadata.Name if api.Shadow { apiName += " (shadow)" } rows = append(rows, []interface{}{ env.Name, apiName, api.Weight, fmt.Sprintf("%d/%d", apiRes.Status.Ready, apiRes.Status.Requested), apiRes.Status.UpToDate, libtime.SinceStr(&lastUpdated), }) } return table.Table{ Headers: []table.Header{ {Title: _titleEnvironment}, {Title: _titleAPIs}, {Title: _trafficSplitterWeights}, {Title: _titleLive}, {Title: _titleUpToDate}, {Title: _titleLastUpdated}, }, Rows: rows, }, nil } func trafficSplitterListTable(trafficSplitter []schema.APIResponse, envNames []string) table.Table { rows := make([][]interface{}, 0, len(trafficSplitter)) for i, splitAPI := range trafficSplitter { if splitAPI.Metadata == nil || splitAPI.NumTrafficSplitterTargets == nil { continue } lastUpdated := time.Unix(splitAPI.Metadata.LastUpdated, 0) rows = append(rows, []interface{}{ envNames[i], splitAPI.Metadata.Name, s.Int32(*splitAPI.NumTrafficSplitterTargets), libtime.SinceStr(&lastUpdated), }) } return table.Table{ Headers: []table.Header{ {Title: _titleEnvironment}, {Title: _titleTrafficSplitter}, {Title: _titleAPIs}, {Title: _titleLastUpdated}, }, Rows: rows, } } ================================================ FILE: cli/cmd/lib_watch.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package cmd import ( "fmt" "os" "os/exec" "strings" "time" "github.com/cortexlabs/cortex/pkg/lib/exit" libmath "github.com/cortexlabs/cortex/pkg/lib/math" s "github.com/cortexlabs/cortex/pkg/lib/strings" libtime "github.com/cortexlabs/cortex/pkg/lib/time" ) func getTerminalWidth() int { cmd := exec.Command("stty", "size") cmd.Stdin = os.Stdin out, err := cmd.Output() if err != nil { return 0 } dimensions := strings.Split(strings.TrimSpace(string(out)), " ") if len(dimensions) != 2 { return 0 } widthStr := dimensions[1] width, ok := s.ParseInt(widthStr) if !ok { return 0 } return width } func watchHeader() string { timeStr := libtime.LocalHourNow() width := getTerminalWidth() numExtraChars := 4 padding := strings.Repeat(" ", libmath.MaxInt(width-len(_cmdStr)-len(timeStr)-numExtraChars, 0)) return fmt.Sprintf("$ %s %s%s", _cmdStr, padding, libtime.LocalHourNow()) } func rerun(watchFlag bool, f func() (string, error)) { if watchFlag { print("\033[H\033[2J") // clear the screen var prevStrSlice []string for true { nextStr, err := f() if err != nil { fmt.Println() exit.Error(err) } nextStr = watchHeader() + "\n\n" + s.EnsureSingleTrailingNewLine(nextStr) nextStrSlice := strings.Split(nextStr, "\n") terminalWidth := getTerminalWidth() if terminalWidth <= 0 { exit.Error(ErrorNoTerminalWidth()) } nextNumLines := 0 for _, strLine := range nextStrSlice { nextNumLines += (len(strLine)-1)/terminalWidth + 1 } prevNumLines := 0 for _, strLine := range prevStrSlice { prevNumLines += (len(strLine)-1)/terminalWidth + 1 } for i := prevNumLines; i > nextNumLines; i-- { fmt.Printf("\033[%dA\033[2K", 1) // move the cursor up and clear the line } for i := 0; i < prevNumLines; i++ { fmt.Printf("\033[%dA", 1) // move the cursor up } for _, strLine := range nextStrSlice { fmt.Printf("\033[2K%s\n", strLine) // clear the line and print the new line } prevStrSlice = nextStrSlice time.Sleep(time.Second * 2) } } else { str, err := f() if err != nil { exit.Error(err) } fmt.Print(s.EnsureSingleTrailingNewLine(str)) } } ================================================ FILE: cli/cmd/logs.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package cmd import ( "fmt" "github.com/cortexlabs/cortex/cli/cluster" "github.com/cortexlabs/cortex/pkg/lib/exit" "github.com/cortexlabs/cortex/pkg/lib/telemetry" "github.com/spf13/cobra" ) var ( _flagLogsEnv string _flagLogsDisallowPrompt bool _flagRandomPod bool _logsOutput = `Navigate to the link below and click "Run Query": %s NOTE: there may be 1-2 minutes of delay for the logs to show up in the results of CloudWatch Insight queries ` ) func logsInit() { _logsCmd.Flags().SortFlags = false _logsCmd.Flags().StringVarP(&_flagLogsEnv, "env", "e", "", "environment to use") _logsCmd.Flags().BoolVarP(&_flagLogsDisallowPrompt, "yes", "y", false, "skip prompts") _logsCmd.Flags().BoolVarP(&_flagRandomPod, "random-pod", "", false, "stream logs from a random pod") } var _logsCmd = &cobra.Command{ Use: "logs API_NAME [JOB_ID]", Short: "get the logs for a workload", Args: cobra.RangeArgs(1, 2), Run: func(cmd *cobra.Command, args []string) { envName, err := getEnvFromFlag(_flagLogsEnv) if err != nil { telemetry.Event("cli.logs") exit.Error(err) } env, err := ReadOrConfigureEnv(envName) if err != nil { telemetry.Event("cli.logs") exit.Error(err) } telemetry.Event("cli.logs", map[string]interface{}{"env_name": env.Name, "random_pod": _flagRandomPod}) err = printEnvIfNotSpecified(env.Name, cmd) if err != nil { exit.Error(err) } operatorConfig := MustGetOperatorConfig(env.Name) apiName := args[0] if len(args) == 1 { if _flagRandomPod { err := cluster.StreamLogs(operatorConfig, apiName) if err != nil { exit.Error(err) } return } logResponse, err := cluster.GetLogs(operatorConfig, apiName) if err != nil { exit.Error(err) } fmt.Printf(_logsOutput, logResponse.LogURL) return } jobID := args[1] if _flagRandomPod { err := cluster.StreamJobLogs(operatorConfig, apiName, jobID) if err != nil { exit.Error(err) } return } logResponse, err := cluster.GetJobLogs(operatorConfig, apiName, jobID) if err != nil { exit.Error(err) } fmt.Printf(_logsOutput, logResponse.LogURL) }, } ================================================ FILE: cli/cmd/refresh.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package cmd import ( "fmt" "strings" "github.com/cortexlabs/cortex/cli/cluster" "github.com/cortexlabs/cortex/cli/types/flags" "github.com/cortexlabs/cortex/pkg/lib/exit" libjson "github.com/cortexlabs/cortex/pkg/lib/json" "github.com/cortexlabs/cortex/pkg/lib/print" "github.com/cortexlabs/cortex/pkg/lib/telemetry" "github.com/spf13/cobra" ) var ( _flagRefreshEnv string _flagRefreshForce bool ) func refreshInit() { _refreshCmd.Flags().SortFlags = false _refreshCmd.Flags().StringVarP(&_flagRefreshEnv, "env", "e", "", "environment to use") _refreshCmd.Flags().BoolVarP(&_flagRefreshForce, "force", "f", false, "override the in-progress api update") _refreshCmd.Flags().VarP(&_flagOutput, "output", "o", fmt.Sprintf("output format: one of %s", strings.Join(flags.OutputTypeStringsExcluding(flags.YAMLOutputType), "|"))) } var _refreshCmd = &cobra.Command{ Use: "refresh API_NAME", Short: "restart all replicas for an api (without downtime)", Args: cobra.ExactArgs(1), Run: func(cmd *cobra.Command, args []string) { envName, err := getEnvFromFlag(_flagRefreshEnv) if err != nil { telemetry.Event("cli.refresh") exit.Error(err) } env, err := ReadOrConfigureEnv(envName) if err != nil { telemetry.Event("cli.refresh") exit.Error(err) } telemetry.Event("cli.refresh", map[string]interface{}{"env_name": env.Name}) err = printEnvIfNotSpecified(env.Name, cmd) if err != nil { exit.Error(err) } refreshResponse, err := cluster.Refresh(MustGetOperatorConfig(env.Name), args[0], _flagRefreshForce) if err != nil { exit.Error(err) } if _flagOutput == flags.JSONOutputType { bytes, err := libjson.Marshal(refreshResponse) if err != nil { exit.Error(err) } fmt.Print(string(bytes)) return } print.BoldFirstLine(refreshResponse.Message) }, } ================================================ FILE: cli/cmd/root.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package cmd import ( "fmt" "os" "path/filepath" "strings" "github.com/cortexlabs/cortex/cli/types/flags" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/exit" "github.com/cortexlabs/cortex/pkg/lib/files" s "github.com/cortexlabs/cortex/pkg/lib/strings" "github.com/cortexlabs/cortex/pkg/lib/telemetry" homedir "github.com/mitchellh/go-homedir" "github.com/spf13/cobra" "github.com/spf13/pflag" ) var ( _cmdStr string _configFileExts = []string{"yaml", "yml"} _flagVerbose bool _flagOutput = flags.PrettyOutputType _credentialsCacheDir string _localDir string _cliConfigPath string _clientIDPath string _emailPath string _debugPath string _cwd string _homeDir string ) func init() { cwd, err := os.Getwd() if err != nil { err := errors.Wrap(err, "unable to determine current working directory") exit.Error(err) } _cwd = s.EnsureSuffix(cwd, "/") homeDir, err := homedir.Dir() if err != nil { err := errors.Wrap(err, "unable to determine home directory") exit.Error(err) } _homeDir = s.EnsureSuffix(homeDir, "/") _localDir = os.Getenv("CORTEX_CLI_CONFIG_DIR") if _localDir != "" { _localDir = files.UserRelToAbsPath(_localDir) } else { _localDir = filepath.Join(homeDir, ".cortex") } err = os.MkdirAll(_localDir, os.ModePerm) if err != nil { err := errors.Wrap(err, "unable to write to home directory", _localDir) exit.Error(err) } // ~/.cortex/credentials/ _credentialsCacheDir = filepath.Join(_localDir, "credentials") err = os.MkdirAll(_credentialsCacheDir, os.ModePerm) if err != nil { err := errors.Wrap(err, "unable to write to home directory", _localDir) exit.Error(err) } _cliConfigPath = filepath.Join(_localDir, "cli.yaml") _clientIDPath = filepath.Join(_localDir, "client-id.txt") _emailPath = filepath.Join(_localDir, "email.txt") _debugPath = filepath.Join(_localDir, "cortex-debug.tgz") cobra.EnablePrefixMatching = true _cmdStr = "cortex" for _, arg := range os.Args[1:] { if arg == "-w" || arg == "--watch" { continue } _cmdStr += " " + arg } enableTelemetry, err := readTelemetryConfig() if err != nil { exit.Error(err) } if enableTelemetry { initTelemetry() } clusterInit() completionInit() deleteInit() describeInit() deployInit() envInit() getInit() logsInit() refreshInit() versionInit() } func initTelemetry() { cID := clientID() invoker := os.Getenv("CORTEX_CLI_INVOKER") if invoker == "" { invoker = "direct" } telemetry.Init(telemetry.Config{ Enabled: true, UserID: cID, Properties: map[string]string{ "client_id": cID, "invoker": invoker, }, Environment: "cli", LogErrors: false, BackoffMode: telemetry.NoBackoff, }) } var _rootCmd = &cobra.Command{ Use: "cortex", Aliases: []string{"cx"}, Short: "cost-effective serverless computing", } func Execute() { defer exit.RecoverAndExit() cobra.EnableCommandSorting = false _rootCmd.AddCommand(_deployCmd) _rootCmd.AddCommand(_getCmd) _rootCmd.AddCommand(_describeCmd) _rootCmd.AddCommand(_logsCmd) _rootCmd.AddCommand(_refreshCmd) _rootCmd.AddCommand(_deleteCmd) _rootCmd.AddCommand(_clusterCmd) _rootCmd.AddCommand(_envCmd) _rootCmd.AddCommand(_versionCmd) _rootCmd.AddCommand(_completionCmd) updateRootUsage() _rootCmd.Execute() exit.Ok() } func updateRootUsage() { defaultUsageFunc := _rootCmd.UsageFunc() usage := _rootCmd.UsageString() _rootCmd.SetUsageFunc(func(cmd *cobra.Command) error { if cmd != _rootCmd { return defaultUsageFunc(cmd) } usage = strings.Replace(usage, "Usage:\n cortex [command]\n\nAliases:\n cortex, cx\n\n", "", 1) usage = strings.Replace(usage, "Available Commands:", "api commands:", 1) usage = strings.Replace(usage, "\n cluster", "\n\ncluster commands:\n cluster", 1) usage = strings.Replace(usage, "\n env ", "\n\nother commands:\n env ", 1) usage = strings.Replace(usage, "\n\nUse \"cortex [command] --help\" for more information about a command.", "", 1) cmd.Print(usage) return nil }) } func addVerboseFlag(cmd *cobra.Command) { cmd.Flags().BoolVarP(&_flagVerbose, "verbose", "v", false, "show additional information (only applies to pretty output format)") } func wasFlagProvided(cmd *cobra.Command, flagName string) bool { flagWasProvided := false cmd.Flags().VisitAll(func(flag *pflag.Flag) { if flag.Name == flagName && flag.Changed && flag.Value.String() != "" { flagWasProvided = true } }) return flagWasProvided } func printEnvIfNotSpecified(envName string, cmd *cobra.Command) error { out, err := envStringIfNotSpecified(envName, cmd) if err != nil { return err } if out != "" { fmt.Print(out) } return nil } func envStringIfNotSpecified(envName string, cmd *cobra.Command) (string, error) { envNames, err := listConfiguredEnvNames() if err != nil { return "", err } if _flagOutput == flags.PrettyOutputType && !wasFlagProvided(cmd, "env") && len(envNames) > 1 { return fmt.Sprintf("using %s environment\n\n", envName), nil } return "", nil } ================================================ FILE: cli/cmd/version.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package cmd import ( "fmt" "github.com/cortexlabs/cortex/cli/cluster" "github.com/cortexlabs/cortex/pkg/consts" "github.com/cortexlabs/cortex/pkg/lib/exit" "github.com/cortexlabs/cortex/pkg/lib/telemetry" "github.com/spf13/cobra" ) var _flagVersionEnv string func versionInit() { _versionCmd.Flags().SortFlags = false _versionCmd.Flags().StringVarP(&_flagVersionEnv, "env", "e", "", "environment to use") } var _versionCmd = &cobra.Command{ Use: "version", Short: "print the cli and cluster versions", Args: cobra.NoArgs, Run: func(cmd *cobra.Command, args []string) { envName, err := getEnvFromFlag(_flagVersionEnv) if err != nil { telemetry.Event("cli.version") fmt.Println("cli version: " + consts.CortexVersion) return } env, err := ReadOrConfigureEnv(envName) if err != nil { telemetry.Event("cli.version") exit.Error(err) } telemetry.Event("cli.version", map[string]interface{}{"env_name": env.Name}) err = printEnvIfNotSpecified(env.Name, cmd) if err != nil { exit.Error(err) } fmt.Println("cli version: " + consts.CortexVersion) infoResponse, err := cluster.Info(MustGetOperatorConfig(env.Name)) if err != nil { exit.Error(err) } fmt.Println("cluster version: " + infoResponse.ClusterConfig.APIVersion) }, } ================================================ FILE: cli/lib/routines/routines.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package routines import ( "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/exit" "github.com/cortexlabs/cortex/pkg/lib/telemetry" ) func RunWithPanicHandler(f func(), exitOnPanic bool) { go func() { defer func() { if r := recover(); r != nil { err := errors.CastRecoverError(r) errors.PrintStacktrace(err) if exitOnPanic { exit.Error(err) } telemetry.Error(err) } }() f() }() } ================================================ FILE: cli/main.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package main import ( "github.com/cortexlabs/cortex/cli/cmd" ) func main() { cmd.Execute() } ================================================ FILE: cli/types/cliconfig/cli_config.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package cliconfig import ( "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/sets/strset" ) type CLIConfig struct { Telemetry *bool `json:"telemetry,omitempty" yaml:"telemetry,omitempty"` DefaultEnvironment *string `json:"default_environment" yaml:"default_environment"` Environments []*Environment `json:"environments" yaml:"environments"` } type UserFacingCLIConfig struct { DefaultEnvironment *string `json:"default_environment" yaml:"default_environment"` Environments []*Environment `json:"environments" yaml:"environments"` } func (cliConfig *CLIConfig) Validate() error { envNames := strset.New() for _, env := range cliConfig.Environments { if envNames.Has(env.Name) { return errors.Wrap(ErrorDuplicateEnvironmentNames(env.Name), EnvironmentsKey) } envNames.Add(env.Name) if err := env.Validate(); err != nil { return errors.Wrap(err, EnvironmentsKey) } } // Backwards compatibility: ignore local default env defaultEnv := cliConfig.DefaultEnvironment if defaultEnv != nil && *defaultEnv == "local" && !envNames.Has(*defaultEnv) { cliConfig.DefaultEnvironment = nil } return nil } func (cliConfig *CLIConfig) ConvertToUserFacingCLIConfig() UserFacingCLIConfig { envs := cliConfig.Environments if envs == nil { envs = []*Environment{} } return UserFacingCLIConfig{ DefaultEnvironment: cliConfig.DefaultEnvironment, Environments: envs, } } ================================================ FILE: cli/types/cliconfig/config_key.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package cliconfig const ( EnvironmentsKey = "environments" DefaultEnvironmentKey = "default_environment" NameKey = "name" OperatorEndpointKey = "operator_endpoint" ) ================================================ FILE: cli/types/cliconfig/environment.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package cliconfig import ( "fmt" "strings" cr "github.com/cortexlabs/cortex/pkg/lib/configreader" "github.com/cortexlabs/cortex/pkg/lib/console" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/urls" ) type Environment struct { Name string `json:"name" yaml:"name"` OperatorEndpoint string `json:"operator_endpoint" yaml:"operator_endpoint"` } func (env Environment) String(isDefault bool) string { var envStr string if isDefault { envStr += console.Bold(env.Name + " (default)") } else { envStr += console.Bold(env.Name) } envStr += fmt.Sprintf("\ncortex operator endpoint: %s\n", env.OperatorEndpoint) return envStr } func CortexEndpointValidator(val string) (string, error) { urlStr := strings.TrimSpace(val) parsedURL, err := urls.Parse(urlStr) if err != nil { return "", err } // default https if parsedURL.Scheme == "" { parsedURL.Scheme = "https" } return parsedURL.String(), nil } func (env *Environment) Validate() error { if env.Name == "" { return errors.Wrap(cr.ErrorMustBeDefined(), NameKey) } validOperatorURL, err := CortexEndpointValidator(env.OperatorEndpoint) if err != nil { return err } env.OperatorEndpoint = validOperatorURL return nil } ================================================ FILE: cli/types/cliconfig/errors.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package cliconfig import ( "fmt" "github.com/cortexlabs/cortex/pkg/lib/errors" s "github.com/cortexlabs/cortex/pkg/lib/strings" ) const ( ErrEnvironmentNotConfigured = "cliconfig.environment_not_configured" ErrEnvironmentAlreadyConfigured = "cliconfig.environment_already_configured" ErrDuplicateEnvironmentNames = "cliconfig.duplicate_environment_names" ) func ErrorEnvironmentNotConfigured(envName string) error { return errors.WithStack(&errors.Error{ Kind: ErrEnvironmentNotConfigured, Message: fmt.Sprintf("there is no environment named %s", envName), }) } func ErrorEnvironmentAlreadyConfigured(envName string) error { return errors.WithStack(&errors.Error{ Kind: ErrEnvironmentAlreadyConfigured, Message: fmt.Sprintf("there is already an environment named %s", envName), }) } func ErrorDuplicateEnvironmentNames(envName string) error { return errors.WithStack(&errors.Error{ Kind: ErrDuplicateEnvironmentNames, Message: fmt.Sprintf("duplicate environment names (%s is defined more than once)", s.UserStr(envName)), }) } ================================================ FILE: cli/types/flags/errors.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package flags import ( "fmt" "github.com/cortexlabs/cortex/pkg/lib/errors" s "github.com/cortexlabs/cortex/pkg/lib/strings" ) const ( ErrInvalidOutputType = "flags.invalid_output_type" ) func ErrorInvalidOutputType(invalidOutputType string) error { return errors.WithStack(&errors.Error{ Kind: ErrInvalidOutputType, Message: fmt.Sprintf("invalid value \"%s\" specified for -o/--output; valid values are %s", invalidOutputType, s.StrsAnd(OutputTypeStrings())), }) } ================================================ FILE: cli/types/flags/output_type.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package flags type OutputType int const ( UnknownOutputType OutputType = iota PrettyOutputType JSONOutputType YAMLOutputType ) var _outputTypes = []string{ "unknown", "pretty", "json", "yaml", } func OutputTypeFromString(s string) OutputType { for i := 0; i < len(_outputTypes); i++ { if s == _outputTypes[i] { return OutputType(i) } } return UnknownOutputType } func OutputTypeStrings() []string { return _outputTypes[1:] } func OutputTypeStringsExcluding(outputType OutputType) []string { var outputTypes []string for _, _outputType := range _outputTypes[1:] { if OutputTypeFromString(_outputType) != outputType { outputTypes = append(outputTypes, _outputType) } } return outputTypes } func (t OutputType) String() string { return _outputTypes[t] } // MarshalText satisfies TextMarshaler func (t OutputType) MarshalText() ([]byte, error) { return []byte(t.String()), nil } // UnmarshalText satisfies TextUnmarshaler func (t *OutputType) UnmarshalText(text []byte) error { enum := string(text) for i := 0; i < len(_outputTypes); i++ { if enum == _outputTypes[i] { *t = OutputType(i) return nil } } *t = UnknownOutputType return nil } // UnmarshalBinary satisfies BinaryUnmarshaler // Needed for msgpack func (t *OutputType) UnmarshalBinary(data []byte) error { return t.UnmarshalText(data) } // MarshalBinary satisfies BinaryMarshaler func (t OutputType) MarshalBinary() ([]byte, error) { return []byte(t.String()), nil } func (t *OutputType) Set(value string) error { output := OutputTypeFromString(value) if output == UnknownOutputType { return ErrorInvalidOutputType(value) } *t = output return nil } func (t OutputType) Type() string { return "string" } ================================================ FILE: cmd/activator/main.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package main import ( "context" "flag" "net/http" "os" "os/signal" "strconv" "strings" "syscall" "time" "github.com/cortexlabs/cortex/pkg/activator" "github.com/cortexlabs/cortex/pkg/autoscaler" "github.com/cortexlabs/cortex/pkg/lib/aws" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/k8s" "github.com/cortexlabs/cortex/pkg/lib/logging" "github.com/cortexlabs/cortex/pkg/lib/telemetry" "github.com/cortexlabs/cortex/pkg/types/userconfig" "go.uber.org/zap" istioinformers "istio.io/client-go/pkg/informers/externalversions" kmeta "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" kinformers "k8s.io/client-go/informers" ) func main() { var ( port int adminPort int inCluster bool autoscalerURL string namespace string ) flag.IntVar(&port, "port", 8000, "port where the activator server will be exposed") flag.IntVar(&adminPort, "admin-port", 15000, "port where the admin server will be exposed") flag.BoolVar(&inCluster, "in-cluster", false, "use when autoscaler runs in-cluster") flag.StringVar(&autoscalerURL, "autoscaler-url", "", "the URL for the cortex autoscaler endpoint") flag.StringVar(&namespace, "namespace", os.Getenv("CORTEX_NAMESPACE"), "kubernetes namespace where the cortex APIs are deployed "+ "(can be set through the CORTEX_NAMESPACE env variable)", ) flag.Parse() log := logging.GetLogger() defer func() { _ = log.Sync() }() switch { case autoscalerURL == "": log.Fatal("--autoscaler-url is a required option") case namespace == "": log.Fatal("--namespace is a required option") } awsClient, err := aws.New() if err != nil { exit(log, err) } _, userID, err := awsClient.CheckCredentials() if err != nil { exit(log, err) } telemetryEnabled := strings.ToLower(os.Getenv("CORTEX_TELEMETRY_DISABLE")) != "true" err = telemetry.Init(telemetry.Config{ Enabled: telemetryEnabled, UserID: userID, Properties: map[string]string{ "kind": userconfig.RealtimeAPIKind.String(), "image_type": "activator", }, Environment: "operator", LogErrors: true, BackoffMode: telemetry.BackoffDuplicateMessages, }) if err != nil { log.Fatalw("failed to initialize telemetry", zap.Error(err)) } defer telemetry.Close() k8sClient, err := k8s.New(namespace, inCluster, nil, runtime.NewScheme()) if err != nil { exit(log, err, "failed to initialize kubernetes client") } istioClient := k8sClient.IstioClientSet() kubeClient := k8sClient.ClientSet() autoscalerClient := autoscaler.NewClient(autoscalerURL) prometheusStatsReporter := activator.NewPrometheusStatsReporter() istioInformerFactory := istioinformers.NewSharedInformerFactoryWithOptions( istioClient, 10*time.Second, // TODO: check how much makes sense istioinformers.WithNamespace(namespace), istioinformers.WithTweakListOptions(informerFilter), ) virtualServiceInformer := istioInformerFactory.Networking().V1beta1().VirtualServices().Informer() virtualServiceClient := istioClient.NetworkingV1beta1().VirtualServices(namespace) kubeInformerFactory := kinformers.NewSharedInformerFactoryWithOptions( kubeClient, 2*time.Second, // TODO: check how much makes sense kinformers.WithNamespace(namespace), kinformers.WithTweakListOptions(informerFilter), ) deploymentInformer := kubeInformerFactory.Apps().V1().Deployments().Informer() act := activator.New( virtualServiceClient, deploymentInformer, virtualServiceInformer, autoscalerClient, prometheusStatsReporter, log, ) handler := activator.NewHandler(act, log) adminHandler := http.NewServeMux() adminHandler.Handle("/metrics", prometheusStatsReporter) servers := map[string]*http.Server{ "activator": { Addr: ":" + strconv.Itoa(port), Handler: handler, }, "admin": { Addr: ":" + strconv.Itoa(adminPort), Handler: adminHandler, }, } stopCh := make(chan struct{}) go virtualServiceInformer.Run(stopCh) go deploymentInformer.Run(stopCh) defer func() { stopCh <- struct{}{} }() errCh := make(chan error) for name, server := range servers { go func(name string, server *http.Server) { log.Infof("Starting %s server on %s", name, server.Addr) errCh <- server.ListenAndServe() }(name, server) } sigint := make(chan os.Signal, 1) signal.Notify(sigint, os.Interrupt, syscall.SIGTERM) select { case err = <-errCh: exit(log, err, "failed to start activator server") case <-sigint: log.Info("Received INT or TERM signal, handling a graceful shutdown...") for name, server := range servers { log.Infof("Shutting down %s server", name) if err = server.Shutdown(context.Background()); err != nil { // Error from closing listeners, or context timeout: log.Warnw("HTTP server Shutdown Error", zap.Error(err)) telemetry.Error(errors.Wrap(err, "HTTP server Shutdown Error")) } } log.Info("Shutdown complete, exiting...") } } func informerFilter(listOptions *kmeta.ListOptions) { listOptions.LabelSelector = kmeta.FormatLabelSelector(&kmeta.LabelSelector{ MatchLabels: map[string]string{ "apiKind": userconfig.RealtimeAPIKind.String(), }, MatchExpressions: []kmeta.LabelSelectorRequirement{ { Key: "apiName", Operator: kmeta.LabelSelectorOpExists, }, }, }) } func exit(log *zap.SugaredLogger, err error, wrapStrs ...string) { if err == nil { os.Exit(0) } for _, str := range wrapStrs { err = errors.Wrap(err, str) } telemetry.Error(err) if !errors.IsNoPrint(err) { log.Fatal(err) } os.Exit(1) } ================================================ FILE: cmd/async-gateway/main.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package main import ( "flag" "net/http" "os" "strings" gateway "github.com/cortexlabs/cortex/pkg/async-gateway" "github.com/cortexlabs/cortex/pkg/lib/aws" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/logging" "github.com/cortexlabs/cortex/pkg/lib/telemetry" "github.com/cortexlabs/cortex/pkg/types/userconfig" "github.com/gorilla/handlers" "github.com/gorilla/mux" "go.uber.org/zap" ) const ( _defaultPort = "8080" ) // usage: ./gateway -bucket -region -port func main() { log := logging.GetLogger() defer func() { _ = log.Sync() }() var ( bucket = flag.String("bucket", "", "bucket") clusterUID = flag.String("cluster-uid", "", "cluster uid") port = flag.String("port", _defaultPort, "port on which the gateway server runs on") ) flag.Parse() switch { case *bucket == "": log.Fatal("missing required option: -bucket") case *clusterUID == "": log.Fatal("missing required option: -cluster-uid") } awsClient, err := aws.New() if err != nil { exit(log, err) } _, userID, err := awsClient.CheckCredentials() if err != nil { exit(log, err) } telemetryEnabled := strings.ToLower(os.Getenv("CORTEX_TELEMETRY_DISABLE")) != "true" err = telemetry.Init(telemetry.Config{ Enabled: telemetryEnabled, UserID: userID, Properties: map[string]string{ "kind": userconfig.AsyncAPIKind.String(), "image_type": "async-gateway", }, Environment: "api", LogErrors: true, BackoffMode: telemetry.BackoffDuplicateMessages, }) if err != nil { log.Fatalw("failed to initialize telemetry", zap.Error(err)) } defer telemetry.Close() sess := awsClient.Session() s3Storage := gateway.NewS3(sess, *bucket) svc := gateway.NewService(*clusterUID, s3Storage, log, *sess) ep := gateway.NewEndpoint(svc, log) router := mux.NewRouter() router.HandleFunc("/", ep.CreateWorkload).Methods("POST") router.HandleFunc( "/healthz", func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "text/plain") w.WriteHeader(http.StatusOK) _, _ = w.Write([]byte("ok")) }, ) router.HandleFunc("/{id}", ep.GetWorkload).Methods("GET") // inspired by our nginx config corsOptions := []handlers.CORSOption{ handlers.AllowedOrigins([]string{"*"}), // custom headers are not supported currently, since "*" is not supported in AllowedHeaders(); here are some common ones: handlers.AllowedHeaders([]string{"Content-Type", "X-Requested-With", "User-Agent", "Accept", "Accept-Language", "Content-Language", "Origin"}), handlers.AllowedMethods([]string{"GET", "HEAD", "POST", "PUT", "OPTIONS"}), handlers.ExposedHeaders([]string{"Content-Length", "Content-Range"}), handlers.AllowCredentials(), } log.Info("Running on port " + *port) if err = http.ListenAndServe(":"+*port, handlers.CORS(corsOptions...)(router)); err != nil { exit(log, err) } } func exit(log *zap.SugaredLogger, err error, wrapStrs ...string) { if err == nil { os.Exit(0) } for _, str := range wrapStrs { err = errors.Wrap(err, str) } telemetry.Error(err) if !errors.IsNoPrint(err) { log.Fatal(err) } os.Exit(1) } ================================================ FILE: cmd/autoscaler/main.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package main import ( "context" "flag" "fmt" "net/http" "os" "os/signal" "strconv" "strings" "syscall" "time" "github.com/cortexlabs/cortex/pkg/autoscaler" "github.com/cortexlabs/cortex/pkg/lib/aws" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/k8s" "github.com/cortexlabs/cortex/pkg/lib/logging" "github.com/cortexlabs/cortex/pkg/lib/telemetry" "github.com/cortexlabs/cortex/pkg/types/userconfig" "github.com/gorilla/mux" promapi "github.com/prometheus/client_golang/api" promv1 "github.com/prometheus/client_golang/api/prometheus/v1" "go.uber.org/zap" istioclient "istio.io/client-go/pkg/clientset/versioned" istioinformers "istio.io/client-go/pkg/informers/externalversions" "k8s.io/apimachinery/pkg/api/meta" kmeta "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/tools/cache" ) func main() { var ( port int inCluster bool prometheusURL string namespace string ) flag.IntVar(&port, "port", 8000, "port where the autoscaler server will be exposed") flag.BoolVar(&inCluster, "in-cluster", false, "use when autoscaler runs in-cluster") flag.StringVar(&prometheusURL, "prometheus-url", os.Getenv("CORTEX_PROMETHEUS_URL"), "prometheus url (can be set through the CORTEX_PROMETHEUS_URL env variable)", ) flag.StringVar(&namespace, "namespace", os.Getenv("CORTEX_NAMESPACE"), "kubernetes namespace where the cortex APIs are deployed "+ "(can be set through the CORTEX_NAMESPACE env variable)", ) flag.Parse() log := logging.GetLogger() defer func() { _ = log.Sync() }() switch { case prometheusURL == "": log.Fatal("--prometheus-url is a required option") case namespace == "": log.Fatal("--namespace is a required option") } awsClient, err := aws.New() if err != nil { exit(log, err) } _, userID, err := awsClient.CheckCredentials() if err != nil { exit(log, err) } telemetryEnabled := strings.ToLower(os.Getenv("CORTEX_TELEMETRY_DISABLE")) != "true" err = telemetry.Init(telemetry.Config{ Enabled: telemetryEnabled, UserID: userID, Properties: map[string]string{ "kind": userconfig.RealtimeAPIKind.String(), "image_type": "autoscaler", }, Environment: "operator", LogErrors: true, BackoffMode: telemetry.BackoffDuplicateMessages, }) if err != nil { log.Fatalw("failed to initialize telemetry", zap.Error(err)) } defer telemetry.Close() scheme := runtime.NewScheme() if err := clientgoscheme.AddToScheme(scheme); err != nil { exit(log, err, "failed to add k8s client-go-scheme to scheme") } k8sClient, err := k8s.New(namespace, inCluster, nil, scheme) if err != nil { exit(log, err, "failed to initialize kubernetes client") } //goland:noinspection GoNilness istioClient, err := istioclient.NewForConfig(k8sClient.RestConfig) if err != nil { exit(log, err, "failed to initialize istio client") } promClient, err := promapi.NewClient( promapi.Config{ Address: prometheusURL, }, ) if err != nil { exit(log, err, "failed to initialize prometheus client") } promAPIClient := promv1.NewAPI(promClient) realtimeScaler := autoscaler.NewRealtimeScaler(k8sClient, promAPIClient, log) asyncScaler := autoscaler.NewAsyncScaler(k8sClient, promAPIClient) autoScaler := autoscaler.New(log) autoScaler.AddScaler(realtimeScaler, userconfig.RealtimeAPIKind) autoScaler.AddScaler(asyncScaler, userconfig.AsyncAPIKind) defer autoScaler.Stop() istioInformerFactory := istioinformers.NewSharedInformerFactoryWithOptions( istioClient, 10*time.Second, // TODO: check how much makes sense istioinformers.WithNamespace(namespace), istioinformers.WithTweakListOptions(informerFilter), ) virtualServiceInformer := istioInformerFactory.Networking().V1beta1().VirtualServices().Informer() virtualServiceInformer.AddEventHandler( cache.ResourceEventHandlerFuncs{ AddFunc: func(obj interface{}) { resource, err := meta.Accessor(obj) if err != nil { log.Errorw("failed to access resource metadata", zap.Error(err)) telemetry.Error(err) return } if resource.GetNamespace() != namespace { // filter out virtual services that are not in the cortex namespace return } api, err := apiResourceFromLabels(resource.GetLabels()) if err != nil { // filter out non-cortex apis return } if err := autoScaler.AddAPI(api); err != nil { log.Errorw("failed to add API to autoscaler", zap.Error(err), zap.String("apiName", api.Name), zap.String("apiKind", api.Kind.String()), ) telemetry.Error(err) return } }, DeleteFunc: func(obj interface{}) { resource, err := meta.Accessor(obj) if err != nil { log.Errorw("failed to access resource metadata", zap.Error(err)) } if resource.GetNamespace() != namespace { // filter out virtual services that are not in the cortex namespace return } api, err := apiResourceFromLabels(resource.GetLabels()) if err != nil { // filter out non-cortex apis return } autoScaler.RemoveAPI(api) }, }, ) handler := autoscaler.NewHandler(autoScaler) router := mux.NewRouter() router.HandleFunc("/awaken", handler.Awaken).Methods(http.MethodPost) router.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "text/plain; charset=utf-8") w.WriteHeader(http.StatusOK) _, _ = w.Write([]byte("ok")) }).Methods(http.MethodGet) server := &http.Server{ Addr: ":" + strconv.Itoa(port), Handler: router, } stopCh := make(chan struct{}) go virtualServiceInformer.Run(stopCh) defer func() { stopCh <- struct{}{} }() errCh := make(chan error) go func() { log.Infof("Starting autoscaler server on %s", server.Addr) errCh <- server.ListenAndServe() }() sigint := make(chan os.Signal, 1) signal.Notify(sigint, os.Interrupt, syscall.SIGTERM) select { case err = <-errCh: exit(log, err, "failed to start autoscaler server") case <-sigint: log.Info("Received INT or TERM signal, handling a graceful shutdown...") log.Info("Shutting down server") if err = server.Shutdown(context.Background()); err != nil { // Error from closing listeners, or context timeout: log.Warnw("HTTP server Shutdown Error", zap.Error(err)) } log.Info("Shutdown complete, exiting...") } } func apiResourceFromLabels(labels map[string]string) (userconfig.Resource, error) { apiName, ok := labels["apiName"] if !ok { return userconfig.Resource{}, fmt.Errorf("apiName key does not exist") } apiKind, ok := labels["apiKind"] if !ok { return userconfig.Resource{}, fmt.Errorf("apiKind key does not exist") } return userconfig.Resource{ Name: apiName, Kind: userconfig.KindFromString(apiKind), }, nil } func informerFilter(listOptions *kmeta.ListOptions) { listOptions.LabelSelector = kmeta.FormatLabelSelector(&kmeta.LabelSelector{ MatchExpressions: []kmeta.LabelSelectorRequirement{ { Key: "apiName", Operator: kmeta.LabelSelectorOpExists, }, { Key: "apiKind", Operator: kmeta.LabelSelectorOpExists, }, }, }) } func exit(log *zap.SugaredLogger, err error, wrapStrs ...string) { if err == nil { os.Exit(0) } for _, str := range wrapStrs { err = errors.Wrap(err, str) } telemetry.Error(err) if !errors.IsNoPrint(err) { log.Fatal(err) } os.Exit(1) } ================================================ FILE: cmd/dequeuer/main.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package main import ( "flag" "fmt" "net/http" "os" "os/signal" "strconv" "syscall" "github.com/DataDog/datadog-go/statsd" "github.com/cortexlabs/cortex/pkg/consts" "github.com/cortexlabs/cortex/pkg/dequeuer" awslib "github.com/cortexlabs/cortex/pkg/lib/aws" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/files" "github.com/cortexlabs/cortex/pkg/lib/logging" "github.com/cortexlabs/cortex/pkg/lib/telemetry" "github.com/cortexlabs/cortex/pkg/probe" "github.com/cortexlabs/cortex/pkg/types/clusterconfig" "github.com/cortexlabs/cortex/pkg/types/userconfig" "go.uber.org/zap" ) func main() { var ( clusterConfigPath string clusterUID string probesPath string queueURL string userContainerPort int apiName string jobID string statsdAddress string apiKind string adminPort int workers int ) flag.StringVar(&clusterConfigPath, "cluster-config", "", "cluster config path") flag.StringVar(&clusterUID, "cluster-uid", "", "cluster unique identifier") flag.StringVar(&probesPath, "probes-path", "", "path to the probes spec") flag.StringVar(&queueURL, "queue", "", "target queue URL from which the api messages will be dequeued") flag.StringVar(&apiKind, "api-kind", "", fmt.Sprintf("api kind (%s|%s)", userconfig.BatchAPIKind.String(), userconfig.AsyncAPIKind.String())) flag.StringVar(&apiName, "api-name", "", "api name") flag.StringVar(&jobID, "job-id", "", "job ID") flag.StringVar(&statsdAddress, "statsd-address", "", "address to push statsd metrics") flag.IntVar(&userContainerPort, "user-port", 8080, "target port to which the dequeued messages will be sent to") flag.IntVar(&adminPort, "admin-port", 0, "port where the admin server (for the probes) will be exposed") flag.IntVar(&workers, "workers", 1, "number of workers pulling from the queue") flag.Parse() version := os.Getenv("CORTEX_VERSION") if version == "" { version = consts.CortexVersion } log := logging.GetLogger() defer func() { _ = log.Sync() }() switch { case clusterConfigPath == "": log.Fatal("--cluster-config is a required option") case probesPath == "": log.Fatal("--probes-path is a required option") case queueURL == "": log.Fatal("--queue is a required option") case apiName == "": log.Fatal("--api-name is a required option") case apiKind == "": log.Fatal("--api-kind is a required option") case adminPort == 0: log.Fatal("--admin-port is a required option") } targetURL := "http://127.0.0.1:" + strconv.Itoa(userContainerPort) clusterConfig, err := clusterconfig.NewForFile(clusterConfigPath) if err != nil { exit(log, err) } awsClient, err := awslib.NewForRegion(clusterConfig.Region) if err != nil { exit(log, err, "failed to create aws client") } _, userID, err := awsClient.CheckCredentials() if err != nil { exit(log, err) } err = telemetry.Init(telemetry.Config{ Enabled: clusterConfig.Telemetry, UserID: userID, Properties: map[string]string{ "kind": apiKind, "image_type": "dequeuer", }, Environment: "api", LogErrors: true, BackoffMode: telemetry.BackoffDuplicateMessages, }) if err != nil { log.Fatalw("failed to initialize telemetry", "error", err) } defer telemetry.Close() var probes []*probe.Probe if files.IsFile(probesPath) { probes, err = dequeuer.ProbesFromFile(probesPath, log) if err != nil { exit(log, err, fmt.Sprintf("unable to read probes from %s", probesPath)) } } if !dequeuer.HasTCPProbeTargetingUserPod(probes, userContainerPort) { probes = append(probes, probe.NewDefaultProbe(fmt.Sprintf("http://localhost:%d", userContainerPort), log)) } adminHandler := http.NewServeMux() adminHandler.Handle("/healthz", dequeuer.HealthcheckHandler(func() bool { return probe.AreProbesHealthy(probes) })) var dequeuerConfig dequeuer.SQSDequeuerConfig var messageHandler dequeuer.MessageHandler switch apiKind { case userconfig.BatchAPIKind.String(): if jobID == "" { log.Fatal("--job-id is a required option") } config := dequeuer.BatchMessageHandlerConfig{ Region: clusterConfig.Region, APIName: apiName, JobID: jobID, QueueURL: queueURL, TargetURL: targetURL, } metricsClient, err := statsd.New(statsdAddress) if err != nil { exit(log, err, "unable to initialize metrics client") } messageHandler = dequeuer.NewBatchMessageHandler(config, awsClient, metricsClient, log) dequeuerConfig = dequeuer.SQSDequeuerConfig{ Region: clusterConfig.Region, QueueURL: queueURL, StopIfNoMessages: true, Workers: workers, } case userconfig.AsyncAPIKind.String(): if clusterUID == "" { log.Fatal("--cluster-uid is a required option") } config := dequeuer.AsyncMessageHandlerConfig{ ClusterUID: clusterUID, Bucket: clusterConfig.Bucket, APIName: apiName, TargetURL: targetURL, } asyncStatsReporter := dequeuer.NewAsyncPrometheusStatsReporter() messageHandler = dequeuer.NewAsyncMessageHandler(config, awsClient, asyncStatsReporter, log) dequeuerConfig = dequeuer.SQSDequeuerConfig{ Region: clusterConfig.Region, QueueURL: queueURL, StopIfNoMessages: false, Workers: workers, } // report prometheus metrics for async api kinds adminHandler.Handle("/metrics", asyncStatsReporter) default: exit(log, err, fmt.Sprintf("kind %s is not supported", apiKind)) } errCh := make(chan error) go func() { server := &http.Server{ Addr: ":" + strconv.Itoa(adminPort), Handler: adminHandler, } log.Infof("Starting %s server on %s", consts.AdminPortName, server.Addr) errCh <- server.ListenAndServe() }() sigint := make(chan os.Signal, 1) signal.Notify(sigint, os.Interrupt, syscall.SIGTERM) sqsDequeuer, err := dequeuer.NewSQSDequeuer(dequeuerConfig, awsClient, log) if err != nil { exit(log, err, "failed to create sqs dequeuer") } go func() { log.Info("Starting dequeuer...") errCh <- sqsDequeuer.Start(messageHandler, func() bool { return probe.AreProbesHealthy(probes) }) }() var stopChs []chan struct{} for _, p := range probes { stopChs = append(stopChs, p.StartProbing()) } defer func() { for _, stopCh := range stopChs { stopCh <- struct{}{} } }() select { case err = <-errCh: exit(log, err, "error during message dequeueing or error from admin server") case <-sigint: log.Info("Received INT or TERM signal, handling a graceful shutdown...") sqsDequeuer.Shutdown() log.Info("Shutdown complete, exiting...") } } func exit(log *zap.SugaredLogger, err error, wrapStrs ...string) { if err == nil { os.Exit(0) } for _, str := range wrapStrs { err = errors.Wrap(err, str) } telemetry.Error(err) if !errors.IsNoPrint(err) { log.Fatal(err) } os.Exit(1) } ================================================ FILE: cmd/enqueuer/main.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package main import ( "flag" "os" "strings" "github.com/cortexlabs/cortex/pkg/consts" "github.com/cortexlabs/cortex/pkg/enqueuer" "go.uber.org/zap" "go.uber.org/zap/zapcore" ) func createLogger() (*zap.Logger, error) { logLevelEnv := strings.ToLower(os.Getenv("CORTEX_LOG_LEVEL")) disableJSONLogging := os.Getenv("CORTEX_DISABLE_JSON_LOGGING") var logLevelZap zapcore.Level switch logLevelEnv { case "debug": logLevelZap = zapcore.DebugLevel case "warning": logLevelZap = zapcore.WarnLevel case "error": logLevelZap = zapcore.ErrorLevel default: logLevelZap = zapcore.InfoLevel } encoderConfig := zap.NewProductionEncoderConfig() encoderConfig.MessageKey = "message" encoding := "json" if strings.ToLower(disableJSONLogging) == "true" { encoding = "console" } return zap.Config{ Level: zap.NewAtomicLevelAt(logLevelZap), Encoding: encoding, EncoderConfig: encoderConfig, OutputPaths: []string{"stdout"}, ErrorOutputPaths: []string{"stderr"}, }.Build() } func main() { var ( clusterUID string region string bucket string queueURL string apiName string jobID string ) flag.StringVar(&clusterUID, "cluster-uid", os.Getenv("CORTEX_CLUSTER_UID"), "cluster UID (can be set throught the CORTEX_CLUSTER_UID env variable)") flag.StringVar(®ion, "region", os.Getenv("CORTEX_REGION"), "cluster region (can be set throught the CORTEX_REGION env variable)") flag.StringVar(&bucket, "bucket", os.Getenv("CORTEX_BUCKET"), "cortex S3 bucket (can be set throught the CORTEX_BUCKET env variable)") flag.StringVar(&queueURL, "queue", "", "target queue URL to where the api messages will be enqueued") flag.StringVar(&apiName, "apiName", "", "api name") flag.StringVar(&jobID, "jobID", "", "job ID") flag.Parse() version := os.Getenv("CORTEX_VERSION") if version == "" { version = consts.CortexVersion } log, err := createLogger() if err != nil { panic(err) } defer func() { _ = log.Sync() }() switch { case clusterUID == "": log.Fatal("-cluster-uid is a required option") case region == "": log.Fatal("-region is a required option") case bucket == "": log.Fatal("-bucket is a required option") case queueURL == "": log.Fatal("-queue is a required option") case apiName == "": log.Fatal("-apiName is a required option") case jobID == "": log.Fatal("-jobID is a required option") } envConfig := enqueuer.EnvConfig{ ClusterUID: clusterUID, Region: region, Version: version, Bucket: bucket, APIName: apiName, JobID: jobID, } eqr, err := enqueuer.NewEnqueuer(envConfig, queueURL, log) if err != nil { log.Fatal("failed to create enqueuer", zap.Error(err)) } totalBatches, err := eqr.Enqueue() if err != nil { log.Fatal("failed to enqueue batches", zap.Error(err)) } if err = eqr.UploadBatchCount(totalBatches); err != nil { log.Fatal("failed to upload batch count", zap.Error(err)) } log.Info("done enqueuing batches", zap.Int("batchCount", totalBatches)) } ================================================ FILE: cmd/operator/main.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package main import ( "net/http" "time" "github.com/cortexlabs/cortex/pkg/config" "github.com/cortexlabs/cortex/pkg/lib/cron" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/logging" "github.com/cortexlabs/cortex/pkg/lib/telemetry" "github.com/cortexlabs/cortex/pkg/operator/endpoints" "github.com/cortexlabs/cortex/pkg/operator/lib/exit" "github.com/cortexlabs/cortex/pkg/operator/operator" "github.com/cortexlabs/cortex/pkg/operator/resources/asyncapi" "github.com/cortexlabs/cortex/pkg/operator/resources/job/taskapi" "github.com/cortexlabs/cortex/pkg/types/userconfig" "github.com/gorilla/handlers" "github.com/gorilla/mux" "github.com/prometheus/client_golang/prometheus/promhttp" ) var operatorLogger = logging.GetLogger() const _operatorPortStr = "8888" func main() { if err := config.Init(); err != nil { exit.ErrorNoTelemetry(errors.Wrap(err, "init")) } telemetry.Event("operator.init") cron.Run(operator.DeleteEvictedPods, operator.ErrorHandler("delete evicted pods"), time.Hour) cron.Run(operator.ClusterTelemetry, operator.ErrorHandler("instance telemetry"), 1*time.Hour) cron.Run(operator.CostBreakdown, operator.ErrorHandler("cost breakdown metrics"), 5*time.Minute) _, err := operator.UpdateMemoryCapacityConfigMap() if err != nil { exit.Error(errors.Wrap(err, "init")) } cron.Run(taskapi.ManageJobResources, operator.ErrorHandler("manage task jobs"), taskapi.ManageJobResourcesCronPeriod) deployments, err := config.K8s.ListDeploymentsWithLabelKeys("apiName") if err != nil { exit.Error(errors.Wrap(err, "init")) } for i := range deployments { deployment := deployments[i] apiKind := deployment.Labels["apiKind"] switch apiKind { case userconfig.AsyncAPIKind.String(): if err := asyncapi.UpdateAPIMetricsCron(&deployment); err != nil { operatorLogger.Fatal(errors.Wrap(err, "init")) } } } router := mux.NewRouter() routerWithoutAuth := router.NewRoute().Subrouter() routerWithoutAuth.Use(endpoints.PanicMiddleware) routerWithoutAuth.HandleFunc("/verifycortex", endpoints.VerifyCortex).Methods("GET") routerWithoutAuth.HandleFunc("/batch/{apiName}", endpoints.SubmitBatchJob).Methods("POST") routerWithoutAuth.HandleFunc("/batch/{apiName}", endpoints.GetBatchJob).Methods("GET") routerWithoutAuth.HandleFunc("/batch/{apiName}", endpoints.StopBatchJob).Methods("DELETE") routerWithoutAuth.HandleFunc("/tasks/{apiName}", endpoints.SubmitTaskJob).Methods("POST") routerWithoutAuth.HandleFunc("/tasks/{apiName}", endpoints.GetTaskJob).Methods("GET") routerWithoutAuth.HandleFunc("/tasks/{apiName}", endpoints.StopTaskJob).Methods("DELETE") // prometheus metrics routerWithoutAuth.Handle("/metrics", promhttp.Handler()).Methods("GET") routerWithAuth := router.NewRoute().Subrouter() routerWithAuth.Use(endpoints.PanicMiddleware) routerWithAuth.Use(endpoints.APIVersionCheckMiddleware) routerWithAuth.Use(endpoints.AWSAuthMiddleware) routerWithAuth.Use(endpoints.ClientIDMiddleware) routerWithAuth.HandleFunc("/info", endpoints.Info).Methods("GET") routerWithAuth.HandleFunc("/deploy", endpoints.Deploy).Methods("POST") routerWithAuth.HandleFunc("/refresh/{apiName}", endpoints.Refresh).Methods("POST") routerWithAuth.HandleFunc("/delete/{apiName}", endpoints.Delete).Methods("DELETE") routerWithAuth.HandleFunc("/get", endpoints.GetAPIs).Methods("GET") routerWithAuth.HandleFunc("/get/{apiName}", endpoints.GetAPI).Methods("GET") routerWithAuth.HandleFunc("/get/{apiName}/{apiID}", endpoints.GetAPIByID).Methods("GET") routerWithAuth.HandleFunc("/describe/{apiName}", endpoints.DescribeAPI).Methods("GET") routerWithAuth.HandleFunc("/streamlogs/{apiName}", endpoints.ReadLogs) routerWithAuth.HandleFunc("/logs/{apiName}", endpoints.GetLogURL).Methods("GET") operatorLogger.Info("Running on port " + _operatorPortStr) // inspired by our nginx config corsOptions := []handlers.CORSOption{ handlers.AllowedOrigins([]string{"*"}), // custom headers are not supported currently, since "*" is not supported in AllowedHeaders(); here are some common ones: handlers.AllowedHeaders([]string{"Content-Type", "X-Requested-With", "User-Agent", "Accept", "Accept-Language", "Content-Language", "Origin"}), handlers.AllowedMethods([]string{"GET", "HEAD", "POST", "PUT", "OPTIONS"}), handlers.ExposedHeaders([]string{"Content-Length", "Content-Range"}), handlers.AllowCredentials(), } operatorLogger.Fatal(http.ListenAndServe(":"+_operatorPortStr, handlers.CORS(corsOptions...)(router))) } ================================================ FILE: cmd/proxy/main.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package main import ( "context" "flag" "fmt" "net" "net/http" "os" "os/signal" "strconv" "syscall" "time" "github.com/cortexlabs/cortex/pkg/lib/aws" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/logging" "github.com/cortexlabs/cortex/pkg/lib/telemetry" "github.com/cortexlabs/cortex/pkg/proxy" "github.com/cortexlabs/cortex/pkg/types/clusterconfig" "github.com/cortexlabs/cortex/pkg/types/userconfig" "go.uber.org/zap" ) const ( _reportInterval = 10 * time.Second _requestSampleInterval = 1 * time.Second ) func main() { var ( port int adminPort int userContainerPort int maxConcurrency int maxQueueLength int hasTCPProbe bool clusterConfigPath string ) flag.IntVar(&port, "port", 8000, "port where the proxy server will be exposed") flag.IntVar(&adminPort, "admin-port", 15000, "port where the admin server (for metrics and probes) will be exposed") flag.IntVar(&userContainerPort, "user-port", 8080, "port where the proxy will redirect to the traffic to") flag.IntVar(&maxConcurrency, "max-concurrency", 0, "max concurrency allowed for user container") flag.IntVar(&maxQueueLength, "max-queue-length", 0, "max request queue length for user container") flag.BoolVar(&hasTCPProbe, "has-tcp-probe", false, "tcp probe to the user-provided container port") flag.StringVar(&clusterConfigPath, "cluster-config", "", "cluster config path") flag.Parse() log := logging.GetLogger() defer func() { _ = log.Sync() }() switch { case maxConcurrency == 0: log.Fatal("--max-concurrency flag is required") case maxQueueLength == 0: log.Fatal("--max-queue-length flag is required") case clusterConfigPath == "": log.Fatal("--cluster-config flag is required") } clusterConfig, err := clusterconfig.NewForFile(clusterConfigPath) if err != nil { exit(log, err) } awsClient, err := aws.NewForRegion(clusterConfig.Region) if err != nil { exit(log, err) } _, userID, err := awsClient.CheckCredentials() if err != nil { exit(log, err) } err = telemetry.Init(telemetry.Config{ Enabled: clusterConfig.Telemetry, UserID: userID, Properties: map[string]string{ "kind": userconfig.RealtimeAPIKind.String(), "image_type": "proxy", }, Environment: "api", LogErrors: true, BackoffMode: telemetry.BackoffDuplicateMessages, }) if err != nil { log.Fatalw("failed to initialize telemetry", zap.Error(err)) } defer telemetry.Close() target := "http://127.0.0.1:" + strconv.Itoa(userContainerPort) httpProxy := proxy.NewReverseProxy(target, maxQueueLength, maxQueueLength) requestCounterStats := &proxy.RequestStats{} breaker := proxy.NewBreaker( proxy.BreakerParams{ QueueDepth: maxQueueLength, MaxConcurrency: maxConcurrency, InitialCapacity: maxConcurrency, }, ) promStats := proxy.NewPrometheusStatsReporter() go func() { reportTicker := time.NewTicker(_reportInterval) defer reportTicker.Stop() requestSamplingTicker := time.NewTicker(_requestSampleInterval) defer requestSamplingTicker.Stop() for { select { case <-reportTicker.C: go func() { report := requestCounterStats.Report() promStats.Report(report) }() case <-requestSamplingTicker.C: go func() { requestCounterStats.Append(breaker.InFlight()) }() } } }() adminHandler := http.NewServeMux() adminHandler.Handle("/metrics", promStats) adminHandler.Handle("/healthz", readinessTCPHandler(userContainerPort, hasTCPProbe, log)) servers := map[string]*http.Server{ "proxy": { Addr: ":" + strconv.Itoa(port), Handler: proxy.Handler(breaker, httpProxy), }, "admin": { Addr: ":" + strconv.Itoa(adminPort), Handler: adminHandler, }, } errCh := make(chan error) for name, server := range servers { go func(name string, server *http.Server) { log.Infof("Starting %s server on %s", name, server.Addr) errCh <- server.ListenAndServe() }(name, server) } sigint := make(chan os.Signal, 1) signal.Notify(sigint, os.Interrupt, syscall.SIGTERM) select { case err = <-errCh: exit(log, errors.Wrap(err, "failed to start proxy server")) case <-sigint: log.Info("Received INT or TERM signal, handling a graceful shutdown...") for name, server := range servers { log.Infof("Shutting down %s server", name) if err = server.Shutdown(context.Background()); err != nil { // Error from closing listeners, or context timeout: log.Warnw("HTTP server Shutdown Error", zap.Error(err)) telemetry.Error(errors.Wrap(err, "HTTP server Shutdown Error")) } } log.Info("Shutdown complete, exiting...") } } func exit(log *zap.SugaredLogger, err error, wrapStrs ...string) { if err == nil { os.Exit(0) } for _, str := range wrapStrs { err = errors.Wrap(err, str) } telemetry.Error(err) if !errors.IsNoPrint(err) { log.Fatal(err) } os.Exit(1) } func readinessTCPHandler(port int, enableTCPProbe bool, logger *zap.SugaredLogger) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { if enableTCPProbe { ctx := r.Context() address := net.JoinHostPort("localhost", fmt.Sprintf("%d", port)) var d net.Dialer conn, err := d.DialContext(ctx, "tcp", address) if err != nil { logger.Warn(errors.Wrap(err, "TCP probe to user-provided container port failed")) w.WriteHeader(http.StatusInternalServerError) _, _ = w.Write([]byte("unhealthy")) return } _ = conn.Close() } w.WriteHeader(http.StatusOK) _, _ = w.Write([]byte("healthy")) } } ================================================ FILE: dev/build_cli.sh ================================================ #!/bin/bash # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. >/dev/null && pwd)" $ROOT/build/cli.sh upload ================================================ FILE: dev/create_user.py ================================================ #!/bin/bash # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import boto3 import configparser from pathlib import Path import sys import os # Usage: python create_user.py $CORTEX_CLUSTER_NAME $CORTEX_ACCOUNT_ID $CORTEX_REGION cluster_name = sys.argv[1] account_id = sys.argv[2] cortex_region = sys.argv[3] dir_path = os.path.dirname(os.path.realpath(__file__)) with open(f"{dir_path}/minimum_aws_policy.json", "r") as f: policy_string = f.read() policy_string = policy_string.replace("$CORTEX_CLUSTER_NAME", cluster_name) policy_string = policy_string.replace("$CORTEX_REGION", cortex_region) policy_string = policy_string.replace("$CORTEX_ACCOUNT_ID", account_id) user_name = f"dev-{cluster_name}-{cortex_region}" iam_client = boto3.client("iam", region_name=cortex_region) try: iam_client.get_user(UserName=user_name) except iam_client.exceptions.NoSuchEntityException: iam_client.create_user(UserName=user_name) partition = "aws" if "us-gov" in cortex_region: partition = "aws-us-gov" policy_arn = f"arn:{partition}:iam::{account_id}:policy/{user_name}" try: iam_client.get_policy(PolicyArn=policy_arn) except iam_client.exceptions.NoSuchEntityException: iam_client.create_policy( PolicyName=user_name, PolicyDocument=policy_string, ) policy_versions = iam_client.list_policy_versions(PolicyArn=policy_arn)["Versions"] if len(policy_versions) == 5: policy_versions.sort(key=lambda x: x["CreateDate"]) oldest_version = policy_versions[0]["VersionId"] iam_client.delete_policy_version(PolicyArn=policy_arn, VersionId=oldest_version) iam_client.create_policy_version( PolicyArn=policy_arn, PolicyDocument=policy_string, SetAsDefault=True ) iam_client.attach_user_policy(UserName=user_name, PolicyArn=policy_arn) aws_credentials_path = Path("~/.aws/credentials").expanduser() if not aws_credentials_path.exists(): aws_credentials_path.parent.mkdir(parents=True, exist_ok=True) aws_credentials_path.touch() aws_credentials = configparser.ConfigParser() aws_credentials.read(aws_credentials_path) if not user_name in aws_credentials: access_keys = iam_client.list_access_keys(UserName=user_name)["AccessKeyMetadata"] if len(access_keys) == 2: access_keys.sort(key=lambda x: x["CreateDate"]) iam_client.delete_access_key(UserName=user_name, AccessKeyId=access_keys[0]["AccessKeyId"]) key = iam_client.create_access_key(UserName=user_name)["AccessKey"] aws_credentials[user_name] = { "aws_access_key_id": key["AccessKeyId"], "aws_secret_access_key": key["SecretAccessKey"], } with open(aws_credentials_path, "w") as configfile: aws_credentials.write(configfile) aws_access_key_id = aws_credentials[user_name]["aws_access_key_id"] aws_secret_access_key = aws_credentials[user_name]["aws_secret_access_key"] print(f"export AWS_ACCESS_KEY_ID={aws_access_key_id}") print(f"export AWS_SECRET_ACCESS_KEY={aws_secret_access_key}") ================================================ FILE: dev/delete_ecr_repos.py ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import boto3 ecr = boto3.client("ecr") response = ecr.describe_repositories(maxResults=1000) for repo in response["repositories"]: ecr.delete_repository( registryId=repo["registryId"], repositoryName=repo["repositoryName"], force=True, ) print(f"deleted{repo['repositoryName']}") print("done") ================================================ FILE: dev/export_images.sh ================================================ #!/bin/bash # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -euo pipefail # usage: ./dev/export_images.sh # e.g. ./dev/export_images.sh us-east-1 123456789 ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. >/dev/null && pwd)" # CORTEX_VERSION cortex_version=master # user set variables ecr_region=$1 aws_account_id=$2 source_registry=quay.io/cortexlabs # this can also be docker.io/cortexlabs destination_ecr_prefix="cortexlabs" destination_registry="${aws_account_id}.dkr.ecr.${ecr_region}.amazonaws.com/${destination_ecr_prefix}" if [[ -f $HOME/.docker/config.json && $(cat $HOME/.docker/config.json | grep "ecr-login" | wc -l) -ne 0 ]]; then echo "skipping docker login because you are using ecr-login with Amazon ECR Docker Credential Helper" else aws ecr get-login-password --region $ecr_region | docker login --username AWS --password-stdin $destination_registry fi source $ROOT/build/images.sh # create the image repositories for image in "${all_images[@]}"; do repository_name=$destination_ecr_prefix/$image if aws ecr describe-repositories --repository-names=$repository_name --region=$ecr_region >/dev/null 2>&1; then echo "repository '$repository_name' already exists" else aws ecr create-repository --repository-name=$repository_name --region=$ecr_region | cat fi done echo # pull the images from source registry and push them to ECR for image in "${all_images[@]}"; do echo "copying $image:$cortex_version from $source_registry to $destination_registry" skopeo copy --src-no-creds "docker://$source_registry/$image:$cortex_version" "docker://$destination_registry/$image:$cortex_version" echo done echo "done ✓" ================================================ FILE: dev/find_missing_docs_links.py ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import sys import os import re import asyncio import aiohttp script_path = os.path.realpath(__file__) root = os.path.dirname(os.path.dirname(script_path)) docs_root = os.path.join(root, "docs") skip_http = False if len(sys.argv) == 2 and sys.argv[1] == "--skip-http": skip_http = True def main(): files = get_docs_file_paths() link_infos = [] for file in files: link_infos += get_links_from_file(file) errors = check_links(link_infos) for error in errors: print(error) def get_docs_file_paths(): file_paths = [] for root, dirs, files in os.walk(docs_root): for file in files: if file.endswith(".md"): file_paths.append(os.path.join(root, file)) return file_paths # link_info is (src_file, line_number, original_link_text, target_file, header) def get_links_from_file(file): link_infos = [] n = 1 with open(file) as f: for line in f: for link in re.findall(r"\]\((.+?)\)", line): if is_external_link(link): link_infos.append((file, n, link, None, None)) continue if link.startswith("#"): link_infos.append((file, n, link, file, link[1:])) continue if link.endswith(".md"): target = os.path.normpath(os.path.join(file, "..", link)) link_infos.append((file, n, link, target, None)) continue if ".md#" in link: parts = link.split("#") if len(parts) == 2: target = os.path.normpath(os.path.join(file, "..", parts[0])) link_infos.append((file, n, link, target, parts[1])) continue # Unexpected link format, will be handled later link_infos.append((file, n, link, None, None)) n += 1 return link_infos def check_links(link_infos): errors = [] http_link_infos = [] for link_info in link_infos: src_file, line_num, original_link_text, target_file, header = link_info if is_external_link(original_link_text): http_link_infos.append(link_info) continue if not target_file and not header: errors.append(err_str(src_file, line_num, original_link_text, "unknown link format")), continue error = check_local_link(src_file, line_num, original_link_text, target_file, header) if error: errors.append(error) # fail fast if there are local link errors if len(errors) > 0: return errors if not skip_http: asyncio.get_event_loop().run_until_complete(check_all_http_links(http_link_infos, errors)) return errors async def check_all_http_links(http_link_infos, errors): links = set() async with aiohttp.ClientSession() as session: tasks = [] for link_info in http_link_infos: src_file, line_num, link, _, _ = link_info if link in links: continue if "://localhost:" in link: continue links.add(link) tasks.append( asyncio.ensure_future(check_http_link(session, src_file, line_num, link, errors)) ) await asyncio.gather(*tasks) async def check_http_link(session, src_file, line_num, link, errors): num_tries = 1 while True: try: async with session.get(link, timeout=5) as resp: if resp.status != 200: errors.append( err_str(src_file, line_num, link, f"http response code {resp.status}") ) return except asyncio.TimeoutError: if num_tries > 2: errors.append(err_str(src_file, line_num, link, "http timeout")) return num_tries += 1 def check_local_link(src_file, line_num, original_link_text, target_file, header): if not os.path.isfile(target_file): return err_str(src_file, line_num, original_link_text, "file does not exist") if header: found_header = False with open(target_file) as f: for line in f: if not line.startswith("#"): continue if header_matches(line, header): found_header = True break if not found_header: return err_str(src_file, line_num, original_link_text, "header does not exist") return None def header_matches(text, header): text_words = re.findall(r"[a-zA-Z]+", text.lower()) for word in re.findall(r"[a-zA-Z]+", header.lower()): if word not in text_words: return False return True def err_str(src_file, line_num, original_link_text, reason): clean_src_file = src_file.split("cortexlabs/cortex/")[-1] return f"{clean_src_file}:{line_num}: {original_link_text} ({reason})" def is_external_link(link): return link.startswith("http://") or link.startswith("https://") if __name__ == "__main__": main() ================================================ FILE: dev/format.sh ================================================ #!/bin/bash # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. >/dev/null && pwd)" if ! command -v gofmt >/dev/null 2>&1; then echo "gofmt must be installed" exit 1 fi if ! command -v black >/dev/null 2>&1; then echo "black must be installed" exit 1 fi gofmt -s -w "$ROOT"/cli "${ROOT}"/pkg black --quiet --line-length=100 --exclude .idea/ "$ROOT" # Trim trailing whitespace if [[ "$OSTYPE" == "darwin"* ]]; then output=$(cd "$ROOT" && find . -type f \ ! -path "./vendor/*" \ ! -path "./bin/*" \ ! -path "./.git/*" \ ! -path "./.idea/*" \ ! -name ".*" \ -print0 | \ xargs -0 sed -i '' -e's/[[:space:]]*$//') else output=$(cd "$ROOT" && find . -type f \ ! -path "./vendor/*" \ ! -path "./bin/*" \ ! -path "./.git/*" \ ! -path "./.idea/*" \ ! -name ".*" \ -print0 | \ xargs -0 sed -i 's/[[:space:]]*$//') fi # Add new line to end of file (cd "$ROOT" && find . -type f \ ! -path "./vendor/*" \ ! -path "./bin/*" \ ! -path "./.git/*" \ ! -path "./.idea/*" \ ! -name ".*" \ -print0 | \ xargs -0 -L1 bash -c 'test "$(tail -c 1 "$0")" && echo "" >> "$0"' || true) # Remove repeated new lines at end of file (cd "$ROOT" && find . -type f \ ! -path "./vendor/*" \ ! -path "./bin/*" \ ! -path "./.git/*" \ ! -path "./.idea/*" \ ! -name ".*" \ -print0 | \ xargs -0 -L1 bash -c 'test "$(tail -c 2 "$0")" || [ ! -s "$0" ] || (trimmed=$(printf "%s" "$(< $0)") && echo "$trimmed" > "$0")' || true) # Remove new lines at beginning of file (cd "$ROOT" && find . -type f \ ! -path "./vendor/*" \ ! -path "./bin/*" \ ! -path "./.git/*" \ ! -path "./.idea/*" \ ! -name ".*" \ -print0 | \ xargs -0 -L1 bash -c 'test "$(head -c 1 "$0")" || [ ! -s "$0" ] || (trimmed=$(sed '"'"'/./,$!d'"'"' "$0") && echo "$trimmed" > "$0")' || true) ================================================ FILE: dev/generate_cli_md.sh ================================================ #!/bin/bash # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -e ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. >/dev/null && pwd)" out_file=$ROOT/docs/clients/cli.md rm -f $out_file echo "building cli ..." make --no-print-directory -C $ROOT cli # Clear default environments cli_config_backup_path=$HOME/.cortex/cli-bak-$RANDOM.yaml mv $HOME/.cortex/cli.yaml $cli_config_backup_path echo "# CLI commands" >> $out_file commands=( "deploy" "get" "describe" "logs" "refresh" "delete" "cluster up" "cluster info" "cluster configure" "cluster down" "cluster export" "cluster health" "env configure" "env list" "env default" "env rename" "env delete" "version" "completion" ) echo "running help commands ..." for cmd in "${commands[@]}"; do echo '' >> $out_file echo "## ${cmd}" >> $out_file echo '' >> $out_file echo '```text' >> $out_file $ROOT/bin/cortex help ${cmd} >> $out_file echo '```' >> $out_file done # Bring back CLI config mv -f $cli_config_backup_path $HOME/.cortex/cli.yaml echo "updated $out_file" ================================================ FILE: dev/generate_python_client_md.sh ================================================ #!/bin/bash # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -euo pipefail if [[ "$OSTYPE" != "linux"* ]]; then echo "error: this script is only designed to run on linux" exit 1 fi ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. >/dev/null && pwd)" docs_path="$ROOT/docs/clients/python.md" pip3 uninstall -y cortex cd $ROOT/python/client pip3 install -e . pydoc-markdown -m cortex -m cortex.client --render-toc > $docs_path # title sed -i "s/# Table of Contents/# Python client/g" $docs_path # delete links sed -i "//g" $docs_path sed -i "s/^## deploy\\\_from\\\_file$/## deploy\\\_from\\\_file\n\n/g" $docs_path pip3 uninstall -y cortex rm -rf $ROOT/python/client/cortex.egg-info ================================================ FILE: dev/get_operator_url.py ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import sys import boto3 def main(): cluster_name = sys.argv[1] region = sys.argv[2] operator_url = get_operator_url(cluster_name, region) print("https://" + operator_url) def get_operator_url(cluster_name, region): client_elbv2 = boto3.client("elbv2", region_name=region) paginator = client_elbv2.get_paginator("describe_load_balancers") for load_balancer_page in paginator.paginate(PaginationConfig={"PageSize": 20}): load_balancers = { load_balancer["LoadBalancerArn"]: load_balancer for load_balancer in load_balancer_page["LoadBalancers"] } tag_descriptions = client_elbv2.describe_tags(ResourceArns=list(load_balancers.keys()))[ "TagDescriptions" ] for tag_description in tag_descriptions: foundClusterNameTag = False foundLoadBalancerTag = False for tags in tag_description["Tags"]: if tags["Key"] == "cortex.dev/cluster-name" and tags["Value"] == cluster_name: foundClusterNameTag = True if tags["Key"] == "cortex.dev/load-balancer" and tags["Value"] == "operator": foundLoadBalancerTag = True if foundClusterNameTag and foundLoadBalancerTag: load_balancer = load_balancers[tag_description["ResourceArn"]] return load_balancer["DNSName"] # usage: python get_operator_url.py CLUSTER_NAME REGION if __name__ == "__main__": main() ================================================ FILE: dev/load.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package main import ( "bytes" "crypto/tls" "fmt" "io/ioutil" "net/http" "os" "os/signal" "strings" "sync" "syscall" "time" "github.com/cortexlabs/cortex/pkg/lib/debug" "github.com/cortexlabs/cortex/pkg/lib/files" "go.uber.org/atomic" ) // usage: go run load.go // configuration options (either set _numConcurrent > 0 or _requestInterval > 0, and configure the corresponding section) const ( // constant in-flight requests _numConcurrent = 3 _requestDelay = 0 * time.Millisecond _numRequestsPerThread = 0 // 0 means loop infinitely _numMainLoops = 1 // only relevant if _numRequestsPerThread > 0 // constant requests per second _requestInterval = 0 * time.Millisecond _numRequests uint64 = 0 // 0 means loop infinitely _maxInFlight = 5 // other options _printSuccessDots = true _printBody = false _printHTTPErrors = true _printGoErrors = true ) type Counter struct { sync.Mutex count int64 } var ( _requestCount = atomic.Uint64{} _successCount = atomic.Uint64{} _httpErrCount = atomic.Uint64{} // HTTP error response codes _goErrCount = atomic.Uint64{} // actual errors in go (includes "connection reset by peer") ) var _client = &http.Client{ Timeout: 0, // no timeout Transport: &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, }, } func main() { if _numConcurrent > 0 && _requestInterval > 0 { fmt.Println("error: you must set either _numConcurrent or _requestInterval > 0, but not both") os.Exit(1) } if _numConcurrent == 0 && _requestInterval == 0 { fmt.Println("error: you must set either _numConcurrent or _requestInterval > 0") os.Exit(1) } url, jsonPathOrString := mustExtractArgs() var jsonBytes []byte if jsonPathOrString == "" { jsonBytes = nil } else if strings.HasPrefix(jsonPathOrString, "{") { jsonBytes = []byte(jsonPathOrString) } else { jsonBytes = mustReadJSONBytes(jsonPathOrString) } if _numConcurrent > 0 { runConstantInFlight(url, jsonBytes) } if _requestInterval > 0 { runConstantRequestsPerSecond(url, jsonBytes) } } func runConstantRequestsPerSecond(url string, jsonBytes []byte) { inFlightCount := Counter{} ticker := time.NewTicker(_requestInterval) done := make(chan bool) c := make(chan os.Signal, 1) signal.Notify(c, os.Interrupt, syscall.SIGTERM) go func() { <-c done <- true }() start := time.Now() FOR_LOOP: for { select { case <-done: break FOR_LOOP case <-ticker.C: go runConstantRequestsPerSecondIteration(url, jsonBytes, &inFlightCount, done) } } elapsed := time.Since(start) requestRate := float64(_requestCount.Load()) / elapsed.Seconds() fmt.Printf("\nelapsed time: %s | %d requests @ %f req/s | %d succeeded | %d http errors | %d go errors\n", elapsed, _requestCount.Load(), requestRate, _successCount.Load(), _httpErrCount.Load(), _goErrCount.Load()) } func runConstantRequestsPerSecondIteration(url string, jsonBytes []byte, inFlightCount *Counter, done chan bool) { if _maxInFlight > 0 { inFlightCount.Lock() if inFlightCount.count >= _maxInFlight { inFlightCount.Unlock() fmt.Printf("\nreached max in-flight (%d)\n", _maxInFlight) return } inFlightCount.count++ inFlightCount.Unlock() } makeRequest(url, jsonBytes) if _numRequests > 0 && _requestCount.Load() >= _numRequests { done <- true } if _maxInFlight > 0 { inFlightCount.Lock() inFlightCount.count-- inFlightCount.Unlock() } } func runConstantInFlight(url string, jsonBytes []byte) { if _numRequestsPerThread > 0 { fmt.Printf("spawning %d threads, %d requests each, %s delay on each\n", _numConcurrent, _numRequestsPerThread, _requestDelay.String()) } else { fmt.Printf("spawning %d infinite threads, %s delay on each\n", _numConcurrent, _requestDelay.String()) } var summedRequestCount uint64 var summedSuccessCount uint64 var summedHTTPErrCount uint64 var summedGoErrCount uint64 start := time.Now() loopNum := 1 for { wasKilled := runConstantInFlightIteration(url, jsonBytes, loopNum) summedRequestCount += _requestCount.Load() summedSuccessCount += _successCount.Load() summedHTTPErrCount += _httpErrCount.Load() summedGoErrCount += _goErrCount.Load() _requestCount.Store(0) _successCount.Store(0) _httpErrCount.Store(0) _goErrCount.Store(0) if loopNum >= _numMainLoops || wasKilled { break } loopNum++ } if _numMainLoops > 1 { elapsed := time.Since(start) requestRate := float64(summedRequestCount) / elapsed.Seconds() fmt.Printf("\ntotal elapsed time: %s | %d requests @ %f req/s | %d succeeded | %d http errors | %d go errors\n", elapsed, summedRequestCount, requestRate, summedSuccessCount, summedHTTPErrCount, summedGoErrCount) } } func runConstantInFlightIteration(url string, jsonBytes []byte, loopNum int) bool { start := time.Now() wasKilled := false killed := make(chan bool) c := make(chan os.Signal, 1) signal.Notify(c, os.Interrupt, syscall.SIGTERM) go func() { <-c killed <- true }() doneChans := make([]chan struct{}, _numConcurrent) for i := range doneChans { doneChans[i] = make(chan struct{}) } for i := range doneChans { doneChan := doneChans[i] go func() { makeRequestLoop(url, jsonBytes) doneChan <- struct{}{} }() } LOOP: for _, doneChan := range doneChans { select { case <-killed: wasKilled = true break LOOP case <-doneChan: continue } } elapsed := time.Now().Sub(start) requestRate := float64(_requestCount.Load()) / elapsed.Seconds() fmt.Printf("\nelapsed time: %s | %d requests @ %f req/s | %d succeeded | %d http errors | %d go errors\n", elapsed, _requestCount.Load(), requestRate, _successCount.Load(), _httpErrCount.Load(), _goErrCount.Load()) return wasKilled } func makeRequestLoop(url string, jsonBytes []byte) { var i int isFirstIteration := true for true { if !isFirstIteration && _requestDelay != 0 { time.Sleep(_requestDelay) } isFirstIteration = false if _numRequestsPerThread > 0 { if i >= _numRequestsPerThread { return } i++ } makeRequest(url, jsonBytes) } } func makeRequest(url string, jsonBytes []byte) { request, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonBytes)) if err != nil { fmt.Print("\n" + debug.Sppg(err)) return } request.Header.Set("Content-Type", "application/json") _requestCount.Inc() response, err := _client.Do(request) if err != nil { _goErrCount.Inc() if _printGoErrors { fmt.Print("\n" + debug.Sppg(err)) } return } body, bodyReadErr := ioutil.ReadAll(response.Body) response.Body.Close() if response.StatusCode == 200 { _successCount.Inc() } else { _httpErrCount.Inc() if _printHTTPErrors { if bodyReadErr == nil { fmt.Printf("\nstatus code: %d; body: %s\n", response.StatusCode, string(body)) } else { fmt.Printf("\nstatus code: %d; error reading body: %s\n", response.StatusCode, bodyReadErr.Error()) } return } } if _printSuccessDots { fmt.Print(".") } if _printBody { bodyStr := string(body) if bodyStr == "" { bodyStr = "(no body)" } fmt.Print(bodyStr) } } func mustReadJSONBytes(jsonPath string) []byte { jsonBytes, err := files.ReadFileBytes(jsonPath) if err != nil { fmt.Println(err.Error()) os.Exit(1) } return jsonBytes } func mustExtractArgs() (string, string) { if len(os.Args) != 3 { fmt.Println("usage: go run load.go ") os.Exit(1) } url := os.Args[1] jsonPathOrString := os.Args[2] return url, jsonPathOrString } ================================================ FILE: dev/minimum_aws_policy.json ================================================ { "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Action": "iam:CreateServiceLinkedRole", "Resource": "*", "Condition": { "StringEquals": { "iam:AWSServiceName": [ "autoscaling.amazonaws.com", "ec2scheduled.amazonaws.com", "elasticloadbalancing.amazonaws.com", "spot.amazonaws.com", "spotfleet.amazonaws.com", "transitgateway.amazonaws.com" ] } } }, { "Effect": "Allow", "Action": "iam:CreateServiceLinkedRole", "Resource": "*", "Condition": { "StringEquals": { "iam:AWSServiceName": [ "eks.amazonaws.com", "eks-nodegroup.amazonaws.com", "eks-fargate.amazonaws.com" ] } } }, { "Effect": "Allow", "Action": [ "logs:ListTagsLogGroup", "iam:GetRole", "logs:TagLogGroup", "ssm:GetParameters", "ssm:GetParameter", "logs:CreateLogGroup" ], "Resource": [ "arn:*:ssm:*:$CORTEX_ACCOUNT_ID:parameter/aws/*", "arn:*:ssm:*::parameter/aws/*", "arn:*:logs:$CORTEX_REGION:$CORTEX_ACCOUNT_ID:log-group:$CORTEX_CLUSTER_NAME", "arn:*:iam::$CORTEX_ACCOUNT_ID:role/*" ] }, { "Effect": "Allow", "Action": [ "iam:CreateInstanceProfile", "logs:ListTagsLogGroup", "logs:DescribeLogStreams", "iam:TagRole", "iam:GetPolicy", "iam:CreatePolicy", "iam:DeletePolicy", "iam:ListPolicyVersions", "iam:RemoveRoleFromInstanceProfile", "iam:CreateRole", "iam:AttachRolePolicy", "iam:PutRolePolicy", "iam:AddRoleToInstanceProfile", "iam:ListInstanceProfilesForRole", "iam:PassRole", "logs:CreateLogStream", "iam:DetachRolePolicy", "logs:TagLogGroup", "iam:ListAttachedRolePolicies", "iam:DeleteRolePolicy", "iam:DeleteOpenIDConnectProvider", "iam:TagOpenIDConnectProvider", "iam:DeleteInstanceProfile", "iam:GetRole", "iam:GetInstanceProfile", "iam:DeleteRole", "iam:ListInstanceProfiles", "logs:CreateLogGroup", "logs:PutLogEvents", "logs:DeleteLogGroup", "iam:CreateOpenIDConnectProvider", "iam:GetOpenIDConnectProvider", "iam:GetRolePolicy" ], "Resource": [ "arn:*:iam::$CORTEX_ACCOUNT_ID:instance-profile/eksctl-*", "arn:*:iam::$CORTEX_ACCOUNT_ID:role/eksctl-*", "arn:*:iam::$CORTEX_ACCOUNT_ID:policy/eksctl-*", "arn:*:iam::$CORTEX_ACCOUNT_ID:role/aws-service-role/eks-nodegroup.amazonaws.com/AWSServiceRoleForAmazonEKSNodegroup", "arn:*:iam::$CORTEX_ACCOUNT_ID:role/eksctl-managed-*", "arn:*:iam::$CORTEX_ACCOUNT_ID:oidc-provider/*", "arn:*:logs:$CORTEX_REGION:$CORTEX_ACCOUNT_ID:log-group:$CORTEX_CLUSTER_NAME:*" ] }, { "Effect": "Allow", "Action": [ "iam:CreatePolicy", "iam:GetPolicyVersion", "iam:ListPolicyVersions", "iam:DeletePolicy", "iam:CreatePolicyVersion", "iam:DeletePolicyVersion" ], "Resource": "arn:*:iam::$CORTEX_ACCOUNT_ID:policy/cortex-*" }, { "Effect": "Allow", "Action": [ "sqs:ListQueues", "iam:GetPolicy", "ecr:GetAuthorizationToken", "cloudformation:*", "elasticloadbalancing:*", "autoscaling:*", "cloudwatch:*", "ecr:BatchGetImage", "kms:DescribeKey", "ec2:*", "sts:GetCallerIdentity", "eks:*", "kms:CreateGrant", "acm:DescribeCertificate", "servicequotas:ListServiceQuotas", "logs:PutRetentionPolicy" ], "Resource": "*" }, { "Effect": "Allow", "Action": "sqs:*", "Resource": "arn:*:sqs:$CORTEX_REGION:$CORTEX_ACCOUNT_ID:cx-*" }, { "Effect": "Allow", "Action": "s3:*", "Resource": "arn:*:s3:::$CORTEX_CLUSTER_NAME*" }, { "Effect": "Allow", "Action": "s3:*", "Resource": "arn:*:s3:::$CORTEX_CLUSTER_NAME*/*" } ] } ================================================ FILE: dev/operator_local.sh ================================================ #!/bin/bash # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -euo pipefail operator_only="false" debug="false" positional_args=() while [[ $# -gt 0 ]]; do key="$1" case $key in --operator-only) operator_only="true" shift ;; --debug) debug="true" shift ;; *) positional_args+=("$1") shift ;; esac done set -- "${positional_args[@]}" positional_args=() for i in "$@"; do case $i in *) positional_args+=("$1") shift ;; esac done set -- "${positional_args[@]}" for arg in "$@"; do if [[ "$arg" == -* ]]; then echo "unknown flag: $arg" exit 1 fi done if [ "$operator_only" = "true" ] && [ "$debug" = "true" ]; then echo "error: --operator-only and --debug cannot both be set" exit 1 fi ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. >/dev/null && pwd)" eval $(python3 $ROOT/manager/cluster_config_env.py "$ROOT/dev/config/cluster.yaml") export CORTEX_DEV_DEFAULT_IMAGE_REGISTRY="$CORTEX_DEV_DEFAULT_IMAGE_REGISTRY" python3 $ROOT/dev/update_cli_config.py "$HOME/.cortex/cli.yaml" "${CORTEX_CLUSTER_NAME}" "http://localhost:8888" cp -r $ROOT/dev/config/cluster.yaml ~/.cortex/cluster-dev.yaml if grep -qiP '^telemetry:\s*false\s*$' ~/.cortex/cli.yaml; then echo "telemetry: false" >> ~/.cortex/cluster-dev.yaml fi export CORTEX_OPERATOR_IN_CLUSTER=false export CORTEX_CLUSTER_CONFIG_PATH=~/.cortex/cluster-dev.yaml export CORTEX_DISABLE_JSON_LOGGING=true export CORTEX_LOG_LEVEL=debug export CORTEX_PROMETHEUS_URL="http://localhost:9090" portForwardCMD="kubectl port-forward -n default prometheus-prometheus-0 9090" kill $(pgrep -f "${portForwardCMD}") >/dev/null 2>&1 || true echo "Port-forwarding Prometheus to localhost:9090" eval "${portForwardCMD}" >/dev/null 2>&1 & mkdir -p $ROOT/bin if [ "$operator_only" = "true" ]; then kill $(pgrep -f rerun) >/dev/null 2>&1 || true rerun -watch $ROOT/pkg $ROOT/dev/config $ROOT/cmd/operator -run sh -c \ "clear && echo 'building operator...' && go build -o $ROOT/bin/operator $ROOT/cmd/operator && echo 'starting local operator...' && $ROOT/bin/operator" elif [ "$debug" = "true" ]; then DEBUG_CMD="dlv --listen=:2345 --headless=true --api-version=2 debug $ROOT/cmd/operator --output ${ROOT}/bin/__debug_bin" kill $(pgrep -f "${DEBUG_CMD}") >/dev/null 2>&1 || true kill $(pgrep -f __debug_bin) >/dev/null 2>&1 || true echo 'starting local operator in debug mode...' && eval "${DEBUG_CMD}" else kill $(pgrep -f rerun) >/dev/null 2>&1 || true rerun -watch $ROOT/pkg $ROOT/cli $ROOT/dev/config $ROOT/cmd/operator -run sh -c \ "clear && echo 'building cli...' && go build -o $ROOT/bin/cortex $ROOT/cli && echo 'building operator...' && go build -o $ROOT/bin/operator $ROOT/cmd/operator && echo 'starting local operator...' && $ROOT/bin/operator" fi # go run -race $ROOT/cmd/operator # Check for race conditions. Doesn't seem to catch them all? ================================================ FILE: dev/prometheus.md ================================================ # Metrics ## Updating metrics When new metrics/labels/exporters are added to be scraped by prometheus, make sure the following list **is updated** as well to keep track of what metrics/labels are needed or not. The following is a list of metrics that are currently in use. #### Cortex metrics 1. cortex_in_flight_requests with the following labels: 1. api_name 1. cortex_async_request_count with the following labels: 1. api_name 1. api_kind 1. status_code 1. cortex_async_active with the following labels: 1. api_name 1. api_kind 1. cortex_async_queued with the following labels: 1. api_name 1. api_kind 1. cortex_async_in_flight with the following labels: 1. api_name 1. api_kind 1. cortex_async_latency_bucket with the following labels: 1. api_name 1. api_kind 1. cortex_batch_succeeded with the following labels: 1. api_name 1. cortex_batch_failed with the following labels: 1. api_name 1. cortex_time_per_batch_sum with the following labels: 1. api_name 1. cortex_time_per_batch_count with the following labels: 1. api_name #### Istio metrics 1. istio_requests_total with the following labels: 1. destination_service 1. response_code 1. istio_request_duration_milliseconds_bucket with the following labels: 1. destination_service 1. le 1. istio_request_duration_milliseconds_sum with the following labels: 1. destination_service 1. istio_request_duration_milliseconds_count with the following labels: 1. destination_service #### Kubelet metrics 1. container_cpu_usage_seconds_total with the following labels: 1. pod 1. container 1. name 1. container_memory_working_set_bytes with the following labels: 1. pod 1. name 1. container #### Kube-state-metrics metrics 1. kube_pod_container_resource_requests with the following labels: 1. exported_pod 1. resource 1. exported_container (required for not dropping the values for each container of each pod) 1. kube_pod_info with the following labels: 1. exported_pod 1. kube_deployment_status_replicas_available with the following labels: 1. deployment 1. kube_job_status_active with the following labels: 1. job_name #### DCGM metrics 1. DCGM_FI_DEV_GPU_UTIL with the following labels: 1. exported_pod 1. DCGM_FI_DEV_FB_USED with the following labels: 1. exported_pod 1. DCGM_FI_DEV_FB_FREE with the following labels: 1. exported_pod #### Node metrics 1. node_cpu_seconds_total with the following labels: 1. job 1. mode 1. instance 1. cpu 1. node_load1 with the following labels: 1. job 1. instance 1. node_load5 with the following labels: 1. job 1. instance 1. node_load15 with the following labels: 1. job 1. instance 1. node_exporter_build_info with the following labels: 1. job 1. instance 1. node_memory_MemTotal_bytes with the following labels: 1. job 1. instance 1. node_memory_MemFree_bytes with the following labels: 1. job 1. instance 1. node_memory_Buffers_bytes with the following labels: 1. job 1. instance 1. node_memory_Cached_bytes with the following labels: 1. job 1. instance 1. node_memory_MemAvailable_bytes with the following labels: 1. job 1. instance 1. node_disk_read_bytes_total with the following labels: 1. job 1. instance 1. device 1. node_disk_written_bytes_total with the following labels: 1. job 1. instance 1. device 1. node_disk_io_time_seconds_total with the following labels: 1. job 1. instance 1. device 1. node_filesystem_size_bytes with the following labels: 1. job 1. instance 1. fstype 1. mountpoint 1. device 1. node_filesystem_avail_bytes with the following labels: 1. job 1. instance 1. fstype 1. device 1. node_network_receive_bytes_total with the following labels: 1. job 1. instance 1. device 1. node_network_transmit_bytes_total with the following labels: 1. job 1. instance 1. device ##### Prometheus rules for the node exporter 1. instance:node_cpu_utilisation:rate1m from the following metrics: 1. node_cpu_seconds_total with the following labels: 1. job 1. mode 1. instance:node_num_cpu:sum from the following metrics: 1. node_cpu_seconds_total with the following labels: 1. job 1. instance:node_load1_per_cpu:ratio from the following metrics: 1. node_load1 with the following labels: 1. job 1. instance:node_memory_utilisation:ratio from the following metrics: 1. node_memory_MemTotal_bytes with the following labels: 1. job 1. node_memory_MemAvailable_bytes with the following labels: 1. job 1. instance:node_vmstat_pgmajfault:rate1m with the following metrics: 1. node_vmstat_pgmajfault with the following labels: 1. job 1. instance_device:node_disk_io_time_seconds:rate1m with the following metrics: 1. node_disk_io_time_seconds_total with the following labels: 1. job 1. device 1. instance_device:node_disk_io_time_weighted_seconds:rate1m with the following metrics: 1. node_disk_io_time_weighted_seconds with the following labels: 1. job 1. device 1. instance:node_network_receive_bytes_excluding_lo:rate1m with the following metrics: 1. node_network_receive_bytes_total with the following labels: 1. job 1. device 1. instance:node_network_transmit_bytes_excluding_lo:rate1m with the following metrics: 1. node_network_transmit_bytes_total with the following labels: 1. job 1. device 1. instance:node_network_receive_drop_excluding_lo:rate1m with the following metrics: 1. node_network_receive_drop_total with the following labels: 1. job 1. device 1. instance:node_network_transmit_drop_excluding_lo:rate1m with the following metrics: 1. node_network_transmit_drop_total with the following labels: 1. job 1. device 1. cluster:cpu_utilization:ratio with the following metrics: 1. instance:node_cpu_utilisation:rate1m 1. instance:node_num_cpu:sum 1. cluster:load1:ratio with the following metrics: 1. instance:node_load1_per_cpu:ratio 1. cluster:memory_utilization:ratio with the following metrics: 1. instance:node_memory_utilisation:ratio 1. cluster:vmstat_pgmajfault:rate1m with the following metrics: 1. instance:node_vmstat_pgmajfault:rate1m 1. cluster:network_receive_bytes_excluding_low:rate1m with the following metrics: 1. instance:node_network_receive_bytes_excluding_lo:rate1m 1. cluster:network_transmit_bytes_excluding_lo:rate1m with the following metrics: 1. instance:node_network_transmit_bytes_excluding_lo:rate1m 1. cluster:network_receive_drop_excluding_lo:rate1m with the following metrics: 1. instance:node_network_receive_drop_excluding_lo:rate1m 1. cluster:network_transmit_drop_excluding_lo:rate1m with the following metrics: 1. instance:node_network_transmit_drop_excluding_lo:rate1m 1. cluster:disk_io_utilization:ratio with the following metrics: 1. instance_device:node_disk_io_time_seconds:rate1m 1. cluster:disk_io_saturation:ratio with the following metrics: 1. instance_device:node_disk_io_time_weighted_seconds:rate1m 1. cluster:disk_space_utilization:ratio with the following metrics: 1. node_filesystem_size_bytes with the following labels: 1. job 1. fstype 1. mountpoint 1. node_filesystem_avail_bytes with the following labels: 1. job 1. fstype 1. mountpoint ## Re-introducing dropped metrics/labels If you need to add some metrics/labels back for some particular use case, comment out every `metricRelabelings:` section (except the one from the `prometheus-operator.yaml` file), determine which metrics/labels you want to add back (i.e. by using the explorer from Grafana) and then re-edit the appropriate `metricRelabelings:` sections to account for the un-dropped metrics/labels. ## Prometheus Analysis ### Go Pprof To analyse the memory allocations of prometheus, run `kubectl port-forward prometheus-prometheus-0 9090:9090`, and then run `go tool pprof -symbolize=remote -inuse_space localhost:9090/debug/pprof/heap`. Once you get the interpreter, you can run `top` or `dot` for a more detailed hierarchy of the memory usage. ### TSDB To analyse the TSDB of prometheus, exec into the `prometheus-prometheus-0` pod, `cd` into `/tmp`, and run the following code-block: ```bash wget https://github.com/prometheus/prometheus/releases/download/v1.7.3/prometheus-1.7.3.linux-amd64.tar.gz tar -xzf prometheus-* cd prometheus-* ./tsdb analyze /prometheus | less ``` *Useful link: https://www.robustperception.io/using-tsdb-analyze-to-investigate-churn-and-cardinality* Or you can go to `localhost:9090` -> `Status` -> `TSDB Status`, but it's not as complete as running a binary analysis. ================================================ FILE: dev/registry.sh ================================================ #!/bin/bash # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. CORTEX_VERSION=master set -eo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. >/dev/null && pwd)" source $ROOT/build/images.sh source $ROOT/dev/util.sh images_that_can_run_locally="operator manager" if [ -f "$ROOT/dev/config/env.sh" ]; then source $ROOT/dev/config/env.sh fi AWS_ACCOUNT_ID=${AWS_ACCOUNT_ID:-} AWS_REGION=${AWS_REGION:-} skip_push="false" include_arm64_arch="false" positional_args=() while [[ $# -gt 0 ]]; do key="$1" case $key in --skip-push) skip_push="true" shift ;; --include-arm64-arch) include_arm64_arch="true" shift ;; *) positional_args+=("$1") shift ;; esac done set -- "${positional_args[@]}" positional_args=() for i in "$@"; do case $i in *) positional_args+=("$1") shift ;; esac done set -- "${positional_args[@]}" for arg in "$@"; do if [[ "$arg" == -* ]]; then echo "unknown flag: $arg" exit 1 fi done cmd=${1:-""} sub_cmd=${2:-""} registry_push_url="" if [ "$skip_push" != "true" ]; then registry_push_url="$AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com" fi is_registry_logged_in="false" function registry_login() { if [ "$is_registry_logged_in" = "false" ]; then blue_echo "Logging in to ECR..." aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin $registry_push_url is_registry_logged_in="true" green_echo "Success\n" fi } function create_ecr_repository() { for image in "${all_images[@]}"; do aws ecr create-repository --repository-name=cortexlabs/$image --region=$AWS_REGION || true done } ### HELPERS ### function build_and_push() { local image=$1 local include_arm64_arch=$2 local dir="${ROOT}/images/${image}" set -euo pipefail if [[ ! " ${multi_arch_images[*]} " =~ " $image " ]]; then include_arm64_arch="false" fi if [ ! -n "$AWS_ACCOUNT_ID" ] || [ ! -n "$AWS_REGION" ]; then echo "AWS_ACCOUNT_ID or AWS_REGION env vars not found" exit 1 fi tag=$CORTEX_VERSION push_or_not_flag="" running_operation="Building" finished_operation="Built" if [ "$skip_push" = "false" ]; then push_or_not_flag="--push" running_operation+=" and pushing" finished_operation+=" and pushed" registry_login fi if [ "$include_arm64_arch" = "true" ]; then blue_echo "$running_operation $image:$tag (amd64 and arm64)..." else blue_echo "$running_operation $image:$tag (amd64)..." fi platforms="linux/amd64" if [ "$include_arm64_arch" = "true" ]; then platforms+=",linux/arm64" fi docker buildx build $ROOT -f $dir/Dockerfile -t $AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com/cortexlabs/$image:$tag --platform $platforms $push_or_not_flag if [ "$include_arm64_arch" = "true" ]; then green_echo "$finished_operation $image:$tag (amd64 and arm64)" else green_echo "$finished_operation $image:$tag (amd64)" fi if [[ " ${images_that_can_run_locally[*]} " =~ " $image " ]] && [[ "$include_arm64_arch" == "false" ]]; then blue_echo "Exporting $image:$tag to local docker..." docker buildx build $ROOT -f $dir/Dockerfile -t cortexlabs/$image:$tag -t $AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com/cortexlabs/$image:$tag --platform $platforms --load green_echo "Exported $image:$tag to local docker" fi } function cleanup_local() { echo "cleaning local repositories..." docker container prune -f docker image prune -f docker buildx prune -f } function cleanup_ecr() { echo "cleaning ECR repositories..." repos=$(aws ecr describe-repositories --output text | awk '{print $6}' | grep -P "\S") echo "$repos" | while IFS= read -r repo; do imageIDs=$(aws ecr list-images --repository-name "$repo" --filter tagStatus=UNTAGGED --query "imageIds[*]" --output text) echo "$imageIDs" | while IFS= read -r imageId; do if [ ! -z "$imageId" ]; then echo "Removing from ECR: $repo/$imageId" aws ecr batch-delete-image --repository-name "$repo" --image-ids imageDigest="$imageId" >/dev/null; fi done done } function delete_ecr() { echo "deleting ECR repositories..." repos=$(aws ecr describe-repositories --output text | awk '{print $6}' | grep -P "\S") echo "$repos" | while IFS= read -r repo; do imageIDs=$(aws ecr delete-repository --force --repository-name "$repo") echo "deleted: $repo" done } function validate_env() { if [ "$skip_push" != "true" ]; then if [ -z ${AWS_REGION} ] || [ -z ${AWS_ACCOUNT_ID} ]; then echo "error: environment variables AWS_REGION and AWS_ACCOUNT_ID should be exported in dev/config/env.sh" exit 1 fi fi } # validate environment is correctly set on env.sh validate_env # usage: registry.sh clean if [ "$cmd" = "clean" ]; then delete_ecr create_ecr_repository # usage: registry.sh create elif [ "$cmd" = "create" ]; then create_ecr_repository # usage: registry.sh update-single IMAGE elif [ "$cmd" = "update-single" ]; then image=$sub_cmd build_and_push $image $include_arm64_arch # usage: registry.sh update all|dev|api elif [ "$cmd" = "update" ]; then images_to_build=() if [ "$sub_cmd" == "all" ]; then images_to_build+=( "${non_dev_images[@]}" ) fi if [[ "$sub_cmd" == "all" || "$sub_cmd" == "dev" ]]; then images_to_build+=( "${dev_images[@]}" ) fi for image in "${images_to_build[@]}"; do build_and_push $image $include_arm64_arch done # usage: registry.sh clean-cache elif [ "$cmd" = "clean-cache" ]; then cleanup_local else echo "unknown command: $cmd" exit 1 fi ================================================ FILE: dev/update_cli_config.py ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import sys import yaml import os def update_cli_config(cli_config_file_path, env_name, operator_endpoint): new_env = { "name": env_name, "operator_endpoint": operator_endpoint, } try: with open(cli_config_file_path, "r") as f: cli_config = yaml.safe_load(f) if cli_config is None: raise Exception("blank cli config file") except: cli_config = {"environments": [new_env]} with open(cli_config_file_path, "w") as f: yaml.dump(cli_config, f, default_flow_style=False) return if len(cli_config.get("environments", [])) == 0: cli_config["environments"] = [new_env] with open(cli_config_file_path, "w") as f: yaml.dump(cli_config, f, default_flow_style=False) return replaced = False for i, prev_env in enumerate(cli_config["environments"]): if prev_env.get("name") == env_name: cli_config["environments"][i] = new_env replaced = True break if not replaced: cli_config["environments"].append(new_env) with open(cli_config_file_path, "w") as f: yaml.dump(cli_config, f, default_flow_style=False) if __name__ == "__main__": update_cli_config( cli_config_file_path=sys.argv[1], env_name=sys.argv[2], operator_endpoint=sys.argv[3], ) ================================================ FILE: dev/util.sh ================================================ #!/bin/bash # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. function blue_echo() { echo -e "\033[1;34m$1\033[0m" } function green_echo() { echo -e "\033[1;32m$1\033[0m" } function error_echo() { echo -e "\033[1;31mERROR: \033[0m$1" } ================================================ FILE: dev/versions.md ================================================ # Upgrade notes ## eksctl 1. Find the latest release on [GitHub](https://github.com/weaveworks/eksctl/releases) and check the changelog 1. Search the code base for the old version to find where to update it (e.g. `manager/Dockerfile`) 1. Update `generate_eks.py` if necessary 1. Check that `eksctl utils write-kubeconfig` log filter still behaves as desired, and logs in `cortex cluster up` look good. 1. Update eksctl on your dev machine: `curl --location "https://github.com/weaveworks/eksctl/releases/download/v0.107.0/eksctl_$(uname -s)_amd64.tar.gz" | tar xz -C /tmp && sudo mv -f /tmp/eksctl /usr/local/bin` 1. Check if eksctl iam polices changed by comparing the previous version of the eksctl policy docs to the new version's and update `./dev/minimum_aws_policy.json` and `docs/clusters/management/auth.md` accordingly. https://github.com/weaveworks/eksctl/blob/v0.107.0/userdocs/src/usage/minimum-iam-policies.md ## Kubernetes 1. Find the latest version of Kubernetes supported by eksctl ([source code](https://github.com/weaveworks/eksctl/blob/master/pkg/apis/eksctl.io/v1alpha5/types.go)) 1. Update the version in `generate_eks.py` 1. Update `ami.json` (see release checklist for instructions) 1. See instructions for upgrading the Kubernetes client below ## kube-proxy (IPVS mode) 1. Before spinning up a Cortex cluster with the new eksctl/kubernetes/eks updates, make sure to have the `setup_ipvs` functional call commented out in the manager. 1. Once the cluster is up, run the `cat /var/lib/kube-proxy-config/config` command on any of the kube-proxy pods of the cluster. Compare the output of that with what the `upgrade_kube_proxy_mode.py` script is applying and make sure it's still applicable, if not, check out the spec of the [KubeProxyConfiguration](https://kubernetes.io/docs/reference/config-api/kube-proxy-config.v1alpha1/) and upgrade `upgrade_kube_proxy_mode.py`. 1. Compare the spec of the `kube-proxy.patch.yaml` patch with the current spec of the kube-proxy daemoset and make sure it's still applicable. You can either inspect the `kube-proxy` command helper by exec-ing into the pod or by looking at the [kube-proxy](https://kubernetes.io/docs/reference/command-line-tools-reference/kube-proxy/) documentation for the respective version of Kubernetes. 1. Once both config map and the daemonset are updated and the kube-proxy pod(s) has/have started, make sure you notice the `Using ipvs Proxier` log. ## aws-iam-authenticator 1. Find the latest release [here](https://docs.aws.amazon.com/eks/latest/userguide/install-aws-iam-authenticator.html) 1. Update the version in `images/manager/Dockerfile` ## kubectl 1. Find the latest release [here](https://storage.googleapis.com/kubernetes-release/release/stable.txt) 1. Update the version in `images/manager/Dockerfile` and `images/operator/Dockerfile` 1. Update your local version and alert developers * Linux: ```shell mkdir -p $HOME/temp && \ cd $HOME/temp && \ curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl && \ chmod +x ./kubectl && \ sudo mv -f ./kubectl /usr/local/bin/kubectl && \ if [ -f $HOME/.bash_profile ]; then source $HOME/.bash_profile; else source $HOME/.bashrc; fi && \ cd - && \ kubectl version ``` * Mac: 1. `brew upgrade kubernetes-cli` 1. refresh shell 1. `kubectl version` ## Istio 1. Find the latest [release](https://istio.io/latest/news/releases) and check the release notes (here are the [latest IstioOperator Options](https://istio.io/latest/docs/reference/config/istio.operator.v1alpha1/)) 1. Update the version in `images/manager/Dockerfile`. 1. Update the version in all `images/istio-*` Dockerfiles. 1. Update `istio.yaml.j2`, `apis.yaml.j2`, `operator.yaml.j2`, and `pkg/lib/k8s` as necessary. 1. Update `install.sh` as necessary. ## AWS CNI 1. Update the CNI version in `generate_eks.py` ([CNI releases](https://github.com/aws/amazon-vpc-cni-k8s/releases)) 1. Update the go module version (see `Go > Non-versioned modules` section below) 1. Check if new instance types were added by running the script below (update the two env vars at the top). 1. If there are new instance types, check if any changes need to be made to `servicequotas.go` or `validateInstanceType()`. ```bash PREV_RELEASE=1.10.1 NEW_RELEASE=1.11.0 wget -q -O cni_supported_instances_prev.txt https://raw.githubusercontent.com/aws/amazon-vpc-cni-k8s/v${PREV_RELEASE}/pkg/awsutils/vpc_ip_resource_limit.go; wget -q -O cni_supported_instances_new.txt https://raw.githubusercontent.com/aws/amazon-vpc-cni-k8s/v${NEW_RELEASE}/pkg/awsutils/vpc_ip_resource_limit.go; git diff --no-index cni_supported_instances_prev.txt cni_supported_instances_new.txt; rm -rf cni_supported_instances_prev.txt; rm -rf cni_supported_instances_new.txt ``` ## Go 1. Find the latest release on Golang's [release page](https://golang.org/doc/devel/release.html) ( or [downloads page](https://golang.org/dl/)) and check the changelog 1. Search the codebase for the current minor version (e.g. `1.17`), update versions as appropriate 1. Update your local version and alert developers: * Linux: ```shell mkdir -p $HOME/temp cd $HOME/temp wget https://dl.google.com/go/go1.17.3.linux-amd64.tar.gz && \ tar -xvf go1.17.3.linux-amd64.tar.gz && \ sudo rm -rf /usr/local/go && \ sudo mv -f go /usr/local && \ rm go1.17.3.linux-amd64.tar.gz && \ if [ -f $HOME/.bash_profile ]; then source $HOME/.bash_profile; else source $HOME/.bashrc; fi && \ cd - && \ go version ``` * Mac: 1. `brew upgrade go` or `brew install go@1.17` 1. refresh shell 1. `go version` 1. Update go modules as necessary ## Go modules ### Kubernetes client 1. Find the latest patch [release](https://github.com/kubernetes/client-go) for the minor kubernetes version that we use (e.g. for k8s 1.21, use `client-go` version `v0.21.X`, where `X` is the latest available patch release) 1. Follow the "Update non-versioned modules" instructions using the updated version for `k8s.io/client-go` ### Istio client 1. Find the version of istio that we use in `images/manager/Dockerfile` 1. Follow the "Update non-versioned modules" instructions using the updated version for `istio.io/client-go` ### docker/engine/client 1. Find the latest tag from [here](https://github.com/docker/engine/tags) 1. Follow the "Update non-versioned modules" instructions using the updated version for `docker/engine` _note: docker client installation may be able to be improved, see https://github.com/moby/moby/issues/39302#issuecomment-639687466_ ### cortexlabs/yaml 1. Check [go-yaml/yaml](https://github.com/go-yaml/yaml/commits/v2) to see if there were new releases since [cortexlabs/yaml](https://github.com/cortexlabs/yaml/commits/v2) 1. `git clone git@github.com:cortexlabs/yaml.git && cd yaml` 1. `git remote add upstream https://github.com/go-yaml/yaml && git fetch upstream` 1. `git merge upstream/v2` 1. `git push origin v2` 1. Follow the "Update non-versioned modules" instructions using the desired commit sha for `cortexlabs/yaml` ### cortexlabs/go-input 1. Check [tcnksm/go-input](https://github.com/tcnksm/go-input/commits/master) to see if there were new releases since [cortexlabs/go-input](https://github.com/cortexlabs/go-input/commits/master) 1. `git clone git@github.com:cortexlabs/go-input.git && cd go-input` 1. `git remote add upstream https://github.com/tcnksm/go-input && git fetch upstream` 1. `git merge upstream/master` 1. `git push origin master` 1. Follow the "Update non-versioned modules" instructions using the desired commit sha for `cortexlabs/go-input` ### Non-versioned modules 1. `rm -rf go.mod go.sum && go mod init && go clean -modcache` 1. `go get k8s.io/client-go@v0.20.15 && go get k8s.io/apimachinery@v0.20.15 && go get k8s.io/api@v0.20.15` 1. `go get istio.io/client-go@v1.11.8 && go get istio.io/api@1.11.8` 1. `go get github.com/aws/amazon-vpc-cni-k8s/pkg/awsutils@v1.11.0` 1. `go get github.com/cortexlabs/yaml@31e52ba8433b683c471ef92cf1711fe67671dac5` 1. `go get github.com/cortexlabs/go-input@8b67a7a7b28d1c45f5c588171b3b50148462b247` 1. `go get github.com/xlab/treeprint@v1.0.0` 1. `go get -u sigs.k8s.io/controller-runtime@v0.8.3` 1. `echo -e '\nreplace github.com/docker/docker => github.com/docker/engine v19.03.13' >> go.mod` 1. `go get -u github.com/docker/distribution` 1. `go mod tidy` 1. Potentially skip these steps 1. For every non-indirect, non-hardcoded dependency in go.mod, update with `go get -u ` 1. `go mod tidy` 1. Re-run the relevant hardcoded `go get` commands above 1. `go mod tidy` 1. `make test` 1. `go mod tidy` 1. Check that the diff in `go.mod` is reasonable ## Nvidia device plugin 1. Update the version in `images/nvidia-device-plugin/Dockerfile` ([releases](https://github.com/NVIDIA/k8s-device-plugin/releases) , [Dockerhub](https://hub.docker.com/r/nvidia/k8s-device-plugin)) 1. In the [GitHub Repo](https://github.com/NVIDIA/k8s-device-plugin), find the latest release and go to this file ( replacing the version number): 1. Copy the contents to `manager/manifests/nvidia.yaml` 1. Update the link at the top of the file to the URL you copied from 1. Check that your diff is reasonable (and put back any of our modifications, e.g. the image path, rolling update strategy, resource requests, tolerations, node selector, priority class, etc) ## Neuron device plugin and scheduler 1. Update `images/neuron-device-plugin/Dockerfile` if necessary (see [here](https://gallery.ecr.aws/neuron/neuron-device-plugin) for the latest tag) 1. Update `images/neuron-scheduler/Dockerfile` if necessary (see [here](https://gallery.ecr.aws/neuron/neuron-scheduler) for the latest tag) 1. Copy the contents of `k8s-neuron-*` in [this folder](https://github.com/aws/aws-neuron-sdk/tree/master/src/k8) into `manager/manifests/inferentia.yaml` 1. Update the link at the top of the file to the URL you copied from 1. Check that your diff is reasonable (and put back any of our modifications) ## Metrics server 1. Find the latest release on [GitHub](https://github.com/kubernetes-incubator/metrics-server/releases) and check the changelog 1. Update the version in `images/metrics-server/Dockerfile` 1. Download the manifest referenced in the latest release in changelog 1. Copy the contents of the manifest into `manager/manifests/metrics-server.yaml` 1. Update accordingly (e.g. image, pull policy, resource request, etc): 1. Check that your diff is reasonable 1. You can confirm the metric server is running by showing the logs of the metrics-server pod, or via `kubectl get deployment metrics-server -n kube-system` and `kubectl get apiservice v1beta1.metrics.k8s.io -o yaml` ## Cluster autoscaler 1. Find the latest patch release for our current version of k8s (e.g. k8s v1.17 -> cluster-autocluster v1.17.3) on [GitHub](https://github.com/kubernetes/autoscaler/releases) and check the changelog 1. In the [GitHub Repo](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws), set the tree to the tag for the chosen release, and open `cloudprovider/aws/examples/cluster-autoscaler-autodiscover.yaml` (e.g. ) 1. Resolve merge conflicts with the template in `manager/manifests/cluster-autoscaler.yaml.j2`. 1. Clone our fork: `git clone git@github.com:cortexlabs/autoscaler.git` 1. Checkout our updated branch: `git checkout cluster-autoscaler-1.21.1-cortex` 1. List the most recent commit: `git log` 1. Reset the latest commit (use the SHA of the last non-cortex commit): `git reset ` 1. `git add *` 1. `git stash` 1. `git remote add upstream https://github.com/kubernetes/autoscaler.git` 1. `git fetch upstream` 1. Checkout the appropriate version tag, e.g. `git checkout cluster-autoscaler-1.22.2 -b cluster-autoscaler-1.22.2-cortex` 1. `git stash pop` 1. Resolve any merge conflicts 1. Unstage and check the diff 1. `git add *; git commit -am "Add rate limiter"` 1. `git push origin cluster-autoscaler-1.22.2-cortex` 1. Update `images/cluster-autoscaler/Dockerfile` to use the new branch name (e.g. "cluster-autoscaler-1.22.2") in the `-b` flag's value from `git clone`. 1. Match the Go version of the builder in `images/cluster-autoscaler/Dockerfile` with that of the [cluster autoscaler's Dockerfile](https://github.com/kubernetes/autoscaler/blob/master/builder/Dockerfile). ## FluentBit 1. Find the latest release on [Docker Hub](https://hub.docker.com/r/amazon/aws-for-fluent-bit/tags?page=1&ordering=last_updated) 1. Update the base image version in `images/fluent-bit/Dockerfile` 1. Update `fluent-bit.yaml` as necessary (make sure to maintain all Cortex environment variables) ## Prometheus Operator / Prometheus Config Reloader 1. Find the latest release in the [GitHub Repo](https://github.com/prometheus-operator/prometheus-operator). 1. Copy the `bundle.yaml` file contents into `prometheus-operator.yaml`. 1. Replace the image in the Deployment resource with a cortex env var. 1. Update the base image versions in `images/prometheus-operator/Dockerfile` and `images/prometheus-config-reloader/Dockerfile`. ## Prometheus 1. Find the latest release on [Docker Hub](https://hub.docker.com/r/prom/prometheus/tags?page=1&ordering=last_updated), compatible to the current version of Prometheus Operator. 1. Update the base image version in `images/prometheus/Dockerfile`. 1. Update `prometheus-monitoring.yaml` as necessary, if that's the case. ## Prometheus StatsD Exporter 1. Find the latest release on [Docker Hub](https://registry.hub.docker.com/r/prom/statsd-exporter/tags?page=1&ordering=last_updated). 1. Update the base image version in `images/prometheus-statsd-exporter/Dockerfile`. 1. Update `prometheus-statsd-exporter.yaml` as necessary, if that's the case. ## Prometheus DCGM Exporter 1. Run `helm template` on the DCGM charts https://github.com/NVIDIA/gpu-monitoring-tools/tree/master/deployment/dcgm-exporter and save the output somewhere temporarily. 1. Update the base image version in `images/prometheus-dcgm-exporter/Dockerfile`. 1. Update `prometheus-dcgm-exporter.yaml` as necessary, if that's the case. Keep in mind that in our k8s template, the `ServiceMonitor` was changed to a `PodMonitor`. Remove any unnecessary labels. ## Prometheus kube-state-metrics Exporter 1. Run `helm template` on the kube-state-metrics charts from https://github.com/kubernetes/kube-state-metrics#helm-chart and save the output somewhere temporarily. 1. Update the base image version in `images/prometheus-kube-state-metrics/Dockerfile`. 1. Update `prometheus-kube-state-metrics.yaml` as necessary, if that's the case. Keep in mind that in our k8s template, the `ServiceMonitor` was changed to a `PodMonitor`. Remove any unnecessary labels. The update can also include adjusting the resource requests. ## Prometheus Kubelet Exporter 1. Check if https://github.com/prometheus-operator/kube-prometheus/blob/main/manifests/kubernetes-serviceMonitorKubelet.yaml has changed when compared to `manager/manifests/prometheus-kubelet-exporter`. ## Prometheus Node Exporter 1. Find the latest release in the Kube Prometheus [GitHub Repo](https://github.com/prometheus-operator/kube-prometheus/blob/main/manifests/). 1. Copy the `node-exporter-*.yaml` files contents into `prometheus-node-exporter.yaml`, but keep the prometheus rules resource. 1. Replace the image in the Deployment resource with a cortex env var. 1. Update the base image version in `images/prometheus-node-exporter/Dockerfile` 1. Update the base branch version in `images/kube-rbac-proxy/Dockerfile` (as well as the rest of the contents if necessary). ## Grafana 1. Find the latest release on [Docker Hub](https://registry.hub.docker.com/r/grafana/grafana/tags?page=1&ordering=last_updated). 1. Update the base image version in `images/grafana/Dockerfile`. 1. Update `grafana.yaml` as necessary, if that's the case. ## Event Exporter 1. Find the latest release on [GitHub](https://github.com/opsgenie/kubernetes-event-exporter) / [GitHub Container Registry](https://github.com/opsgenie/kubernetes-event-exporter/pkgs/container/kubernetes-event-exporter). 1. Update the base image version in `images/event-exporter/Dockerfile`. 1. Update `event-exporter.yaml` as necessary, if that's the case. ## Alpine base images 1. Find the latest release on [Dockerhub](https://hub.docker.com/_/alpine) 1. Search the codebase for `alpine` and update accordingly ## Python client dependencies 1. Update package versions in `install_requires` in `python/client/setup.py`. ================================================ FILE: docs/README.md ================================================ **Please read our documentation at [docs.cortexlabs.com](https://docs.cortexlabs.com)** ================================================ FILE: docs/clients/cli.md ================================================ # CLI commands ## deploy ```text create or update apis Usage: cortex deploy [CONFIG_FILE] [flags] Flags: -e, --env string environment to use -f, --force override the in-progress api update -y, --yes skip prompts -o, --output string output format: one of pretty|json (default "pretty") -h, --help help for deploy ``` ## get ```text get information about apis or jobs Usage: cortex get [API_NAME] [JOB_ID] [flags] Flags: -e, --env string environment to use -w, --watch re-run the command every 2 seconds -o, --output string output format: one of pretty|json (default "pretty") -v, --verbose show additional information (only applies to pretty output format) -h, --help help for get ``` ## describe ```text describe an api Usage: cortex describe [API_NAME] [flags] Flags: -e, --env string environment to use -w, --watch re-run the command every 2 seconds -h, --help help for describe ``` ## logs ```text get the logs for a workload Usage: cortex logs API_NAME [JOB_ID] [flags] Flags: -e, --env string environment to use -y, --yes skip prompts --random-pod stream logs from a random pod -h, --help help for logs ``` ## refresh ```text restart all replicas for an api (without downtime) Usage: cortex refresh API_NAME [flags] Flags: -e, --env string environment to use -f, --force override the in-progress api update -o, --output string output format: one of pretty|json (default "pretty") -h, --help help for refresh ``` ## delete ```text delete an api or stop a job Usage: cortex delete API_NAME [JOB_ID] [flags] Flags: -e, --env string environment to use -f, --force delete the api without confirmation -c, --keep-cache keep cached data for the api -o, --output string output format: one of pretty|json (default "pretty") -h, --help help for delete ``` ## cluster up ```text spin up a cluster on aws Usage: cortex cluster up CLUSTER_CONFIG_FILE [flags] Flags: -e, --configure-env string name of environment to configure (default: the name of your cluster) -y, --yes skip prompts -h, --help help for up ``` ## cluster info ```text get information about a cluster Usage: cortex cluster info [flags] Flags: -c, --config string path to a cluster configuration file -n, --name string name of the cluster -r, --region string aws region of the cluster -o, --output string output format: one of pretty|json|yaml (default "pretty") -e, --configure-env string name of environment to configure -d, --debug save the current cluster state to a file --print-config print the cluster config -y, --yes skip prompts -h, --help help for info ``` ## cluster configure ```text update the cluster's configuration Usage: cortex cluster configure CLUSTER_CONFIG_FILE [flags] Flags: -y, --yes skip prompts -h, --help help for configure ``` ## cluster down ```text spin down a cluster Usage: cortex cluster down [flags] Flags: -c, --config string path to a cluster configuration file -n, --name string name of the cluster -r, --region string aws region of the cluster -y, --yes skip prompts --keep-aws-resources skip deletion of resources that cortex provisioned on aws (bucket contents, ebs volumes, log group) -h, --help help for down ``` ## cluster export ```text download the configurations for all APIs Usage: cortex cluster export [flags] Flags: -c, --config string path to a cluster configuration file -n, --name string name of the cluster -r, --region string aws region of the cluster -h, --help help for export ``` ## cluster health ```text inspect the health of components in the cluster Usage: cortex cluster health [flags] Flags: -c, --config string path to a cluster configuration file -n, --name string name of the cluster -r, --region string aws region of the cluster -o, --output string output format: one of pretty|json (default "pretty") -h, --help help for health ``` ## env configure ```text configure an environment Usage: cortex env configure [ENVIRONMENT_NAME] [flags] Flags: -o, --operator-endpoint string set the operator endpoint without prompting -h, --help help for configure ``` ## env list ```text list all configured environments Usage: cortex env list [flags] Flags: -o, --output string output format: one of pretty|json (default "pretty") -h, --help help for list ``` ## env default ```text set the default environment Usage: cortex env default [ENVIRONMENT_NAME] [flags] Flags: -h, --help help for default ``` ## env rename ```text rename an environment Usage: cortex env rename EXISTING_NAME NEW_NAME [flags] Flags: -h, --help help for rename ``` ## env delete ```text delete an environment configuration Usage: cortex env delete [ENVIRONMENT_NAME] [flags] Flags: -h, --help help for delete ``` ## version ```text print the cli and cluster versions Usage: cortex version [flags] Flags: -e, --env string environment to use -h, --help help for version ``` ## completion ```text generate shell completion scripts to enable cortex shell completion: bash: add this to ~/.bash_profile (mac) or ~/.bashrc (linux): source <(cortex completion bash) note: bash-completion must be installed on your system; example installation instructions: mac: 1) install bash completion: brew install bash-completion 2) add this to your ~/.bash_profile: source $(brew --prefix)/etc/bash_completion 3) log out and back in, or close your terminal window and reopen it ubuntu: 1) install bash completion: apt update && apt install -y bash-completion # you may need sudo 2) open ~/.bashrc and uncomment the bash completion section, or add this: if [ -f /etc/bash_completion ] && ! shopt -oq posix; then . /etc/bash_completion; fi 3) log out and back in, or close your terminal window and reopen it zsh: option 1: add this to ~/.zshrc: source <(cortex completion zsh) if that failed, you can try adding this line (above the source command you just added): autoload -Uz compinit && compinit option 2: create a _cortex file in your fpath, for example: cortex completion zsh > /usr/local/share/zsh/site-functions/_cortex Note: this will also add the "cx" alias for cortex for convenience Usage: cortex completion SHELL [flags] Flags: -h, --help help for completion ``` ================================================ FILE: docs/clients/install.md ================================================ # Install ## Install the CLI ```bash # download CLI version 0.42.1 (Note the "v"): bash -c "$(curl -sS https://raw.githubusercontent.com/cortexlabs/cortex/v0.42.1/get-cli.sh)" ``` By default, the Cortex CLI is installed at `/usr/local/bin/cortex`. To install the executable elsewhere, export the `CORTEX_INSTALL_PATH` environment variable to your desired location before running the command above. ## Install the CLI and Python client via pip To install the latest version: ```bash pip install cortex ``` To install or upgrade to a specific version (e.g. v0.42.1): ```bash pip install cortex==0.42.1 ``` To upgrade to the latest version: ```bash pip install --upgrade cortex ``` ## Changing the CLI/client configuration directory By default, the CLI/client creates a directory at `~/.cortex/` and uses it to store environment configuration. To use a different directory, export the `CORTEX_CLI_CONFIG_DIR` environment variable before running any `cortex` commands. ================================================ FILE: docs/clients/python.md ================================================ # Python client * [cortex](#cortex) * [client](#client) * [new\_client](#new_client) * [env\_list](#env_list) * [env\_delete](#env_delete) * [cortex.client.Client](#cortex-client-client) * [deploy](#deploy) * [deploy\_from\_file](#deploy_from_file) * [get\_api](#get_api) * [list\_apis](#list_apis) * [get\_job](#get_job) * [refresh](#refresh) * [delete](#delete) * [stop\_job](#stop_job) # cortex ## client ```python client(env_name: Optional[str] = None) -> Client ``` Initialize a client based on the specified environment. If no environment is specified, it will attempt to use the default environment. **Arguments**: - `env_name` - Name of the environment to use. **Returns**: Cortex client that can be used to deploy and manage APIs in the specified environment. ## new\_client ```python new_client(env_name: str, operator_endpoint: str) -> Client ``` Create a new environment to connect to an existing cluster, and initialize a client to deploy and manage APIs on that cluster. **Arguments**: - `env_name` - Name of the environment to create. - `operator_endpoint` - The endpoint for the operator of your Cortex cluster. You can get this endpoint by running the CLI command `cortex cluster info`. **Returns**: Cortex client that can be used to deploy and manage APIs on a cluster. ## env\_list ```python env_list() -> List ``` List all environments configured on this machine. ## env\_delete ```python env_delete(name: str) ``` Delete an environment configured on this machine. **Arguments**: - `name` - Name of the environment to delete. # cortex.client.Client ## deploy ```python | deploy(api_spec: Dict[str, Any], force: bool = True, wait: bool = False) ``` Deploy or update an API. **Arguments**: - `api_spec` - A dictionary defining a single Cortex API. See https://docs.cortexlabs.com/v/master/ for schema. - `force` - Override any in-progress api updates. - `wait` - Block until the API is ready. **Returns**: Deployment status, API specification, and endpoint for each API. ## deploy\_from\_file ```python | deploy_from_file(config_file: str, force: bool = False, wait: bool = False) -> Dict ``` Deploy or update APIs specified in a configuration file. **Arguments**: - `config_file` - Local path to a yaml file defining Cortex API(s). See https://docs.cortexlabs.com/v/master/ for schema. - `force` - Override any in-progress api updates. - `wait` - Block until the API is ready. **Returns**: Deployment status, API specification, and endpoint for each API. ## get\_api ```python | get_api(api_name: str) -> Dict ``` Get information about an API. **Arguments**: - `api_name` - Name of the API. **Returns**: Information about the API, including the API specification, endpoint, status, and metrics (if applicable). ## list\_apis ```python | list_apis() -> List ``` List all APIs in the environment. **Returns**: List of APIs, including information such as the API specification, endpoint, status, and metrics (if applicable). ## get\_job ```python | get_job(api_name: str, job_id: str) -> Dict ``` Get information about a submitted job. **Arguments**: - `api_name` - Name of the Batch/Task API. - `job_id` - Job ID. **Returns**: Information about the job, including the job status, worker status, and job progress. ## refresh ```python | refresh(api_name: str, force: bool = False) ``` Restart all of the replicas for a Realtime API without downtime. **Arguments**: - `api_name` - Name of the API to refresh. - `force` - Override an already in-progress API update. ## delete ```python | delete(api_name: str, keep_cache: bool = False) ``` Delete an API. **Arguments**: - `api_name` - Name of the API to delete. - `keep_cache` - Whether to retain the cached data for this API. ## stop\_job ```python | stop_job(api_name: str, job_id: str, keep_cache: bool = False) ``` Stop a running job. **Arguments**: - `api_name` - Name of the Batch/Task API. - `job_id` - ID of the Job to stop. ================================================ FILE: docs/clients/uninstall.md ================================================ # Uninstall ## Uninstall (if installed with pip) ```bash pip uninstall cortex rm -rf ~/.cortex ``` ## Uninstall (if installed without pip) ```bash rm /usr/local/bin/cortex rm -rf ~/.cortex ``` ================================================ FILE: docs/clusters/advanced/kubectl.md ================================================ # Setting up `kubectl` ## Install `kubectl` Follow these [instructions](https://kubernetes.io/docs/tasks/tools/install-kubectl). ## Install the AWS CLI Follow these [instructions](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-install.html). ## Configure the AWS CLI ```bash aws --version # should be >= 1.16 aws configure ``` ## Update `kubeconfig` ```bash aws eks update-kubeconfig --name= --region= ``` ## Test `kubectl` ```bash kubectl get pods ``` ================================================ FILE: docs/clusters/advanced/registry.md ================================================ # Private Docker registry ## Configuring `kubectl` Follow the instructions [here](kubectl.md). ## Setting credentials ```bash DOCKER_USERNAME=*** DOCKER_PASSWORD=*** kubectl create secret docker-registry registry-credentials \ --namespace default \ --docker-username=$DOCKER_USERNAME \ --docker-password=$DOCKER_PASSWORD kubectl patch serviceaccount default --namespace default \ -p "{\"imagePullSecrets\": [{\"name\": \"registry-credentials\"}]}" ``` ## Deleting credentials ```bash kubectl delete secret --namespace default registry-credentials kubectl patch serviceaccount default --namespace default \ -p "{\"imagePullSecrets\": []}" ``` ================================================ FILE: docs/clusters/advanced/self-hosted-images.md ================================================ # Self-hosted Docker images Self-hosting the Cortex cluster's system Docker images can be useful for reducing the ingress costs, for accelerating image pulls, or for eliminating the dependency on Cortex's public container registry. In this guide, we'll use [ECR](https://aws.amazon.com/ecr/) as the destination container registry. When an ECR repository resides in the same region as your Cortex cluster, there are no costs incurred when pulling images. ## Step 1 Make sure you have the [aws](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv1.html), [docker](https://docs.docker.com/get-docker/), and [skopeo](https://github.com/containers/skopeo/blob/master/install.md) utilities installed. ## Step 2 Export the `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` environment variables in your current shell, or run `aws configure`. These credentials must have access to push to ECR. ## Step 3 Clone the Cortex repo using the release tag corresponding to your version (which you can check by running `cortex version`): ```bash export CORTEX_VERSION=0.42.1 git clone --depth 1 --branch v$CORTEX_VERSION https://github.com/cortexlabs/cortex.git ``` ## Step 4 Run the script below to export images to ECR in the same region and account as your cluster. The script will automatically create ECR Repositories with prefix `cortexlabs` if they don't already exist. Feel free to modify the script if you would like to export the images to a different registry such as a private docker hub. ```bash ./cortex/dev/export_images.sh ``` You can now configure Cortex to use your images when creating a cluster (see [here](../management/create.md) for instructions). ## Cleanup You can delete your ECR images from the [AWS ECR dashboard](https://console.aws.amazon.com/ecr/repositories) (set your region in the upper right corner). Make sure all of your Cortex clusters have been deleted before deleting any ECR images. ================================================ FILE: docs/clusters/instances/multi.md ================================================ # Multi-instance type clusters Cortex can be configured to provision different instance types to improve workload performance and reduce cloud infrastructure spend. ## Best practices 1. Spot node groups should have a higher priority than on-demand node groups. 1. CPU node groups should have higher priorities than GPU/Inferentia node groups. 1. Node groups with small instance types should have higher priorities than node groups with large instance types. ## Examples ### CPU spot cluster, with on-demand backup ```yaml # cluster.yaml node_groups: - name: cpu-spot instance_type: m5.large min_instances: 0 max_instances: 5 priority: 100 spot: true spot_config: instance_distribution: [m5a.large, m5d.large, m5n.large, m5ad.large, m5dn.large, m4.large, t3.large, t3a.large, t2.large] - name: cpu-on-demand instance_type: m5.large min_instances: 0 max_instances: 5 ``` ### On-demand cluster supporting CPU, GPU, and Inferentia ```yaml # cluster.yaml node_groups: - name: cpu instance_type: m5.large min_instances: 0 max_instances: 5 priority: 100 - name: gpu instance_type: g4dn.xlarge min_instances: 0 max_instances: 5 - name: inf instance_type: inf.xlarge min_instances: 0 max_instances: 5 ``` ### Spot cluster supporting CPU and GPU (with on-demand backup) ```yaml # cluster.yaml node_groups: - name: cpu-spot instance_type: m5.large min_instances: 0 max_instances: 5 priority: 100 spot: true spot_config: instance_distribution: [m5a.large, m5d.large, m5n.large, m5ad.large, m5dn.large, m4.large, t3.large, t3a.large, t2.large] - name: cpu-on-demand instance_type: m5.large min_instances: 0 max_instances: 5 priority: 50 - name: gpu-spot instance_type: g4dn.xlarge min_instances: 0 max_instances: 5 priority: 20 spot: true - name: gpu-on-demand instance_type: g4dn.xlarge min_instances: 0 max_instances: 5 ``` ### CPU spot cluster with multiple instance types and on-demand backup ```yaml # cluster.yaml node_groups: - name: cpu-1 instance_type: t3.medium min_instances: 0 max_instances: 5 priority: 100 spot: true - name: cpu-2 instance_type: m5.2xlarge min_instances: 0 max_instances: 5 priority: 70 spot: true - name: cpu-3 instance_type: m5.8xlarge min_instances: 0 max_instances: 5 priority: 30 spot: true - name: cpu-4 instance_type: m5.24xlarge min_instances: 0 max_instances: 5 ``` ================================================ FILE: docs/clusters/instances/spot.md ================================================ # Spot instances ```yaml # cluster.yaml node_groups: - name: node-group-1 # whether to use spot instances for this node group (default: false) spot: false # this must be set to true to use spot instances spot_config: # additional instance types with identical or better specs than the primary cluster instance type (defaults to only the primary instance type) instance_distribution: # [similar_instance_type_1, similar_instance_type_2] # minimum number of on demand instances (default: 0) on_demand_base_capacity: 0 # percentage of on demand instances to use after the on demand base capacity has been met [0, 100] (default: 50) # note: setting this to 0 may hinder cluster scale-up when spot instances are not available on_demand_percentage_above_base_capacity: 0 # max price for spot instances (default: the on-demand price of the primary instance type) max_price: # # number of spot instance pools across which to allocate spot instances [1, 20] (default: number of instances in instance distribution) instance_pools: 3 ``` Spot instances are not guaranteed to be available. The chances of getting spot instances can be improved by providing `instance_distribution`, a list of alternative instance types to the primary `instance_type` you specified. If left blank, Cortex will only include the primary instance type in the `instance_distribution`. When using `instance_distribution`, use the instance type with the fewest compute resources as your primary `instance_type`. Note that the default value for `max_price` is the on-demand price of the primary instance type, but you may wish to set this to the on-demand price of the most expensive instance type in your `instance_distribution`. Spot instances can be mixed with on-demand instances in a single node group by configuring `on_demand_base_capacity` and `on_demand_percentage_above_base_capacity`. `on_demand_base_capacity` enforces the minimum number of nodes that will be fulfilled by on-demand instances as your cluster is scaling up. `on_demand_percentage_above_base_capacity` defines the percentage of instances that will be on-demand after the base capacity has been fulfilled (the rest being spot instances). `instance_pools` is the number of pools per availability zone to allocate your instances from. See [here](https://docs.aws.amazon.com/autoscaling/ec2/APIReference/API_InstancesDistribution.html) for more details. Even if multiple instances are specified in your `instance_distribution`, it is still possible that AWS will not be able to provision a spot instance when requested. One possibility is that AWS has exhausted all of the available spot instances of your requested type(s) in your availability zones. Another possibility is that the current price of your requested instance type(s) is higher than your `max_price`. To mitigate this, you may add a second node group to your cluster configuration which is configured to use on-demand instances as a backup. When doing this, it is important to position the on-demand node group after the spot node group in the `node_groups` list (since node groups with lower indices have higher priority). See [here](multi.md) for docs and examples. There is a spot instance limit associated with your AWS account for each instance family in each region. You can check your current limit and request an increase [here](https://console.aws.amazon.com/servicequotas/home?#!/services/ec2/quotas) (set the region in the upper right corner to your desired region, type "spot" in the search bar, and click on the quota that matches your instance type). Note that the quota values indicate the number of vCPUs available, not the number of instances; different instances have a different numbers of vCPUs, which can be seen [here](https://aws.amazon.com/ec2/instance-types/). ## Example spot configuration ### Only spot instances ```yaml node_groups: - name: cpu-spot instance_type: m5.large min_instances: 0 max_instances: 5 spot: true spot_config: instance_distribution: [m5a.large, m5d.large, m5n.large, m5ad.large, m5dn.large, m4.large, t3.large, t3a.large, t2.large] ``` ### 3 on-demand base capacity with 0% on-demand above base capacity ```yaml node_groups: - name: gpu-spot instance_type: g4dn.xlarge min_instances: 0 max_instances: 5 spot: true spot_config: on_demand_base_capacity: 3 on_demand_percentage_above_base_capacity: 0 # instance 1-3: on-demand # instance 4-5: spot ``` ### 0 on-demand base capacity with 50% on-demand above base capacity ```yaml node_groups: - name: gpu-spot instance_type: g4dn.xlarge min_instances: 0 max_instances: 4 spot: true spot_config: on_demand_base_capacity: 0 on_demand_percentage_above_base_capacity: 50 # instance 1: on-demand # instance 2: spot # instance 3: on-demand # instance 4: spot ``` ================================================ FILE: docs/clusters/management/auth.md ================================================ # Auth ## Client The Cortex CLI and Python client use the default credential provider chain to get credentials for cluster and api management. Credentials will be read in the following order of precedence: - environment variables - the name of the profile specified by `AWS_PROFILE` environment variable - `default` profile from `~/.aws/credentials` ### Cluster management It is recommended that your AWS credentials have AdministratorAccess when running `cortex cluster *` commands. If you are unable to use AdministratorAccess, see the [minimum IAM policy](#minimum-iam-policy) below for the minimum permissions required to run `cortex cluster *` commands. After spinning up a cluster using `cortex cluster up`, the IAM user or role that created the cluster is automatically granted `system:masters` permission to the cluster's RBAC. Make sure to keep track of which IAM entity originally created the cluster. #### Running `cortex cluster` commands from different IAM users By default, the `cortex cluster *` commands can only be executed by the IAM user who created the cluster. To grant access to additional IAM users, follow these steps: 1. Install `eksctl` by following these [instructions](https://eksctl.io/introduction/#installation). 1. Determine the ARN of the IAM user that you would like to grant access to. You can get the ARN via the [IAM dashboard](https://console.aws.amazon.com/iam/home#/users), or by running `aws iam get-user` on a machine that is authenticated as the IAM user (or `AWS_ACCESS_KEY_ID=*** AWS_SECRET_ACCESS_KEY=*** aws iam get-user` on any machine, using the credentials of the IAM user). The ARN should look similar to `arn:aws:iam::764403040417:user/my-username`. 1. Set the following environment variables: ```bash CREATOR_AWS_ACCESS_KEY_ID=*** # access key ID for the IAM user that created the cluster CREATOR_AWS_SECRET_ACCESS_KEY=*** # secret access key for the IAM user that created the cluster NEW_USER_ARN=*** # ARN of the IAM user to grant access to CLUSTER_NAME=*** # the name of your cortex cluster (will be "cortex" unless you specified a different name in your cluster configuration file) CLUSTER_REGION=*** # the region of your cortex cluster ``` 1. Run the following command: ```bash AWS_ACCESS_KEY_ID=$CREATOR_AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY=$CREATOR_AWS_SECRET_ACCESS_KEY eksctl create iamidentitymapping --region $CLUSTER_REGION --cluster $CLUSTER_NAME --arn $NEW_USER_ARN --group system:masters --username $NEW_USER_ARN ``` 1. To revoke access in the future, run: ```bash AWS_ACCESS_KEY_ID=$CREATOR_AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY=$CREATOR_AWS_SECRET_ACCESS_KEY eksctl delete iamidentitymapping --region $CLUSTER_REGION --cluster $CLUSTER_NAME --arn $NEW_USER_ARN --all ``` ### API management The Cortex CLI and Python client rely on AWS IAM to authenticate requests to a cluster on AWS (e.g. `cortex deploy`, `cortex get`). AWS credentials required to authenticate Cortex client requests to the operator don't require any specific permissions; they must only be valid credentials within the same AWS account as the Cortex cluster. However, managing the cluster (i.e. running `cortex cluster *` commands) does require permissions. ## Authorizing your APIs When spinning up a cortex cluster, you can provide additional policies to authorize your APIs to access AWS resources by creating a policy and adding it to the `iam_policy_arns` list in your cluster configuration file. If you already have a cluster running and would like to add additional permissions, you can update the policy that is created automatically during `cortex cluster up`. In the [IAM console](https://console.aws.amazon.com/iam/home?policies#/policies), search for `cortex--` to find the policy that has been attached to your cluster. Adding more permissions to this policy will automatically give more access to all of your Cortex APIs. _NOTE: The policy created during `cortex cluster up` will automatically be deleted during `cortex cluster down`. It is recommended to create your own policies that can be specified in `iam_policy_arns` field in cluster configuration. The precreated policy should only be updated for development and testing purposes._ ## Minimum IAM Policy The policy shown below contains the minimum permissions required to manage a Cortex cluster (i.e. via `cortex cluster *` commands). Replace the following placeholders with their respective values in the policy template below: `$CORTEX_CLUSTER_NAME`, `$CORTEX_ACCOUNT_ID`, `$CORTEX_REGION`. ```json { "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Action": "iam:CreateServiceLinkedRole", "Resource": "*", "Condition": { "StringEquals": { "iam:AWSServiceName": [ "autoscaling.amazonaws.com", "ec2scheduled.amazonaws.com", "elasticloadbalancing.amazonaws.com", "spot.amazonaws.com", "spotfleet.amazonaws.com", "transitgateway.amazonaws.com" ] } } }, { "Effect": "Allow", "Action": "iam:CreateServiceLinkedRole", "Resource": "*", "Condition": { "StringEquals": { "iam:AWSServiceName": [ "eks.amazonaws.com", "eks-nodegroup.amazonaws.com", "eks-fargate.amazonaws.com" ] } } }, { "Effect": "Allow", "Action": [ "logs:ListTagsLogGroup", "iam:GetRole", "logs:TagLogGroup", "ssm:GetParameters", "ssm:GetParameter", "logs:CreateLogGroup" ], "Resource": [ "arn:*:ssm:*:$CORTEX_ACCOUNT_ID:parameter/aws/*", "arn:*:ssm:*::parameter/aws/*", "arn:*:logs:$CORTEX_REGION:$CORTEX_ACCOUNT_ID:log-group:$CORTEX_CLUSTER_NAME", "arn:*:iam::$CORTEX_ACCOUNT_ID:role/*" ] }, { "Effect": "Allow", "Action": [ "iam:CreateInstanceProfile", "logs:ListTagsLogGroup", "logs:DescribeLogStreams", "iam:TagRole", "iam:GetPolicy", "iam:CreatePolicy", "iam:DeletePolicy", "iam:ListPolicyVersions", "iam:RemoveRoleFromInstanceProfile", "iam:CreateRole", "iam:AttachRolePolicy", "iam:PutRolePolicy", "iam:AddRoleToInstanceProfile", "iam:ListInstanceProfilesForRole", "iam:PassRole", "logs:CreateLogStream", "iam:DetachRolePolicy", "logs:TagLogGroup", "iam:ListAttachedRolePolicies", "iam:DeleteRolePolicy", "iam:DeleteOpenIDConnectProvider", "iam:TagOpenIDConnectProvider", "iam:DeleteInstanceProfile", "iam:GetRole", "iam:GetInstanceProfile", "iam:DeleteRole", "iam:ListInstanceProfiles", "logs:CreateLogGroup", "logs:PutLogEvents", "logs:DeleteLogGroup", "iam:CreateOpenIDConnectProvider", "iam:GetOpenIDConnectProvider", "iam:GetRolePolicy" ], "Resource": [ "arn:*:iam::$CORTEX_ACCOUNT_ID:instance-profile/eksctl-*", "arn:*:iam::$CORTEX_ACCOUNT_ID:role/eksctl-*", "arn:*:iam::$CORTEX_ACCOUNT_ID:policy/eksctl-*", "arn:*:iam::$CORTEX_ACCOUNT_ID:role/aws-service-role/eks-nodegroup.amazonaws.com/AWSServiceRoleForAmazonEKSNodegroup", "arn:*:iam::$CORTEX_ACCOUNT_ID:role/eksctl-managed-*", "arn:*:iam::$CORTEX_ACCOUNT_ID:oidc-provider/*", "arn:*:logs:$CORTEX_REGION:$CORTEX_ACCOUNT_ID:log-group:$CORTEX_CLUSTER_NAME:*" ] }, { "Effect": "Allow", "Action": [ "iam:CreatePolicy", "iam:GetPolicyVersion", "iam:ListPolicyVersions", "iam:DeletePolicy", "iam:CreatePolicyVersion", "iam:DeletePolicyVersion" ], "Resource": "arn:*:iam::$CORTEX_ACCOUNT_ID:policy/cortex-*" }, { "Effect": "Allow", "Action": [ "sqs:ListQueues", "iam:GetPolicy", "ecr:GetAuthorizationToken", "cloudformation:*", "elasticloadbalancing:*", "autoscaling:*", "cloudwatch:*", "ecr:BatchGetImage", "kms:DescribeKey", "ec2:*", "sts:GetCallerIdentity", "eks:*", "kms:CreateGrant", "acm:DescribeCertificate", "servicequotas:ListServiceQuotas", "logs:PutRetentionPolicy" ], "Resource": "*" }, { "Effect": "Allow", "Action": "sqs:*", "Resource": "arn:*:sqs:$CORTEX_REGION:$CORTEX_ACCOUNT_ID:cx-*" }, { "Effect": "Allow", "Action": "s3:*", "Resource": "arn:*:s3:::$CORTEX_CLUSTER_NAME*" }, { "Effect": "Allow", "Action": "s3:*", "Resource": "arn:*:s3:::$CORTEX_CLUSTER_NAME*/*" } ] } ``` ================================================ FILE: docs/clusters/management/create.md ================================================ # Install ## Prerequisites 1. Install and run [Docker](https://docs.docker.com/install) on your machine. 1. Subscribe to the [AMI with GPU support](https://aws.amazon.com/marketplace/pp/B07GRHFXGM) (for GPU clusters). 1. Create an IAM user with `AdministratorAccess` and programmatic access. 1. You may need to [request limit increases](https://console.aws.amazon.com/servicequotas/home?#!/services/ec2/quotas) for your desired instance types. ## Create a cluster on your AWS account ```bash # install the cortex CLI bash -c "$(curl -sS https://raw.githubusercontent.com/cortexlabs/cortex/v0.42.1/get-cli.sh)" # create a cluster cortex cluster up cluster.yaml ``` ## `cluster.yaml` ```yaml # cluster name cluster_name: cortex # AWS region region: us-east-1 # list of availability zones for your region availability_zones: # default: 3 random availability zones in your region, e.g. [us-east-1a, us-east-1b, us-east-1c] # list of cluster node groups; node_groups: - name: ng-cpu # name of the node group instance_type: m5.large # instance type min_instances: 1 # minimum number of instances max_instances: 5 # maximum number of instances priority: 1 # priority of the node group; the higher the value, the higher the priority [1-100] instance_volume_size: 50 # disk storage size per instance (GB) instance_volume_type: gp3 # instance volume type [gp2 | gp3 | io1 | st1 | sc1] # instance_volume_iops: 3000 # instance volume iops (only applicable to io1/gp3) # instance_volume_throughput: 125 # instance volume throughput (only applicable to gp3) spot: false # whether to use spot instances - name: ng-gpu instance_type: g4dn.xlarge min_instances: 1 max_instances: 5 instance_volume_size: 50 instance_volume_type: gp3 spot: false # ... # subnet visibility for instances [public (instances will have public IPs) | private (instances will not have public IPs)] # when using private subnets, you may wish to enable VPC endpoints (via the AWS console) for S3 and ECR to avoid extra NAT Gateway charges subnet_visibility: public # NAT gateway (required when using private subnets) [none | single | highly_available (a NAT gateway per availability zone)] nat_gateway: none # API load balancer type [nlb | elb] api_load_balancer_type: nlb # API load balancer scheme [internet-facing | internal] api_load_balancer_scheme: internet-facing # operator load balancer scheme [internet-facing | internal] # note: if using "internal", you must configure VPC Peering to connect your CLI to your cluster operator operator_load_balancer_scheme: internet-facing # to install Cortex in an existing VPC, you can provide a list of subnets for your cluster to use # subnet_visibility (specified above in this file) must match your subnets' visibility # this is an advanced feature (not recommended for first-time users) and requires your VPC to be configured correctly; see https://eksctl.io/usage/vpc-networking/#use-existing-vpc-other-custom-configuration # here is an example: # subnets: # - availability_zone: us-west-2a # subnet_id: subnet-060f3961c876872ae # - availability_zone: us-west-2b # subnet_id: subnet-0faed05adf6042ab7 # restrict access to APIs by cidr blocks/ip address ranges api_load_balancer_cidr_white_list: [0.0.0.0/0] # restrict access to the Operator by cidr blocks/ip address ranges operator_load_balancer_cidr_white_list: [0.0.0.0/0] # additional tags to assign to AWS resources (all resources will automatically be tagged with cortex.dev/cluster-name: ) tags: # : map of key/value pairs # SSL certificate ARN (only necessary when using a custom domain) ssl_certificate_arn: # list of IAM policies to attach to your Cortex APIs iam_policy_arns: ["arn:aws:iam::aws:policy/AmazonS3FullAccess"] # primary CIDR block for the cluster's VPC vpc_cidr: 192.168.0.0/16 # instance type for prometheus (use an instance with more memory for clusters exceeding 300 nodes or 300 pods) prometheus_instance_type: "t3.medium" ``` The docker images used by the cluster can also be overridden. They can be configured by adding any of these keys to your cluster configuration file (default values are shown): ```yaml image_manager: quay.io/cortexlabs/manager:master image_operator: quay.io/cortexlabs/operator:master image_controller_manager: quay.io/cortexlabs/controller-manager:master image_autoscaler: quay.io/cortexlabs/autoscaler:master image_proxy: quay.io/cortexlabs/proxy:master image_async_gateway: quay.io/cortexlabs/async-gateway:master image_activator: quay.io/cortexlabs/activator:master image_enqueuer: quay.io/cortexlabs/enqueuer:master image_dequeuer: quay.io/cortexlabs/dequeuer:master image_cluster_autoscaler: quay.io/cortexlabs/cluster-autoscaler:master image_metrics_server: quay.io/cortexlabs/metrics-server:master image_nvidia_device_plugin: quay.io/cortexlabs/nvidia-device-plugin:master image_neuron_device_plugin: quay.io/cortexlabs/neuron-device-plugin:master image_neuron_scheduler: quay.io/cortexlabs/neuron-scheduler:master image_fluent_bit: quay.io/cortexlabs/fluent-bit:master image_istio_proxy: quay.io/cortexlabs/istio-proxy:master image_istio_pilot: quay.io/cortexlabs/istio-pilot:master image_prometheus: quay.io/cortexlabs/prometheus:master image_prometheus_config_reloader: quay.io/cortexlabs/prometheus-config-reloader:master image_prometheus_operator: quay.io/cortexlabs/prometheus-operator:master image_prometheus_statsd_exporter: quay.io/cortexlabs/prometheus-statsd-exporter:master image_prometheus_dcgm_exporter: quay.io/cortexlabs/prometheus-dcgm-exporter:master image_prometheus_kube_state_metrics: quay.io/cortexlabs/prometheus-kube-state-metrics:master image_prometheus_node_exporter: quay.io/cortexlabs/prometheus-node-exporter:master image_kube_rbac_proxy: quay.io/cortexlabs/kube-rbac-proxy:master image_grafana: quay.io/cortexlabs/grafana:master image_event_exporter: quay.io/cortexlabs/event-exporter:master image_kubexit: quay.io/cortexlabs/kubexit:master ``` ================================================ FILE: docs/clusters/management/delete.md ================================================ # Uninstall ```bash cortex cluster down ``` ## Bucket Contents When a Cortex cluster is created, an S3 bucket is created for its internal use. When running `cortex cluster down`, a lifecycle rule is applied to the bucket such that its entire contents are removed within the next 24 hours. You can safely delete the bucket at any time after `cortex cluster down` has finished running. ## Delete SSL Certificate If you've set up HTTPS, you can remove the SSL Certificate by following these [instructions](../networking/https.md#cleanup). ## Delete Hosted Zone If you've configured a custom domain for your APIs, follow these [instructions](../networking/custom-domain.md#cleanup) to delete the Hosted Zone. ## Keep Cortex Resources The contents of Cortex's S3 bucket, the EBS volumes (used by Cortex's Prometheus and Grafana instances), and the log group are deleted by default when running `cortex cluster down`. If you want to keep these resources, you can pass the `--keep-aws-resources` flag to the `cortex cluster down` command. ## Troubleshooting On rare occasions, `cortex cluster down` may not be able to spin down your Cortex cluster. When this happens, follow these steps: 1. If you've manually created any AWS networking resources that are pointed to the cluster or its VPC (e.g. API Gateway VPC links, custom domains, etc), delete them from the AWS console. 1. Replace "" and "" in the following URL, and open it in your browser: `https://console.aws.amazon.com/cloudformation/home?region=#/stacks?filteringText=eksctl--` ![image](https://user-images.githubusercontent.com/808475/97790394-963b4880-1b85-11eb-8e27-ba5a551606b3.png) 1. For each CloudFormation stack which contains the word "nodegroup", select the stack and click "Delete". 1. Select the final stack (the one that ends in "-cluster") and click "Delete". If deleting the stack fails, navigate to the EC2 dashboard in the AWS console, delete the load balancers that are associated with the cluster, and try again (you can determine which load balancers are associated with the cluster by setting the correct region in the console and checking the `cortex.dev/cluster-name` tag on all load balancers). If the problem still persists, delete any other AWS resources that are blocking the stack deletion and try again. 1. In rare cases, you may need to delete other AWS resources associated with your Cortex cluster. For each the following resources, go to the appropriate AWS Dashboard (in the region that your cluster was in), and confirm that there are no resources left behind by the cluster: CloudWatch Dashboard, SQS Queues, S3 Bucket, and CloudWatch LogGroups (the Cortex bucket and log groups are not deleted by `cluster down` in order to preserve your data). ================================================ FILE: docs/clusters/management/environments.md ================================================ # Environments When you create a cluster with `cortex cluster up`, an environment with the same name as your cluster is automatically created to point to your cluster and is configured to be the default environment. You can name the environment something else via the `--configure-env` flag, e.g. `cortex cluster up --configure-env prod`. You can also use the `--configure-env` flag with `cortex cluster info` to create / update the specified environment. You can list your environments with `cortex env list`, change the default environment with `cortex env default`, rename an environment with `cortex env rename`, delete an environment with `cortex env delete`, and create/update an environment with `cortex env configure`. ## Multiple clusters ```bash cortex cluster up cluster1.yaml --configure-env cluster1 # configures the cluster1 env cortex cluster up cluster2.yaml --configure-env cluster2 # configures the cluster2 env cortex deploy --env cluster1 cortex delete my-api --env cluster1 cortex deploy --env cluster2 cortex delete my-api --env cluster2 ``` ## Multiple clusters, if you omitted the `--configure-env` on `cortex cluster up` ```bash cortex cluster info cluster1.yaml --configure-env cluster1 # configures the cluster1 env cortex cluster info cluster2.yaml --configure-env cluster2 # configures the cluster2 env cortex deploy --env cluster1 cortex delete my-api --env cluster1 cortex deploy --env cluster2 cortex delete my-api --env cluster2 ``` ## Configure `cortex` CLI to connect to an existing cluster If you are installing the `cortex` CLI on a new machine, you can configure it to access an existing Cortex cluster. On the machine which already has the CLI configured, run: ```bash cortex env list ``` Take note of the environment name and operator endpoint of the desired environment. On your new machine, run: ```bash cortex env configure ``` ================================================ FILE: docs/clusters/management/production.md ================================================ # Production guide As you take Cortex from development to production, here are a few pointers that might be useful. ## Use images from a colocated ECR Configure your cluster and APIs to use images from ECR in the same region as your cluster to accelerate scale-ups, reduce ingress costs, and remove the dependency on Cortex's public quay.io registry. You can find instructions for mirroring Cortex images [here](../advanced/self-hosted-images.md) ## Handling Cortex updates/upgrades Use a Route 53 hosted zone as a proxy in front of your Cortex cluster. Every new Cortex cluster provisions a new API load balancer with a unique endpoint. Using a Route 53 hosted zone configured with a subdomain will expose your Cortex cluster API endpoint as a static endpoint (e.g. `cortex.your-company.com`). You will be able to upgrade Cortex versions without downtime, and you will avoid the need to updated your client code every time you migrate to a new cluster. You can find instructions for setting up a custom domain with a Route 53 hosted zone [here](../networking/custom-domain.md), and instructions for updating/upgrading your cluster [here](update.md). ## Production cluster configuration ### Securing your cluster The following configuration will improve security by preventing your cluster's nodes from being publicly accessible. ```yaml subnet_visibility: private nat_gateway: single # use "highly_available" for large clusters making requests to services outside of the cluster ``` You can make your load balancer private to prevent your APIs from being publicly accessed. In order to access your APIs, you will need to set up VPC peering between the Cortex cluster's VPC and the VPC containing the consumers of the Cortex APIs. See the [VPC peering guide](../networking/vpc-peering.md) for more details. ```yaml api_load_balancer_scheme: internal ``` You can also restrict access to your load balancers by IP address: ```yaml api_load_balancer_cidr_white_list: [0.0.0.0/0] ``` These two fields are also available for the operator load balancer. Keep in mind that if you make the operator load balancer private, you'll need to configure VPC peering to use the `cortex` CLI or Python client. ```yaml operator_load_balancer_scheme: internal operator_load_balancer_cidr_white_list: [0.0.0.0/0] ``` See [here](../networking/load-balancers.md) for more information about the load balancers. ### Workload load-balancing Depending on your application's requirements, you might have different needs from the cluster's api load balancer. By default, the api load balancer is a [Network load balancer](https://docs.aws.amazon.com/elasticloadbalancing/latest/network/introduction.html) (NLB). In some situations, a [Classic load balancer](https://docs.aws.amazon.com/elasticloadbalancing/latest/classic/introduction.html) (ELB) may be preferred, and can be selected in your cluster config by setting `api_load_balancer_type: elb`. This selection can only be made before creating your cluster. ### Ensure node provisioning You can take advantage of the cost savings of spot instances and the reliability of on-demand instances by utilizing the `priority` field in node groups. You can deploy two node groups, one that is spot and another that is on-demand. Set the priority of the spot node group to be higher than the priority of the on-demand node group. This encourages the cluster-autoscaler to try to spin up instances from the spot node group first. If there are no more spot instances available, the on-demand node group will be used instead. ```yaml node_groups: - name: gpu-spot instance_type: g4dn.xlarge min_instances: 0 max_instances: 5 spot: true priority: 100 - name: gpu-on-demand instance_type: g4dn.xlarge min_instances: 0 max_instances: 5 priority: 1 ``` ### Considerations for large clusters If you plan on scaling your Cortex cluster past 300 nodes or 300 pods, it is recommended to set `prometheus_instance_type` to an instance type with more memory (the default is `t3.medium`, which has 4gb). ## API Spec ### Container design Configure your health checks to be as accurate as possible to prevent requests from being routed to pods that aren't ready to handle traffic. ### Pods section Make sure that `max_concurrency` is set to match the concurrency supported by your container. Tune `max_queue_length` to lower values if you would like to more aggressively redistribute requests to newer pods as your API scales up rather than allowing requests to linger in queues. This would mean that the clients consuming your APIs should implement retry logic with a delay (such as exponential backoff). ### Compute section Make sure to specify all of the relevant compute resources (especially cpu and memory) to ensure that your pods aren't starved for resources. ### Autoscaling Revisit the autoscaling docs for [Realtime APIs](../../workloads/realtime/autoscaling.md) and/or [Async APIs](../../workloads/async/autoscaling.md) to effectively handle production traffic by tuning the scaling rate, sensitivity, and over-provisioning. ================================================ FILE: docs/clusters/management/update.md ================================================ # Update ## Modify existing cluster You can add or remove node groups, resize existing node groups, and update some configuration fields of a running cluster. Fetch the current cluster configuration: ```bash cortex cluster info --print-config --name CLUSTER_NAME --region REGION > cluster.yaml ``` Make your desired changes, and then apply them: ```bash cortex cluster configure cluster.yaml ``` Cortex will calculate the difference and you will be prompted with the update plan. If you would like to update fields that cannot be modified on a running cluster, you must create a new cluster with your desired configuration. ## Upgrade to a new version Updating an existing Cortex cluster is not supported at the moment. Please spin down the previous version of the cluster, install the latest version of the Cortex CLI, and use it to spin up a new Cortex cluster. See the next section for how to do this without downtime. ## Update or upgrade without downtime It is possible to update to a new version Cortex or to migrate from one cluster to another without downtime. Note: it is important to not spin down your previous cluster until after your new cluster is receiving traffic. ### Set up a subdomain using a Route 53 hosted zone If you've already set up a subdomain with a Route 53 hosted zone pointing to your cluster, skip this step. Setting up a Route 53 hosted zone allows you to transfer traffic seamlessly from from an existing cluster to a new cluster, thereby avoiding downtime. You can find the instructions for setting up a subdomain [here](../networking/custom-domain.md). You will need to update any clients interacting with your Cortex APIs to point to the new subdomain. ### Export all APIs from your previous cluster The `cluster export` command can be used to get the YAML specifications of all APIs deployed in your cluster: ```bash cortex cluster export --name --region ``` ### Spin up a new cortex cluster If you are creating a new cluster with the same Cortex version: ```bash cortex cluster up new-cluster.yaml --configure-env cortex2 ``` This will create a CLI environment named `cortex2` for accessing the new cluster. If you are spinning a up a new cluster with a different Cortex version, first install the cortex CLI matching the desired cluster version: ```bash # download the desired CLI version, replace 0.42.1 with the desired version (Note the "v"): bash -c "$(curl -sS https://raw.githubusercontent.com/cortexlabs/cortex/v0.42.1/get-cli.sh)" # confirm Cortex CLI version cortex version # spin up your cluster using the new CLI version cortex cluster up cluster.yaml --configure-env cortex2 ``` You can use different Cortex CLIs to interact with the different versioned clusters; here is an example: ```bash # download the desired CLI version, replace 0.42.1 with the desired version (Note the "v"): CORTEX_INSTALL_PATH=$(pwd)/cortex0.42.1 bash -c "$(curl -sS https://raw.githubusercontent.com/cortexlabs/cortex/v0.42.1/get-cli.sh)" # confirm cortex CLI version ./cortex0.42.1 version ``` ### Deploy the APIs to your new cluster Please read the [changelogs](https://github.com/cortexlabs/cortex/releases) and the latest documentation to identify any features and breaking changes in the new version. You may need to make modifications to your cluster and/or API configuration files. ```bash cortex deploy -e cortex2 ``` After you've updated the API specifications and images if necessary, you can deploy them onto your new cluster. ### Point your custom domain to your new cluster Verify that all of the APIs in your new cluster are working as expected by accessing via the cluster's API load balancer URL. Get the cluster's API load balancer URL: ```bash cortex cluster info --name --region ``` Once the APIs on the new cluster have been verified as working properly, it is recommended to update `min_replicas` of your APIs on the new cluster to match the current values in your previous cluster. This will avoid large sudden scale-up events as traffic is shifted to the new cluster. Then, navigate to the A record in your custom domains's Route 53 hosted zone and update the Alias to point the new cluster's API load balancer URL. Rather than suddenly routing all of your traffic from the previous cluster to the new cluster, you can use [weighted records](https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/routing-policy.html#routing-policy-weighted) to incrementally route more traffic to your new cluster. If you increased `min_replicas` for your APIs in the new cluster during the transition, you may reduce `min_replicas` back to your desired level once all traffic has been shifted. ### Spin down the previous cluster After confirming that your previous cluster has completed servicing all existing traffic and is not receiving any new traffic, spin down your previous cluster: ```bash # Note: it is recommended to install the Cortex CLI matching the previous cluster's version to ensure proper deletion. cortex cluster down --name --region ``` ================================================ FILE: docs/clusters/networking/api-gateway.md ================================================ # API Gateway This guide shows how set up AWS API Gateway for your Cortex APIs, which is the simplest way to enable HTTPS if a custom domain is not required. See [here](https.md) for how to set up a custom domain with SSL certificates instead. Please note that one limitation of API Gateway is that there is a 30-second time limit for all requests. If your API load balancer is internet-facing (which is the default, or you set `api_load_balancer_scheme: internet-facing` in your cluster configuration file before creating your cluster), use the [first section](#internet-facing-load-balancer) of this guide. If your API load balancer is internal (i.e. you set `api_load_balancer_scheme: internal` in your cluster configuration file before creating your cluster), use the [second section](#internal-load-balancer) of this guide. ## Internet-facing load balancer _This section applies if your API load balancer is internet-facing (which is the default, or you set `api_load_balancer_scheme: internet-facing` in your cluster configuration file before creating your cluster). If your API load balancer is internal, see the [internal load balancer](#internal-load-balancer) section below._ There are two types of API Gateways that can be used when your load balancer is internet-facing: HTTP and REST. HTTP APIs are faster and less expensive, while REST APIs have more features an allow for more control. See the following section for creating an [HTTP API](#http-api), or skip to the next section for creating a [REST API](#rest-api). ### HTTP API #### Create an API Gateway Go to the [API Gateway console](https://console.aws.amazon.com/apigateway/home), select "HTTP API" under "Choose an API type", and click "Build". ![](https://user-images.githubusercontent.com/808475/125668597-35ad8d8e-8b5f-4274-bbb3-62ff47e5d544.png) Click "Add integration". ![](https://user-images.githubusercontent.com/808475/125668635-f92df672-f516-45e0-a152-538aff901c0d.png) Click the drop-down menu, and select "HTTP". ![](https://user-images.githubusercontent.com/808475/125668655-78754a51-c77a-4a03-a548-78ab991d1486.png) Set the "URL endpoint" to `http://API_LOAD_BALANCER_ENDPOINT/{proxy}`. You can get your API load balancer endpoint via `cortex cluster info`; make sure to prepend `http://` and append `/{proxy}`. For example, mine is: `http://aa9f5fdabfa6446aca53a526f59bc3c5-18cd00a628421fa3.elb.us-east-1.amazonaws.com/{proxy}`. Choose a name for your API (e.g. "cortex"), and click "Next". ![](https://user-images.githubusercontent.com/808475/125668735-b57976c7-f68e-4a28-ab2a-92cfad5b49bb.png) On the next page, update the pre-populated route's resource path to `/{proxy+}`, and click "Next". ![](https://user-images.githubusercontent.com/808475/125668752-21aef8dd-1e75-41a5-bfc5-1e7157efbe01.png) Click "Next" again. ![](https://user-images.githubusercontent.com/808475/125668761-8a177f6e-1977-48a2-be84-4b0df0105283.png) Click "Create". ![](https://user-images.githubusercontent.com/808475/125668770-69fd3fbe-24e4-4a8a-8413-d50c779392c5.png) Copy your "Invoke URL" for the `$default` stage ![](https://user-images.githubusercontent.com/808475/125668795-b18b564c-7091-432c-a4fd-a5a8bb157946.png) #### Use your new endpoint You may now use the "Invoke URL" in place of your API load balancer endpoint in your client. For example, this curl request: ```bash curl -X POST http://aa9f5fdabfa6446aca53a526f59bc3c5-18cd00a628421fa3.elb.us-east-1.amazonaws.com/hello-world ``` Would become: ```bash curl -X POST https://nj3f5l96oe.execute-api.us-east-1.amazonaws.com/hello-world ``` #### Cleanup Delete the API Gateway before spinning down your Cortex cluster: ![](https://user-images.githubusercontent.com/808475/125668816-83bce6bf-cf0e-4835-8d18-d641eb6cdb29.png) ### REST API #### Create an API Gateway Go to the [API Gateway console](https://console.aws.amazon.com/apigateway/home), select "REST API" under "Choose an API type", and click "Build". ![](https://user-images.githubusercontent.com/808475/78293216-18269e80-74dd-11ea-9e68-86922c2cbc7c.png) Select "REST" and "New API", name your API (e.g. "cortex"), select either "Regional" or "Edge optimized" (depending on your preference), and click "Create API". ![](https://user-images.githubusercontent.com/808475/78293434-66d43880-74dd-11ea-92d6-692158171a3f.png) Select "Actions" > "Create Resource": ![](https://user-images.githubusercontent.com/808475/80154502-8b6b7f80-8574-11ea-9c78-7d9f277bf55b.png) Select "Configure as proxy resource" and "Enable API Gateway CORS", and click "Create Resource" ![](https://user-images.githubusercontent.com/808475/80154565-ad650200-8574-11ea-8753-808cd35902e2.png) Select "HTTP Proxy" and set "Endpoint URL" to `http://API_LOAD_BALANCER_ENDPOINT/{proxy}`. You can get your API load balancer endpoint via `cortex cluster info`; make sure to prepend `http://` and append `/{proxy}`. For example, mine is: `http://a9eaf69fd125947abb1065f62de59047-81cdebc0275f7d96.elb.us-west-2.amazonaws.com/{proxy}`. Leave "Content Handling" set to "Passthrough" and Click "Save". ![](https://user-images.githubusercontent.com/808475/80154735-13ea2000-8575-11ea-83ca-58f182df83c6.png) Select "Actions" > "Deploy API" ![](https://user-images.githubusercontent.com/808475/80154802-2c5a3a80-8575-11ea-9ab3-de89885fd658.png) Create a new stage (e.g. "dev") and click "Deploy" ![](https://user-images.githubusercontent.com/808475/80154859-4431be80-8575-11ea-9305-50384b1f9847.png) Copy your "Invoke URL" ![](https://user-images.githubusercontent.com/808475/80154911-5dd30600-8575-11ea-9682-1a7328783011.png) #### Use your new endpoint You may now use the "Invoke URL" in place of your API load balancer endpoint in your client. For example, this curl request: ```bash curl -X POST http://a9eaf69fd125947abb1065f62de59047-81cdebc0275f7d96.elb.us-west-2.amazonaws.com/hello-world ``` Would become: ```bash curl -X POST https://31qjv48rs6.execute-api.us-west-2.amazonaws.com/dev/hello-world ``` #### Cleanup Delete the API Gateway before spinning down your Cortex cluster: ![](https://user-images.githubusercontent.com/808475/80155073-bdc9ac80-8575-11ea-99a1-95c0579da79e.png) ## Internal load balancer _This section applies if your API load balancer is internal (i.e. you set `api_load_balancer_scheme: internal` in your cluster configuration file before creating your cluster). If your API load balancer is internet-facing, see the [internet-facing load balancer](#internet-facing-load-balancer) section above._ ### Create a VPC Link Navigate to AWS's EC2 Load Balancer dashboard and locate the Cortex API load balancer. You can determine which is the API load balancer by inspecting the `kubernetes.io/service-name` tag: ![](https://user-images.githubusercontent.com/808475/80142777-961c1980-8560-11ea-9202-40964dbff5e9.png) Take note of the load balancer's name. Go to the [API Gateway console](https://console.aws.amazon.com/apigateway/home), click "VPC Links" on the left sidebar, and click "Create" ![](https://user-images.githubusercontent.com/808475/80142466-0c6c4c00-8560-11ea-8293-eb5e5572b797.png) Select "VPC link for REST APIs", name your VPC link (e.g. "cortex"), select the API load balancer, and click "Create". ![](https://user-images.githubusercontent.com/808475/80143027-03c84580-8561-11ea-92de-9ed0a5dfa593.png) Wait for the VPC link to be created (it will take a few minutes) ![](https://user-images.githubusercontent.com/808475/80144088-bbaa2280-8562-11ea-901b-8520eb253df7.png) ### Create an API Gateway Go to the [API Gateway console](https://console.aws.amazon.com/apigateway/home), select "REST API" under "Choose an API type", and click "Build" ![](https://user-images.githubusercontent.com/808475/78293216-18269e80-74dd-11ea-9e68-86922c2cbc7c.png) Select "REST" and "New API", name your API (e.g. "cortex"), select either "Regional" or "Edge optimized" (depending on your preference), and click "Create API" ![](https://user-images.githubusercontent.com/808475/78293434-66d43880-74dd-11ea-92d6-692158171a3f.png) Select "Actions" > "Create Resource" ![](https://user-images.githubusercontent.com/808475/80141938-3cffb600-855f-11ea-9c1c-132ca4503b7a.png) Select "Configure as proxy resource" and "Enable API Gateway CORS", and click "Create Resource" ![](https://user-images.githubusercontent.com/808475/80142124-80f2bb00-855f-11ea-8e4e-9413146e0815.png) Select "VPC Link", select "Use Proxy Integration", choose your newly-created VPC Link, and set "Endpoint URL" to `http://API_LOAD_BALANCER_ENDPOINT/{proxy}`. You can get your API load balancer endpoint via `cortex cluster info`; make sure to prepend `http://` and append `/{proxy}`. For example, mine is: `http://a5044e34a352d44b0945adcd455c7fa3-32fa161d3e5bcbf9.elb.us-west-2.amazonaws.com/{proxy}`. Click "Save" ![](https://user-images.githubusercontent.com/808475/80147407-4f322200-8568-11ea-8ef5-df5164c1375f.png) Select "Actions" > "Deploy API" ![](https://user-images.githubusercontent.com/808475/80147555-86083800-8568-11ea-86af-1b1e38c9d322.png) Create a new stage (e.g. "dev") and click "Deploy" ![](https://user-images.githubusercontent.com/808475/80147631-a7692400-8568-11ea-8a09-13dbd50b17b9.png) Copy your "Invoke URL" ![](https://user-images.githubusercontent.com/808475/80147716-c798e300-8568-11ea-9aef-7dd6fdf4a68a.png) ### Use your new endpoint You may now use the "Invoke URL" in place of your API load balancer endpoint in your client. For example, this curl request: ```bash curl -X POST http://a5044e34a352d44b0945adcd455c7fa3-32fa161d3e5bcbf9.elb.us-west-2.amazonaws.com/hello-world ``` Would become: ```bash curl -X POST https://lrivodooqh.execute-api.us-west-2.amazonaws.com/dev/hello-world ``` ### Cleanup Delete the API Gateway and VPC Link before spinning down your Cortex cluster: ![](https://user-images.githubusercontent.com/808475/80149163-05970680-856b-11ea-9f82-61f4061a3321.png) ![](https://user-images.githubusercontent.com/808475/80149204-1ba4c700-856b-11ea-83f7-9741c78b6b95.png) ================================================ FILE: docs/clusters/networking/custom-domain.md ================================================ # Custom domain You can set up DNS to use a custom domain for your Cortex APIs. For example, you can make your API accessible via `api.example.com/hello-world`. This guide will demonstrate how to create a dedicated subdomain in AWS Route 53. After completing this guide, if you want to enable HTTPS with your custom subdomain, see [these instructions](https.md). ## Configure DNS Decide on a subdomain that you want to dedicate to Cortex APIs. For example if your domain is `example.com`, a valid subdomain can be `api.example.com`. This guide will use `cortexlabs.dev` as the domain and `api.cortexlabs.dev` as the subdomain. We will set up a hosted zone on Route 53 to manage the DNS records for the subdomain. Go to the [Route 53 console](https://console.aws.amazon.com/route53/home) and click "Hosted Zones". ![](https://user-images.githubusercontent.com/4365343/82210754-a6b07d00-98dd-11ea-9cec-9f6b07282aa8.png) Click "Create Hosted Zone" and then enter your subdomain as the domain name for your hosted zone and click "Create". ![](https://user-images.githubusercontent.com/4365343/82211091-4968fb80-98de-11ea-8ec4-8d26d1aea77a.png) Take note of the values in the NS record. ![](https://user-images.githubusercontent.com/4365343/82211656-386cba00-98df-11ea-8c86-4961082b5f49.png) Navigate to your root DNS service provider (e.g. Google Domains, AWS Route 53, Go Daddy). Your root DNS service provider is typically the registrar where you purchased your domain (unless you have transferred DNS management elsewhere). The procedure for adding DNS records may vary based on your service provider. We are going to add an NS (name server) record that specifies that any traffic to your subdomain should use the name servers of your hosted zone in Route 53 for DNS resolution. `cortexlabs.dev` is managed by Google Domains. The image below is a screenshot for adding a DNS record in Google Domains (your UI may differ based on your DNS service provider). ![](https://user-images.githubusercontent.com/808475/109039458-abcb0580-7681-11eb-8644-76436328687e.png) ## Add DNS record Navigate to your [EC2 Load Balancer console](https://us-west-2.console.aws.amazon.com/ec2/v2/home#LoadBalancers:sort=loadBalancerName) and locate the Cortex API load balancer. You can determine which is the API load balancer by inspecting the `kubernetes.io/service-name` tag. Take note of the load balancer's name. ![](https://user-images.githubusercontent.com/808475/80142777-961c1980-8560-11ea-9202-40964dbff5e9.png) Go back to the [Route 53 console](https://console.aws.amazon.com/route53/home#hosted-zones:) and select the hosted zone you created earlier. Click "Create Record Set", and add an Alias record that routes traffic to your Cortex cluster's API load balancer (leave "Name" blank). ![](https://user-images.githubusercontent.com/808475/84083422-6ac97e80-a996-11ea-9679-be37268a2133.png) ## Debugging connectivity issues You could run into connectivity issues if you make a request to your API without waiting long enough for your DNS records to propagate after creating them (it usually takes 5-10 minutes). If you are updating existing DNS records, it could take anywhere from a few minutes to 48 hours for the DNS cache to expire (until then, your previous DNS configuration will be used). To test connectivity, try the following steps: 1. Deploy an api. 1. Make a request to your api (e.g. `curl http://api.cortexlabs.dev/hello-world` or paste the url into your browser if your API supports GET requests). 1. If you run into an error such as `curl: (6) Could not resolve host: api.cortexlabs.dev` wait a few minutes and make the request from another device that hasn't made a request to that url in a while. ## Cleanup Spin down your Cortex cluster. Delete the hosted zone for your subdomain in the [Route 53 console](https://console.aws.amazon.com/route53/home#hosted-zones:): ![](https://user-images.githubusercontent.com/4365343/82228729-81306d00-98f7-11ea-8570-e9de15f5267f.png) ================================================ FILE: docs/clusters/networking/https.md ================================================ # Setting up HTTPS This guide shows how to support HTTPS traffic to Cortex APIs via a custom domain. It is also possible to use AWS API Gateway to enable HTTPS without using your own domain (see [here](api-gateway.md) for instructions). In order to create a valid SSL certificate for your domain, you must have the ability to configure DNS to satisfy the DNS challenges which prove that you own the domain. This guide assumes that you are using a Route 53 hosted zone to manage a subdomain. Follow this [guide](./custom-domain.md) to set up a subdomain managed by a Route 53 hosted zone. ## Generate an SSL certificate To create an SSL certificate, go to the [ACM console](https://us-west-2.console.aws.amazon.com/acm/home) and click "Get Started" under the "Provision certificates" section. ![](https://user-images.githubusercontent.com/4365343/82202340-c04ac800-98cf-11ea-9472-89dd6d67eb0d.png) Select "Request a public certificate" and then "Request a certificate". ![](https://user-images.githubusercontent.com/4365343/82202654-3e0ed380-98d0-11ea-8c57-025f0b69c54f.png) Enter your subdomain and then click "Next". ![](https://user-images.githubusercontent.com/4365343/82224652-1cbedf00-98f2-11ea-912b-466cee2f6e25.png) Select "DNS validation" and then click "Next". ![](https://user-images.githubusercontent.com/4365343/82205311-66003600-98d4-11ea-90e3-da7e8b0b2b9c.png) Add tags for searchability (optional) then click "Review". ![](https://user-images.githubusercontent.com/4365343/82206485-52ee6580-98d6-11ea-95a9-1d0ebafc178a.png) Click "Confirm and request". ![](https://user-images.githubusercontent.com/4365343/82206602-84ffc780-98d6-11ea-9f2f-ce383404ec67.png) Click "Create record in Route 53". A popup will appear indicating that a Record is going to be added to Route 53. Click "Create" to automatically add the DNS record to your subdomain's hosted zone. Then click "Continue". ![](https://user-images.githubusercontent.com/4365343/82223539-c8ffc600-98f0-11ea-93a2-044aa0c9670d.png) Wait for the Certificate Status to be "issued". This might take a few minutes. ![](https://user-images.githubusercontent.com/4365343/82209663-a616e700-98db-11ea-95cb-c6efedadb942.png) Take note of the certificate's ARN. The certificate is ineligible for renewal because it is currently not being used. It will be eligible for renewal once it's used in Cortex. ![](https://user-images.githubusercontent.com/4365343/82222684-9e613d80-98ef-11ea-98c0-5a20b457f062.png) ## Create or update your cluster Add the following field to your cluster configuration: ```yaml # cluster.yaml ... ssl_certificate_arn: ``` Create a cluster: ```bash cortex cluster up cluster.yaml ``` Or update an existing cluster: ```bash cortex cluster configure cluster.yaml ``` ## Use your new endpoint Wait a few minutes to allow the DNS changes to propagate. You may now use your subdomain in place of your API load balancer endpoint in your client. For example, this curl request: ```bash curl http://a5044e34a352d44b0945adcd455c7fa3-32fa161d3e5bcbf9.elb.us-west-2.amazonaws.com/hello-world -X POST -H "Content-Type: application/json" -d @sample.json ``` Would become: ```bash # add the `-k` flag or use http:// instead of https:// if you didn't configure an SSL certificate curl https://api.cortexlabs.dev/hello-world -X POST -H "Content-Type: application/json" -d @sample.json ``` ## Cleanup Spin down your Cortex cluster. If you created an SSL certificate, delete it from the [ACM console](https://us-west-2.console.aws.amazon.com/acm/home): ![](https://user-images.githubusercontent.com/4365343/82228835-a624e000-98f7-11ea-92e2-cb4fb0f591e2.png) ================================================ FILE: docs/clusters/networking/load-balancers.md ================================================ # Load balancers ![api architecture diagram](https://user-images.githubusercontent.com/808475/103417256-dd6e9700-4b3e-11eb-901e-90425f1f8fd4.png) All APIs share a single API load balancer. By default, the API load balancer is public. You can configure your API load balancer to be private by setting `api_load_balancer_scheme: internal` in your cluster configuration file (before creating your cluster). This will make your API only accessible through [VPC Peering](vpc-peering.md). You can enforce that incoming requests to APIs must originate from specific ip address ranges by specifying `api_load_balancer_cidr_white_list: []` in your cluster configuration. The SSL certificate on the API load balancer is autogenerated during installation using `localhost` as the Common Name (CN). Therefore, clients will need to skip certificate verification when making HTTPS requests to your APIs (e.g. `curl -k https://***`), or make HTTP requests instead (e.g. `curl http://***`). Alternatively, you can enable HTTPS by using a [custom domain](custom-domain.md) and setting up [https](https.md) or by [creating an API Gateway](api-gateway.md) to forward requests to your API load balancer. There is a separate load balancer for the Cortex operator. By default, the operator load balancer is public. You can configure your operator load balancer to be private by setting `operator_load_balancer_scheme: internal` in your cluster configuration file (before creating your cluster). You can use [VPC Peering](vpc-peering.md) to enable your Cortex CLI to connect to your cluster operator from another VPC. You can enforce that incoming requests to the Cortex operator must originate from specific ip address ranges by specifying `operator_load_balancer_cidr_white_list: []` in your cluster configuration. By default, the API load balancer and Operator load balancer are both [Network load balancers](https://docs.aws.amazon.com/elasticloadbalancing/latest/network/introduction.html) (NLB). The api load balancer can be configured as a [Classic load balancer](https://docs.aws.amazon.com/elasticloadbalancing/latest/classic/introduction.html) (ELB) instead if desired. The API load balancer type must be specified before creating your cluster. ================================================ FILE: docs/clusters/networking/vpc-peering.md ================================================ # VPC peering If you are using an internal operator load balancer (i.e. you set `operator_load_balancer_scheme: internal` in your cluster configuration file before creating your cluster), you can use VPC Peering to enable your Cortex CLI to connect to your cluster operator from another VPC so that you may run `cortex` commands. If you are using an internal API load balancer (i.e. you set `api_load_balancer_scheme: internal` in your cluster configuration file before creating your cluster), you can use VPC Peering to make requests from another VPC. This guide illustrates how to create a VPC Peering connection between a VPC of your choice and the Cortex load balancers. ## Gather Cortex's VPC information Navigate to AWS's EC2 Load Balancer dashboard and locate the Cortex operator's load balancer. You can determine which is the operator load balancer by inspecting the `kubernetes.io/service-name` tag: ![](https://user-images.githubusercontent.com/808475/80126132-804e2a80-8547-11ea-8ce4-57d3fd96e2c4.png) Click back to the "Description" tab and note the VPC ID of the load balancer and the ID of each of the subnets associated with the load balancer: ![](https://user-images.githubusercontent.com/808475/80127144-c2c43700-8548-11ea-95b4-ce9d1df024cc.png) Navigate to AWS's VPC dashboard and identify the ID and CIDR block of Cortex's VPC: ![](https://user-images.githubusercontent.com/808475/80125554-af17d100-8546-11ea-96ec-00e2aaee7100.png) The VPC ID here should match that of the load balancer. ## Create peering connection Identify the ID and CIDR block of the VPC from which you'd like to connect to the Cortex VPC. In my case, I have a VPC in the same AWS account and region, and I can locate its ID and CIDR block from AWS's VPC dashboard: ![](https://user-images.githubusercontent.com/808475/80125729-eb4b3180-8546-11ea-8d20-6bc2478747ae.png) From AWS's VPC dashboard, navigate to the "Peering Connections" page, and click "Create Peering Connection": ![](https://user-images.githubusercontent.com/808475/80127600-67df0f80-8549-11ea-9e10-765a6e273b54.png) Name your new VPC Peering Connection (I used "cortex-operator", but "cortex" or "cortex-api" may make more sense depending on your use case). Then configure the connection such that the "Requester" is the VPC from which you'll connect to the Cortex VPC, and the "Accepter" is Cortex's VPC. ![](https://user-images.githubusercontent.com/808475/80131545-3f5a1400-854f-11ea-9ca0-c51433d3fa3d.png) Click "Create Peering Connection", navigate back to the Peering Connections dashboard, select the newly created peering connection, and click "Actions" > "Accept Request": ![](https://user-images.githubusercontent.com/808475/80132168-21d97a00-8550-11ea-8c22-79c65710d369.png) ![](https://user-images.githubusercontent.com/808475/80132179-26059780-8550-11ea-80fc-6670fcab7026.png) ## Update route tables Navigate to the VPC Route Tables page. Select the route table for the VPC from which you'd like to connect to the Cortex cluster (in my case, I just have one route table for this VPC). Select the "Routes" tab, and click "Edit routes": ![](https://user-images.githubusercontent.com/808475/80135180-b940cc00-8554-11ea-8162-c7409090897b.png) Add a route where the "Destination" is the CIDR block for Cortex's VPC, and the "Target" is the newly-created Peering Connection: ![](https://user-images.githubusercontent.com/808475/80137033-78968200-8557-11ea-9d84-9221b772f0fc.png) Do not create new route tables or change subnet associations. Navigate back to the VPC Route Tables page. There will be a route table for each of the subnets associated with the Cortex operator load balancer: ![](https://user-images.githubusercontent.com/808475/80138244-5dc50d00-8559-11ea-9248-fc201d011530.png) For each of these route tables, click "Edit routes" and add a new route where the "Destination" is the CIDR block for the VPC from which you will be connecting to the Cortex cluster, and the "Target" is the newly-created Peering Connection: ![](https://user-images.githubusercontent.com/808475/80138653-f78cba00-8559-11ea-8444-406e218c3bab.png) Repeat adding this route for each route table associated with the Cortex operator's subnets; in my case there were three. Do not create new route tables or change subnet associations. You should now be able to use the Cortex CLI and make requests from your VPC. ## Cleanup Delete the VPC Peering connection before spinning down your Cortex cluster: ![](https://user-images.githubusercontent.com/808475/80138851-57836080-855a-11ea-92f1-06d501932a41.png) ================================================ FILE: docs/clusters/observability/alerting.md ================================================ # Alerting Cortex supports setting alerts for your APIs out-of-the-box. Alerts are an effective way of identifying problems in your system as they occur. The following dashboards can be configured with alerts: - RealtimeAPI - BatchAPI - Cluster resources - Node resources This page demonstrates the process for configuring alerts for a realtime API. The same principles apply to the others as well. Alerts can be configured for a variety of notification channels such as for Slack, Discord, Microsoft Teams, PagerDuty, Telegram and traditional webhooks. In this example, we'll use Slack. If you don't know how to access the Grafana dashboard for your API, make sure you check out [this page](metrics.md) first. ## Create a Slack channel Create a slack channel on your team's Slack workspace. We'll name ours "cortex-alerts". Add an _Incoming Webhook_ to your channel and retrieve the webhook URL. It will look like something like `https://hooks.slack.com/services///`. ## Create a Grafana notification channel Go to Grafana and on the left-hand side panel, hover over the alerting bell and select _"Notification channels"_. ![](https://user-images.githubusercontent.com/26958764/114937638-b6667780-9e46-11eb-963a-8a53e5655c3d.png) Click on _"Create channel"_, add the name of your channel, select _Slack_ as the type of the channel, and paste your Slack webhook URL. ![](https://user-images.githubusercontent.com/26958764/114937856-06ddd500-9e47-11eb-8f47-47b043b0bb5c.png) Click _"Test"_ and see if a sample notification is sent to your Slack channel. If the message goes through, click on _"Save"_. ![](https://user-images.githubusercontent.com/26958764/114938358-b2872500-9e47-11eb-87aa-ee818aae4cd0.png) ## Create alerts Now that the notification channel is functioning properly, we can create alerts for our APIs and cluster. For all of our examples, we are using the `mpg-estimator` API as an example. ![](https://user-images.githubusercontent.com/26958764/114939831-a8662600-9e49-11eb-8774-fbac3ce627d9.png) ### API replica threshold alert Let's create an alert for the _"Active Replicas"_ panel. We want to send notifications every time the number of replicas for the given API exceeds a certain threshold. Edit the _"Active Replicas"_ panel. ![](https://user-images.githubusercontent.com/26958764/114941416-d2b8e300-9e4b-11eb-8def-ee64535fc799.png) Create a copy of the primary query by clicking on the "duplicate" icon. ![](https://user-images.githubusercontent.com/26958764/114941557-fe3bcd80-9e4b-11eb-8f69-b9d43ff8eb28.png) In the copied query, replace the `$api_name` variable with the name of the API you want to create the alert for. In our case, it's `mpg-estimator`. Also, click on the eye icon to disable the query from being shown on the graph - otherwise, you'll see duplicates. ![](https://user-images.githubusercontent.com/26958764/114941701-275c5e00-9e4c-11eb-991b-34d660d0d05c.png) Go to the _"Alert"_ tab and click _"Create Alert"_. ![](https://user-images.githubusercontent.com/26958764/114941779-40fda580-9e4c-11eb-9aee-514e6b4832ba.png) Configure your alert like in the following example and click _"Apply"_. ![](https://user-images.githubusercontent.com/26958764/114944749-d7cc6100-9e50-11eb-9b6b-b2c3dabcc78c.png) The next time the threshold is exceeded, a notification will be sent to your Slack channel. ![](https://user-images.githubusercontent.com/26958764/114948423-a3a86e80-9e57-11eb-8717-94e456a15298.png) ### In-flight requests spike alert Let's add an alert on the _"In-Flight Requests"_ panel. We want to send an alert if the metric exceeds 50 in-flight requests. For this, follow the same set of instructions as for the previous alert, but this time configure the alert to match the following screenshot: ![](https://user-images.githubusercontent.com/26958764/114949182-1bc36400-9e59-11eb-9c19-0d788872a388.png) An alert triggered for this will look like: ![](https://user-images.githubusercontent.com/26958764/114949593-000c8d80-9e5a-11eb-8cb5-b2c9a2b344e8.png) #### Memory usage alert Let's add another alert, this time for the _"Avg Memory Usage"_ panel. We want to send an alert if the average memory usage per API replica exceeds its memory request. For this, we need to follow the same set of instructions as for the first alert, but this time the hidden query needs to be expressed as the ratio between the memory usage and memory request: ![](https://user-images.githubusercontent.com/26958764/114951903-f1c07080-9e5d-11eb-9aaf-898d46efb7ef.png) The memory usage alert can to be defined like in the following screenshot: ![](https://user-images.githubusercontent.com/26958764/114951782-bfaf0e80-9e5d-11eb-834d-e48ab3546d3c.png) The resulting alert will look like this: ![](https://user-images.githubusercontent.com/26958764/114952346-bd00e900-9e5e-11eb-879a-5851dab7630b.png) ## Persistent changes To save your changes permanently, go back to your dashboard and click on the save icon on the top-right corner. ![](https://user-images.githubusercontent.com/26958764/114953264-af4c6300-9e60-11eb-8095-40e438c125d8.png) Copy the JSON to your clipboard. ![](https://user-images.githubusercontent.com/26958764/114953338-d6a33000-9e60-11eb-8390-0f24704c5b7d.png) Click on the settings button on the top-right corner of your dashboard. ![](https://user-images.githubusercontent.com/26958764/114953437-00f4ed80-9e61-11eb-91f6-4b669ffe0c16.png) Go to the _"JSON Model"_ section and replace the JSON with the one you've copied to your clipboard. Then click _"Save Changes"_. ![](https://user-images.githubusercontent.com/26958764/114953473-1ec25280-9e61-11eb-8fcc-12615b73067a.png) Your dashboard now has stored the alert configuration permanently. ## Multiple APIs alerts Due to how Grafana was built, you'll need to re-do the steps of setting a given alert for each individual API. That's because Grafana doesn't currently support alerts on template or transformation queries. ## Enabling email alerts It is possible to manually configure SMTP to enable email alerts (we plan on automating this process, see [#2210](https://github.com/cortexlabs/cortex/issues/2210)). **Step 1** Install [kubectl](../advanced/kubectl.md). **Step 2** ```bash kubectl create secret generic grafana-smtp \ --from-literal=GF_SMTP_ENABLED=true \ --from-literal=GF_SMTP_HOST= \ --from-literal=GF_SMTP_USER= \ --from-literal=GF_SMTP_FROM_ADDRESS= \ --from-literal=GF_SMTP_PASSWORD= ``` The `` varies from provider to provider (e.g. Gmail's is `smtp.gmail.com:587`). **Step 3** Edit Grafana's statefulset by running `kubectl edit statefulset grafana` (this will open a code editor). Inside the container named `grafana` (in the `containers` section), add an `envFrom` section that will mount the SMTP secret. Here is an example of what it looks like after the addition: ```yaml # ... containers: - env: - name: GF_SERVER_ROOT_URL value: '%(protocol)s://%(domain)s:%(http_port)s/dashboard' - name: GF_SERVER_SERVE_FROM_SUB_PATH value: "true" - name: GF_USERS_DEFAULT_THEME value: light envFrom: - secretRef: name: grafana-smtp image: quay.io/cortexlabs/grafana:0.42.1 imagePullPolicy: IfNotPresent name: grafana # ... ``` Save and close your editor. It will take 30-60 seconds for Grafana to restart, after which you can access the dashboard. You can check the logs with `kubectl logs -f grafana-0`. ================================================ FILE: docs/clusters/observability/logging.md ================================================ # Logging Logs are collected with Fluent Bit and are exported to CloudWatch. ## Logs on AWS Logs will automatically be pushed to CloudWatch and a log group with the same name as your cluster will be created to store your logs. API logs are tagged with labels to help with log aggregation and filtering. Log lines greater than 5 MB in size will be ignored. You can use the `cortex logs` command to get a CloudWatch Insights URL of query to fetch logs for your API. Please note that there may be a few minutes of delay from when a message is logged to when it is available in CloudWatch Insights. **RealtimeAPI:** ```text fields @timestamp, message | filter cortex.labels.apiName="" | filter cortex.labels.apiKind="RealtimeAPI" | sort @timestamp asc | limit 1000 ``` **AsyncAPI:** ```text fields @timestamp, message | filter cortex.labels.apiName="" | filter cortex.labels.apiKind="AsyncAPI" | sort @timestamp asc | limit 1000 ``` **BatchAPI:** ```text fields @timestamp, message | filter cortex.labels.apiName="" | filter cortex.labels.jobID="" | filter cortex.labels.apiKind="BatchAPI" | sort @timestamp asc | limit 1000 ``` **TaskAPI:** ```text fields @timestamp, message | filter cortex.labels.apiName="" | filter cortex.labels.jobID="" | filter cortex.labels.apiKind="TaskAPI" | sort @timestamp asc | limit 1000 ``` ## Streaming logs from the CLI You can stream logs directly from a random pod of a running workload to iterate and debug quickly. These logs will not be as comprehensive as the logs that are available in CloudWatch. ```bash # RealtimeAPI cortex logs --random-pod # BatchAPI or TaskAPI cortex logs --random-pod # the job must be in a running state ``` ## Structured logging If you log JSON strings from your APIs, they will be automatically parsed before pushing to CloudWatch. It is recommended to configure your JSON logger to use `message` or `msg` as the key for the log line if you would like the sample queries above to display the messages in your logs. Avoid using top-level keys that start with "cortex" to prevent collisions with Cortex's internal logging. ## Exporting logs You can export both the Cortex system logs and your application logs to your desired destination by configuring FluentBit. ### Configure kubectl Follow these [instructions](../advanced/kubectl.md) to set up kubectl. ### Find supported destinations in FluentBit Visit FluentBit's [output docs](https://docs.fluentbit.io/manual/concepts/data-pipeline/output) to see a list supported destinations. Make sure to navigate to the version of FluentBit being used in your cluster. You can find the version of FluentBit by looking at the first view lines of one of the FluentBit pod logs. Get the FluentBit pods: ```bash kubectl get pods --selector app=fluent-bit ``` FluentBit's version should be in the first few log lines of a FluentBit pod: ```bash kubectl logs fluent-bit-kxmzn | head -n 20 ``` ### Update FluentBit configuration Define `patch.yaml` with your new output configuration: ```yaml data: output.conf: | [OUTPUT] Name es Match k8s_container.* Host https://abc123.us-west-2.es.amazonaws.com Port 443 AWS_Region us-west-2 AWS_Auth On tls On Logstash_Format On Logstash_Prefix my-logs ``` Update FluentBit's configuration: ```bash kubectl patch configmap fluent-bit-config --patch-file patch.yaml --type merge ``` ### Restart FluentBit Restart FluentBit to apply the new configuration: ```bash kubectl rollout restart daemonset/fluent-bit ``` ================================================ FILE: docs/clusters/observability/metrics.md ================================================ # Metrics Cortex includes Prometheus for metrics collection and Grafana for visualization. You can monitor your APIs with the default Grafana dashboards, or create custom metrics and dashboards. ## Accessing the dashboard The dashboard URL is displayed once you run a `cortex get ` command. Alternatively, you can access it on `http:///dashboard`. Run the following command to get the operator URL: ```bash cortex env list ``` If your operator load balancer is configured to be internal, there are a few options for accessing the dashboard: 1. Access the dashboard from a machine that has VPC Peering configured to your cluster's VPC, or which is inside of your cluster's VPC. 1. Run `kubectl port-forward -n default grafana-0 3000:3000` to forward Grafana's port to your local machine, and access the dashboard on [http://localhost:3000](http://localhost:3000) (see instructions for setting up `kubectl` [here](../advanced/kubectl.md)). 1. Set up VPN access to your cluster's VPC ([docs](https://docs.aws.amazon.com/vpc/latest/userguide/vpn-connections.html)). ### Default credentials The dashboard is protected with username / password authentication, which by default are: - Username: admin - Password: admin You will be prompted to change the admin user password in the first time you log in. Grafana allows managing the access of several users and managing teams. For more information on this topic check the [grafana documentation](https://grafana.com/docs/grafana/latest/manage-users). ### Selecting an API You can select one or more APIs to visualize in the top left corner of the dashboard. ![](https://user-images.githubusercontent.com/7456627/107375721-57545180-6ae9-11eb-9474-ba58ad7eb0c5.png) ### Selecting a time range Grafana allows you to select a time range on which the metrics will be visualized. You can do so in the top right corner of the dashboard. ![](https://user-images.githubusercontent.com/7456627/107376148-d9dd1100-6ae9-11eb-8c2b-c678b41ade01.png) **Note: Cortex only retains a maximum of 2 weeks worth of data at any moment in time** ### Available dashboards There are more than one dashboard available by default. You can view the available dashboards by accessing the Grafana menu: `Dashboards -> Manage -> Cortex folder`. The dashboards that Cortex ships with are the following: - RealtimeAPI - BatchAPI - Cluster resources - Node resources ### Available metrics Cortex exposes additional metrics with Prometheus. To view all available metrics, navigate to the `Explore` menu in Grafana and click the `Metrics` button. ![](https://user-images.githubusercontent.com/7456627/107377492-515f7000-6aeb-11eb-9b46-909120335060.png) You can use any of these metrics to set up your own dashboards. ## Exporting metrics to monitoring solutions You can scrape metrics from the in-cluster Prometheus server via the `/federate` endpoint and push them to monitoring solutions such as Datadog. The steps for exporting metrics from Prometheus will vary based on your monitoring solution. Here are a few high-level steps to get you started. We will be using Datadog as an example; feel free to reach out to us on [Slack](https://community.cortexlabs.com/) if you need help setting up your monitoring tool. ### Configure kubectl Follow these [instructions](../advanced/kubectl.md) to set up kubectl. ### Install agent Monitoring solutions provide Kubernetes agents that are capable of scraping Prometheus metrics. Follow the appropriate instructions to install the agent onto your cluster (here are the [instructions](https://docs.datadoghq.com/agent/kubernetes/?tab=helm#installation) for Datadog). ### Scrape Prometheus Some agents require a Prometheus endpoint to scrape directly. You can provide `http://prometheus.default:9090/federate?match[]={job=~".+"}` as the target url to indicate that all metrics should be scraped. Some agents look for targets to scrape via annotations. You can update Cortex's Prometheus server with the correct annotations. First, Create a `patch.yaml` file and add the relevant annotations for your monitoring solution. Below is an example for [Datadog](https://docs.datadoghq.com/agent/kubernetes/prometheus/). These annotations instruct the Datadog agent to scrape the Prometheus server at the endpoint `/federate?match[]={job=~".+"}` and extract `cortex_in_flight_requests`. Note that Datadog specifically requires the query params in the Prometheus url to be encoded. ```yaml spec: podMetadata: annotations: ad.datadoghq.com/prometheus.check_names: | ["prometheus"] ad.datadoghq.com/prometheus.init_configs: | [{}] ad.datadoghq.com/prometheus.instances: | [ { "prometheus_url": "http://%%host%%:%%port%%/federate?match[]=%7Bjob%3D~%22.%2B%22%7D", "namespace": "cortex", "metrics": [{"cortex_in_flight_requests":"in_flight_requests"}] } ] ``` Then, update Prometheus with your annotations: ```bash kubectl patch --namespace prometheus prometheuses.monitoring.coreos.com prometheus --patch-file patch.yaml --type merge ``` ## Long term metric storage Prometheus can be configured to write metrics to other monitoring solutions or databases for long term storage. You can attach a remote storage adapter to Prometheus that will receive samples from Prometheus and write to your destination. You can find a list of Prometheus remote storage adapters [here](https://prometheus.io/docs/operating/integrations/#remote-endpoints-and-storage). Additional remote storage adapters can be found online if yours isn't on the list. Once you've found an adapter that works for you, follow the steps below: ### Configure kubectl Follow these [instructions](../advanced/kubectl.md) to set up kubectl. ### Update Prometheus Define a `patch.yaml` file with your changes to the Prometheus server: ```yaml spec: containers: # container for your adapter ... remoteWrite: url: "http://localhost:9201/write" # http endpoint for your adapter ``` Update Prometheus with your changes: ```bash kubectl patch --namespace prometheus prometheuses.monitoring.coreos.com prometheus --patch-file patch.yaml --type merge ``` ================================================ FILE: docs/overview.md ================================================ # Overview ## Cluster The Cortex cluster runs on an EKS (Kubernetes) cluster in a dedicated VPC on your AWS account. ### Worker node groups The Kubernetes cluster uses EC2 autoscaling groups for its worker node groups. Cortex supports most EC2 instance types, and the necessary device drivers are installed to expose GPU and Inferentia hardware to your workloads. Reserved and spot instances can be used to reduce costs. Cortex uses the Kubernetes Cluster Autoscaler to scale the appropriate node groups to satisfy the compute demands of your workloads. ### Networking By default, a new dedicated VPC is created for the cluster during installation. Two AWS load balancers are created to route traffic to the cluster. One load balancer is dedicated for traffic to your APIs, and the other load balancer is dedicated for API management requests to Cortex from your CLI or Python client. Traffic to the load balancers can be secured and restricted based on your cluster configuration. ### Observability All logs from the Cortex cluster are pushed to a CloudWatch log group using FluentBit. An in-cluster Prometheus installation is used to collect metrics for observability and autoscaling purposes. Metrics and dashboards pertaining to your workloads and instance usage can be viewed and modified via Grafana. ## Deploying to the cluster After a successful cluster creation, you can use the CLI or Python Client to deploy different types of workloads. The clients use AWS credentials to authenticate to the cluster. Cortex uses a collection of containers, referred to as a pod, as the atomic unit; scaling and replication occurs at the pod level. The orchestration and scaling of pods is unique to the different types of workloads: * Realtime * Async * Batch * Task Visit the workload-specific documentation for more details. ## Architecture Diagram ![](https://user-images.githubusercontent.com/808475/146854233-505fb8c2-513d-4836-920b-5d447da95dee.png) ================================================ FILE: docs/start.md ================================================ # Get started ## Create a cluster on your AWS account ```bash # install the CLI bash -c "$(curl -sS https://raw.githubusercontent.com/cortexlabs/cortex/v0.42.1/get-cli.sh)" # create a cluster cortex cluster up cluster.yaml ``` * [Client installation](clients/install.md) - customize your client installation. * [Cluster configuration](clusters/management/create.md) - optimize your cluster for your workloads. * [Environments](clusters/management/environments.md) - manage multiple clusters. ## Build scalable APIs ```bash # deploy APIs cortex deploy apis.yaml ``` * [Realtime](workloads/realtime/example.md) - create APIs that respond to requests in real-time. * [Async](workloads/async/example.md) - create APIs that respond to requests asynchronously. * [Batch](workloads/batch/example.md) - create APIs that run distributed batch jobs. * [Task](workloads/task/example.md) - create APIs that run jobs on-demand. ================================================ FILE: docs/summary.md ================================================ # Summary * [Get started](start.md) * [Overview](overview.md) ## Clusters * Management * [Auth](clusters/management/auth.md) * [Create](clusters/management/create.md) * [Update](clusters/management/update.md) * [Delete](clusters/management/delete.md) * [Environments](clusters/management/environments.md) * [Production Guide](clusters/management/production.md) * Instances * [Multi-instance](clusters/instances/multi.md) * [Spot instances](clusters/instances/spot.md) * Observability * [Logging](clusters/observability/logging.md) * [Metrics](clusters/observability/metrics.md) * [Alerting](clusters/observability/alerting.md) * Networking * [Load balancers](clusters/networking/load-balancers.md) * [Custom domain](clusters/networking/custom-domain.md) * [HTTPS](clusters/networking/https.md) * [HTTPS with API Gateway](clusters/networking/api-gateway.md) * [VPC peering](clusters/networking/vpc-peering.md) * Advanced * [Setting up kubectl](clusters/advanced/kubectl.md) * [Private Docker registry](clusters/advanced/registry.md) * [Self hosted images](clusters/advanced/self-hosted-images.md) ## Workloads * [Realtime](workloads/realtime/realtime.md) * [Example](workloads/realtime/example.md) * [Configuration](workloads/realtime/configuration.md) * [Containers](workloads/realtime/containers.md) * [Autoscaling](workloads/realtime/autoscaling.md) * [Traffic Splitter](workloads/realtime/traffic-splitter.md) * [Metrics](workloads/realtime/metrics.md) * [Statuses](workloads/realtime/statuses.md) * [Troubleshooting](workloads/realtime/troubleshooting.md) * [Async](workloads/async/async.md) * [Example](workloads/async/example.md) * [Configuration](workloads/async/configuration.md) * [Containers](workloads/async/containers.md) * [Autoscaling](workloads/async/autoscaling.md) * [Statuses](workloads/async/statuses.md) * [Batch](workloads/batch/batch.md) * [Example](workloads/batch/example.md) * [Configuration](workloads/batch/configuration.md) * [Containers](workloads/batch/containers.md) * [Jobs](workloads/batch/jobs.md) * [Statuses](workloads/batch/statuses.md) * [Task](workloads/task/task.md) * [Example](workloads/task/example.md) * [Configuration](workloads/task/configuration.md) * [Containers](workloads/task/containers.md) * [Jobs](workloads/task/jobs.md) * [Statuses](workloads/task/statuses.md) ## Clients * [Install](clients/install.md) * [Uninstall](clients/uninstall.md) * [CLI commands](clients/cli.md) * [Python client](clients/python.md) ================================================ FILE: docs/workloads/async/async.md ================================================ # Async Async APIs are designed for asynchronous workloads in which the user submits an asynchronous request and retrieves the result later (either by polling or through a webhook). Async APIs are a good fit for users who want to submit longer workloads (such as video, audio or document processing), and do not need the result immediately or synchronously. **Key features** * asynchronously process requests * retrieve status and response via HTTP endpoint * autoscale based on queue length * avoid cold starts * scale to zero * perform rolling updates * automatically recover from failures and spot instance termination ## How it works When you deploy an AsyncAPI, Cortex creates an SQS queue, a pool of Async Gateway workers, and a pool of worker pods. Each worker pod is running a dequeuer sidecar and your containers. Upon receiving a request, the Async Gateway will save the request payload to S3, enqueue the request ID onto an SQS FIFO queue, and respond with the request ID. The dequeuer sidecar in the worker pod will pull the request from the SQS queue, download the request's payload from S3, and make a POST request to your containers. After the dequeuer receives a response, the corresponding request payload will be deleted from S3 and the response will be saved in S3 for 7 days. You can fetch the result by making a GET request to the AsyncAPI endpoint with the request ID. The Async Gateway will respond with the status and the result (if the request has been completed). The pool of workers running your containers autoscales based on the average number of messages in the queue and can scale down to 0 (if configured to do so). ![](https://user-images.githubusercontent.com/808475/146854251-fed4235f-3627-4cd0-bc86-066272d7f138.png) ================================================ FILE: docs/workloads/async/autoscaling.md ================================================ # Autoscaling Cortex auto-scales AsyncAPIs on a per-API basis based on your configuration. ## Autoscaling replicas ### Relevant pod configuration In addition to the autoscaling configuration options (described below), there is one field in the pod configuration which is relevant to replica autoscaling: **`max_concurrency`** (default: 1): The maximum number of requests that will be concurrently sent into the container by Cortex. If your web server is designed to handle multiple concurrent requests, increasing `max_concurrency` will increase the throughput of a replica (and result in fewer total replicas for a given load).
### Autoscaling configuration **`min_replicas`** (default: 1): The lower bound on how many replicas can be running for an API. Scale-to-zero is supported.
**`max_replicas`** (default: 100): The upper bound on how many replicas can be running for an API.
**`target_in_flight`** (default: `max_concurrency` in the pod configuration): This is the desired number of in-flight requests per replica, and is the metric which the autoscaler uses to make scaling decisions. The number of in-flight requests is simply how many requests have been submitted and are not yet finished being processed. Therefore, this number includes requests which are actively being processed as well as requests which are waiting in the queue. The autoscaler uses this formula to determine the number of desired replicas: `desired replicas = total in-flight requests / target_in_flight` For example, setting `target_in_flight` to `max_concurrency` (the default) causes the cluster to adjust the number of replicas so that on average, there are no requests waiting in the queue.
**`window`** (default: 60s): The time over which to average the API's in-flight requests. The longer the window, the slower the autoscaler will react to changes in in-flight requests, since it is averaged over the `window`. An API's in-flight requests is calculated every 10 seconds, so `window` must be a multiple of 10 seconds.
**`downscale_stabilization_period`** (default: 5m): The API will not scale below the highest recommendation made during this period. Every 10 seconds, the autoscaler makes a recommendation based on all of the other configuration parameters described here. It will then take the max of the current recommendation and all recommendations made during the `downscale_stabilization_period`, and use that to determine the final number of replicas to scale to. Increasing this value will cause the cluster to react more slowly to decreased traffic, and will reduce thrashing.
**`upscale_stabilization_period`** (default: 1m): The API will not scale above the lowest recommendation made during this period. Every 10 seconds, the autoscaler makes a recommendation based on all of the other configuration parameters described here. It will then take the min of the current recommendation and all recommendations made during the `upscale_stabilization_period`, and use that to determine the final number of replicas to scale to. Increasing this value will cause the cluster to react more slowly to increased traffic, and will reduce thrashing.
**`max_downscale_factor`** (default: 0.75): The maximum factor by which to scale down the API on a single scaling event. For example, if `max_downscale_factor` is 0.5 and there are 10 running replicas, the autoscaler will not recommend fewer than 5 replicas. Increasing this number will allow the cluster to shrink more quickly in response to dramatic dips in traffic.
**`max_upscale_factor`** (default: 1.5): The maximum factor by which to scale up the API on a single scaling event. For example, if `max_upscale_factor` is 10 and there are 5 running replicas, the autoscaler will not recommend more than 50 replicas. Increasing this number will allow the cluster to grow more quickly in response to dramatic spikes in traffic.
**`downscale_tolerance`** (default: 0.05): Any recommendation falling within this factor below the current number of replicas will not trigger a scale down event. For example, if `downscale_tolerance` is 0.1 and there are 20 running replicas, a recommendation of 18 or 19 replicas will not be acted on, and the API will remain at 20 replicas. Increasing this value will prevent thrashing, but setting it too high will prevent the cluster from maintaining it's optimal size.
**`upscale_tolerance`** (default: 0.05): Any recommendation falling within this factor above the current number of replicas will not trigger a scale-up event. For example, if `upscale_tolerance` is 0.1 and there are 20 running replicas, a recommendation of 21 or 22 replicas will not be acted on, and the API will remain at 20 replicas. Increasing this value will prevent thrashing, but setting it too high will prevent the cluster from maintaining it's optimal size.
## Autoscaling instances Cortex spins up and down instances based on the aggregate resource requests of all APIs. The number of instances will be at least `min_instances` and no more than `max_instances` for each node group (configured during installation and modifiable via `cortex cluster configure`). ## Overprovisioning The default value for `target_in_flight` is `max_concurrency`, which behaves well in many situations (see above for an explanation of how `target_in_flight` affects autoscaling). However, if your application is sensitive to spikes in traffic or if creating new replicas takes too long (see below), you may find it helpful to maintain extra capacity to handle the increased traffic while new replicas are being created. This can be accomplished by setting `target_in_flight` to a lower value relative to the expected replica's concurrency. The smaller `target_in_flight` is, the more unused capacity your API will have, and the more room it will have to handle sudden increased load. The increased request rate will still trigger the autoscaler, and your API will stabilize again (maintaining the overprovisioned capacity). For example, if you've determined that each replica in your API can efficiently handle 2 concurrent requests, you would typically set `target_in_flight` to 2. In a scenario where your API is receiving 8 concurrent requests on average, the autoscaler would maintain 4 live replicas (8/2 = 4). If you wanted to overprovision by 25%, you could set `target_in_flight` to 1.6, causing the autoscaler maintain 5 live replicas (8/1.6 = 5). ## Autoscaling responsiveness Assuming that `window` and `upscale_stabilization_period` are set to their default values (1 minute), it could take up to 2 minutes of increased traffic before an extra replica is requested. As soon as the additional replica is requested, the replica request will be visible in the output of `cortex get`, but the replica won't yet be running. If an extra instance is required to schedule the newly requested replica, it could take a few minutes for AWS to provision the instance (depending on the instance type), plus a few minutes for the newly provisioned instance to download your api image and for the api to initialize. Keep these delays in mind when considering overprovisioning (see above) and when determining appropriate values for `window` and `upscale_stabilization_period`. If you want the autoscaler to react as quickly as possible, set `upscale_stabilization_period` and `window` to their minimum values (0s and 10s respectively). ================================================ FILE: docs/workloads/async/configuration.md ================================================ # Configuration ```yaml - name: # name of the API (required) kind: AsyncAPI # must be "AsyncAPI" for async APIs (required) pod: # pod configuration (required) port: # port to which requests will be sent (default: 8080; exported as $CORTEX_PORT) max_concurrency: # maximum number of requests that will be concurrently sent into the container (default: 1, max allowed: 100) containers: # configurations for the containers to run (at least one constainer must be provided) - name: # name of the container (required) image: # docker image to use for the container (required) command: # entrypoint (not executed within a shell); env vars can be used with e.g. $(CORTEX_PORT) (default: the docker image's ENTRYPOINT) args: # arguments to the entrypoint; env vars can be used with e.g. $(CORTEX_PORT) (default: the docker image's CMD) env: # dictionary of environment variables to set in the container (optional) compute: # compute resource requests (default: see below) cpu: # CPU request for the container; one unit of CPU corresponds to one virtual CPU; fractional requests are allowed, and can be specified as a floating point number or via the "m" suffix (default: 200m) gpu: # GPU request for the container; one unit of GPU corresponds to one virtual GPU (default: 0) inf: # Inferentia request for the container; one unit of inf corresponds to one virtual Inferentia chip (default: 0) mem: # memory request for the container; one unit of memory is one byte and can be expressed as an integer or by using one of these suffixes: K, M, G, T (or their power-of two counterparts: Ki, Mi, Gi, Ti) (default: Null) shm: # size of shared memory (/dev/shm) for sharing data between multiple processes, e.g. 64Mi or 1Gi (default: Null) readiness_probe: # periodic probe of container readiness; traffic will not be sent into the pod unless all containers' readiness probes are succeeding (optional) http_get: # specifies an http endpoint which must respond with status code 200 (only one of http_get and tcp_socket may be specified) port: # the port to access on the container (required) path: # the path to access on the HTTP server (default: /) tcp_socket: # specifies a port which must be ready to receive traffic (only one of http_get and tcp_socket may be specified) port: # the port to access on the container (required) initial_delay_seconds: # number of seconds after the container has started before the probe is initiated (default: 0) timeout_seconds: # number of seconds until the probe times out (default: 1) period_seconds: # how often (in seconds) to perform the probe (default: 10) success_threshold: # minimum consecutive successes for the probe to be considered successful after having failed (default: 1) failure_threshold: # minimum consecutive failures for the probe to be considered failed after having succeeded (default: 3) liveness_probe: # periodic probe of container liveness; container will be restarted if the probe fails (optional) http_get: # specifies an http endpoint which must respond with status code 200 (only one of http_get, tcp_socket, and exec may be specified) port: # the port to access on the container (required) path: # the path to access on the HTTP server (default: /) tcp_socket: # specifies a port which must be ready to receive traffic (only one of http_get, tcp_socket, and exec may be specified) port: # the port to access on the container (required) exec: # specifies a command to run which must exit with code 0 (only one of http_get, tcp_socket, and exec may be specified) command: # the command to execute inside the container, which is exec'd (not run inside a shell); the working directory is root ('/') in the container's filesystem (required) initial_delay_seconds: # number of seconds after the container has started before the probe is initiated (default: 0) timeout_seconds: # number of seconds until the probe times out (default: 1) period_seconds: # how often (in seconds) to perform the probe (default: 10) success_threshold: # minimum consecutive successes for the probe to be considered successful after having failed (default: 1) failure_threshold: # minimum consecutive failures for the probe to be considered failed after having succeeded (default: 3) pre_stop: # a pre-stop lifecycle hook for the container; will be executed before container termination (optional) http_get: # specifies an http endpoint to send a request to (only one of http_get, tcp_socket, and exec may be specified) port: # the port to access on the container (required) path: # the path to access on the HTTP server (default: /) exec: # specifies a command to run (only one of http_get, tcp_socket, and exec may be specified) command: # the command to execute inside the container, which is exec'd (not run inside a shell); the working directory is root ('/') in the container's filesystem (required) autoscaling: # autoscaling configuration (default: see below) min_replicas: # minimum number of replicas (default: 1; min value: 0) max_replicas: # maximum number of replicas (default: 100) init_replicas: # initial number of replicas (default: ) target_in_flight: # desired number of in-flight requests per replica (including requests actively being processed as well as queued), which the autoscaler tries to maintain (default: ) window: # duration over which to average the API's in-flight requests per replica (default: 60s) downscale_stabilization_period: # the API will not scale below the highest recommendation made during this period (default: 5m) upscale_stabilization_period: # the API will not scale above the lowest recommendation made during this period (default: 1m) max_downscale_factor: # maximum factor by which to scale down the API on a single scaling event (default: 0.75) max_upscale_factor: # maximum factor by which to scale up the API on a single scaling event (default: 1.5) downscale_tolerance: # any recommendation falling within this factor below the current number of replicas will not trigger a scale down event (default: 0.05) upscale_tolerance: # any recommendation falling within this factor above the current number of replicas will not trigger a scale-up event (default: 0.05) node_groups: # a list of node groups on which this API can run (default: all node groups are eligible) update_strategy: # deployment strategy to use when replacing existing replicas with new ones (default: see below) max_surge: # maximum number of replicas that can be scheduled above the desired number of replicas during an update; can be an absolute number, e.g. 5, or a percentage of desired replicas, e.g. 10% (default: 25%) (set to 0 to disable rolling updates) max_unavailable: # maximum number of replicas that can be unavailable during an update; can be an absolute number, e.g. 5, or a percentage of desired replicas, e.g. 10% (default: 25%) networking: # networking configuration (default: see below) endpoint: # endpoint for the API (default: ) ``` ================================================ FILE: docs/workloads/async/containers.md ================================================ # Containers ## Handling requests In order to handle requests to your Async API, one of your containers must run a web server which is listening for HTTP requests on the port which is configured in the `pod.port` field of your [API configuration](configuration.md) (default: 8080). Requests will be sent to your web server via HTTP POST requests to the root path (`/`) as they are pulled off of the queue. The payload and the content type header of the HTTP request to your web server will match those of the original request to your Async API. In addition, the request's ID will be passed in via the "X-Cortex-Request-ID" header. Your web server must respond with valid JSON (with the `Content-Type` header set to "application/json"). The response will remain queryable for 7 days. ## Readiness checks It is often important to implement a readiness check for your API. By default, as soon as your web server has bound to the port, it will start receiving traffic. In some cases, the web server may start listening on the port before its workers are ready to handle traffic (e.g. `tiangolo/uvicorn-gunicorn-fastapi` behaves this way). Readiness checks ensure that traffic is not sent into your web server before it's ready to handle them. There are two types of readiness checks which are supported: `http_get` and `tcp_socket` (see [API configuration](configuration.md) for usage instructions). A simple and often effective approach is to add a route to your web server (e.g. `/healthz`) which responds with status code 200, and configure your readiness probe accordingly: ```yaml readiness_probe: http_get: port: 8080 path: /healthz ``` ## Multiple containers Your API pod can contain multiple containers, only one of which can be listening for requests on the target port (it can be any of the containers). The `/mnt` directory is mounted to each container's filesystem, and is shared across all containers. ## Resource requests Each container in the pod requests its own amount of CPU, memory, GPU, and Inferentia resources. In addition, Cortex's dequeuer sidecar container (which is automatically added to the pod) requests 100m CPU and 100Mi memory. ## Observability See docs for [logging](../../clusters/observability/logging.md), [metrics](../../clusters/observability/metrics.md), and [alerting](../../clusters/observability/metrics.md). ## Using the Cortex CLI or client It is possible to use the Cortex CLI or client to interact with your cluster's APIs from within your API containers. All containers will have a CLI configuration file present at `/cortex/client/cli.yaml`, which is configured to connect to the cluster. In addition, the `CORTEX_CLI_CONFIG_DIR` environment variable is set to `/cortex/client` by default. Therefore, no additional configuration is required to use the CLI or Python client (which can be instantiated via `cortex.client()`). Note: your Cortex CLI or client must match the version of your cluster (available in the `CORTEX_VERSION` environment variable). ## Chaining APIs It is possible to submit requests to Async APIs from any Cortex API within a Cortex cluster. Requests can be made to `http://ingressgateway-apis.istio-system.svc.cluster.local/`, where `` is the name of the Async API you are making a request to. For example, if there is an Async API named `hello-world` running in the cluster, you can make a request to it from a different API in Python by using: ```python import requests # make a request to an Async API response = requests.post( "http://ingressgateway-apis.istio-system.svc.cluster.local/hello-world", json={"text": "hello world"}, ) # retreive a result from an Async API response = requests.get("http://ingressgateway-apis.istio-system.svc.cluster.local/hello-world/") ``` To make requests from your Async API to a Realtime, Batch, or Task API running within the cluster, see the "Chaining APIs" docs associated with the target workload type. ================================================ FILE: docs/workloads/async/example.md ================================================ # AsyncAPI ### Define an API ```python # main.py from fastapi import FastAPI from pydantic import BaseModel app = FastAPI() class Data(BaseModel): msg: str @app.post("/") def handle_async(data: Data): return data ``` ### Create a `Dockerfile` ```Dockerfile FROM python:3.8-slim RUN pip install --no-cache-dir fastapi uvicorn COPY main.py / CMD uvicorn --host 0.0.0.0 --port 8080 main:app ``` ### Build an image ```bash docker build . -t hello-world ``` ### Run a container locally ```bash docker run -p 8080:8080 hello-world ``` ### Make a request ```bash curl -X POST -H "Content-Type: application/json" -d '{"msg": "hello world"}' localhost:8080 ``` ### Login to ECR ```bash aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin .dkr.ecr.us-east-1.amazonaws.com ``` ### Create a repository ```bash aws ecr create-repository --repository-name hello-world ``` ### Tag the image ```bash docker tag hello-world .dkr.ecr.us-east-1.amazonaws.com/hello-world ``` ### Push the image ```bash docker push .dkr.ecr.us-east-1.amazonaws.com/hello-world ``` ### Configure a Cortex deployment ```yaml # cortex.yaml - name: hello-world kind: AsyncAPI pod: containers: - name: api image: .dkr.ecr.us-east-1.amazonaws.com/hello-world ``` ### Create a Cortex deployment ```bash cortex deploy ``` ### Wait for the API to be ready ```bash cortex get --watch ``` ### Get the API endpoint ```bash cortex get hello-world ``` ### Make a request ```bash curl -X POST -H "Content-Type: application/json" -d '{"msg": "hello world"}' http://***.amazonaws.com/hello-world ``` ### Get the response ```bash curl http://***.amazonaws.com/hello-world/ ``` ================================================ FILE: docs/workloads/async/statuses.md ================================================ # Request statuses | Status | Meaning | | :--- | :--- | | in_queue | Workload is in the queue and is yet to be consumed by the API | | in_progress | Workload has been pulled by the API and is currently being processed | | completed | Workload has completed with success | | failed | Workload encountered an error during processing | # Replica states The replica states of an API can be inspected by running `cortex describe `. Here are the possible states for each replica in an API: | State | Meaning | |:---|:---| | Ready | Replica is running and it has passed the readiness checks | | ReadyOutOfDate | Replica is running and it has passed the readiness checks (for an out-of-date replica) | | NotReady | Replica is running but it's not passing the readiness checks; make sure the server is listening on the designed port of the API | | Pending | Replica is in a pending state (waiting to get scheduled onto a node) | | Creating | Replica is in the process of having its containers created | | ErrImagePull | Replica was not created because one of the specified Docker images was inaccessible at runtime; check that your API's docker images exist and are accessible via your cluster's AWS credentials | | Failed | Replica couldn't start due to an error; run `cortex logs ` to view the logs | | Killed | Replica has had one of its containers killed | | KilledOOM | Replica was terminated due to excessive memory usage; try allocating more memory to the API and re-deploy | | Stalled | Replica has been in a pending state for more than 15 minutes; see [troubleshooting](../realtime/troubleshooting.md) | | Terminating | Replica is currently in the process of being terminated | | Unknown | Replica is in an unknown state | ================================================ FILE: docs/workloads/batch/batch.md ================================================ # Batch Batch APIs run distributed and fault-tolerant batch processing jobs on demand. Batch APIs are a good fit for users who want to break up their workloads and distribute them across a dedicated pool of workers (for example, running inference on a set of images). **Key features** * distribute a batch job across multiple workers * scale to 0 (when there are no batch jobs) * trigger `/on-job-complete` hook once all batches have been processed * attempt all batches at least once * reroute failed batches to a dead letter queue * automatically recover from failures and spot instance termination ## How it works When you deploy a Batch API, Cortex creates an endpoint to receive job submissions. Upon submitting a job, Cortex will respond with a Job ID, and will asynchronously trigger a Batch Job. A Batch Job begins with the deployment of an enqueuer process which breaks up the data in the job into batches and pushes them onto an SQS FIFO queue. After enqueuing is complete, Cortex initializes the requested number of worker pods and attaches a dequeuer sidecar to each pod. The dequeuer is responsible for retrieving batches from the queue and making an http request to your pod for each batch. After the worker pods have emptied the queue, the job is marked as complete, and Cortex will terminate the worker pods and delete the SQS queue. You can make GET requests to the BatchAPI endpoint to get the status of the Job and metrics such as the number of batches completed and failed. ![](https://user-images.githubusercontent.com/808475/146854256-b5b0c9a0-1753-4018-bda2-5ebddd8a6ffa.png) ================================================ FILE: docs/workloads/batch/configuration.md ================================================ # Configuration ```yaml - name: # name of the API (required) kind: BatchAPI # must be "BatchAPI" for batch APIs (required) pod: # pod configuration (required) port: # port to which requests will be sent (default: 8080; exported as $CORTEX_PORT) containers: # configurations for the containers to run (at least one constainer must be provided) - name: # name of the container (required) image: # docker image to use for the container (required) command: # entrypoint (not executed within a shell); env vars can be used with e.g. $(CORTEX_PORT) (required) args: # arguments to the entrypoint; env vars can be used with e.g. $(CORTEX_PORT) (default: no args) env: # dictionary of environment variables to set in the container (optional) compute: # compute resource requests (default: see below) cpu: # CPU request for the container; one unit of CPU corresponds to one virtual CPU; fractional requests are allowed, and can be specified as a floating point number or via the "m" suffix (default: 200m) gpu: # GPU request for the container; one unit of GPU corresponds to one virtual GPU (default: 0) inf: # Inferentia request for the container; one unit of inf corresponds to one virtual Inferentia chip (default: 0) mem: # memory request for the container; one unit of memory is one byte and can be expressed as an integer or by using one of these suffixes: K, M, G, T (or their power-of two counterparts: Ki, Mi, Gi, Ti) (default: Null) shm: # size of shared memory (/dev/shm) for sharing data between multiple processes, e.g. 64Mi or 1Gi (default: Null) readiness_probe: # periodic probe of container readiness; traffic will not be sent into the pod unless all containers' readiness probes are succeeding (optional) http_get: # specifies an http endpoint which must respond with status code 200 (only one of http_get, tcp_socket, and exec may be specified) port: # the port to access on the container (required) path: # the path to access on the HTTP server (default: /) tcp_socket: # specifies a port which must be ready to receive traffic (only one of http_get, tcp_socket, and exec may be specified) port: # the port to access on the container (required) initial_delay_seconds: # number of seconds after the container has started before the probe is initiated (default: 0) timeout_seconds: # number of seconds until the probe times out (default: 1) period_seconds: # how often (in seconds) to perform the probe (default: 10) success_threshold: # minimum consecutive successes for the probe to be considered successful after having failed (default: 1) failure_threshold: # minimum consecutive failures for the probe to be considered failed after having succeeded (default: 3) liveness_probe: # periodic probe of container liveness; container will be restarted if the probe fails (optional) http_get: # specifies an http endpoint which must respond with status code 200 (only one of http_get, tcp_socket, and exec may be specified) port: # the port to access on the container (required) path: # the path to access on the HTTP server (default: /) tcp_socket: # specifies a port which must be ready to receive traffic (only one of http_get, tcp_socket, and exec may be specified) port: # the port to access on the container (required) exec: # specifies a command to run which must exit with code 0 (only one of http_get, tcp_socket, and exec may be specified) command: # the command to execute inside the container, which is exec'd (not run inside a shell); the working directory is root ('/') in the container's filesystem (required) initial_delay_seconds: # number of seconds after the container has started before the probe is initiated (default: 0) timeout_seconds: # number of seconds until the probe times out (default: 1) period_seconds: # how often (in seconds) to perform the probe (default: 10) success_threshold: # minimum consecutive successes for the probe to be considered successful after having failed (default: 1) failure_threshold: # minimum consecutive failures for the probe to be considered failed after having succeeded (default: 3) node_groups: # a list of node groups on which this API can run (default: all node groups are eligible) networking: # networking configuration (default: see below) endpoint: # endpoint for the API (default: ) ``` ================================================ FILE: docs/workloads/batch/containers.md ================================================ # Containers ## Handling requests In order to receive batches in your Batch API, one of your containers must run a web server which is listening for HTTP requests on the port which is configured in the `pod.port` field of your [API configuration](configuration.md) (default: 8080). Batches will be sent to your web server via HTTP POST requests to the root path (`/`). The payload will be a JSON-encoded array representing one batch, and the `Content-Type` header will be set to "application/json". In addition, the job's ID will be passed in via the "X-Cortex-Job-ID" header. Your web server must respond with status code 200 for the batch to be marked as succeeded (the response body will be ignored). Once all batches have been processed, one of your workers will receive an HTTP POST request to `/on-job-complete`. It is not necessary for your web server to handle requests to `/on-job-complete` (404 errors will be ignored). ## Job specification If you need access to any parameters in the job submission (e.g. `config`), the entire job specification is available at `/cortex/spec/job.json` in your API containers' filesystems. ## Readiness checks It is often important to implement a readiness check for your API. By default, as soon as your web server has bound to the port, it will start receiving batches. In some cases, the web server may start listening on the port before its workers are ready to handle traffic (e.g. `tiangolo/uvicorn-gunicorn-fastapi` behaves this way). Readiness checks ensure that traffic is not sent into your web server before it's ready to handle them. There are two types of readiness checks which are supported: `http_get` and `tcp_socket` (see [API configuration](configuration.md) for usage instructions). A simple and often effective approach is to add a route to your web server (e.g. `/healthz`) which responds with status code 200, and configure your readiness probe accordingly: ```yaml readiness_probe: http_get: port: 8080 path: /healthz ``` ## Multiple containers Your API pod can contain multiple containers, only one of which can be listening for requests on the target port (it can be any of the containers). The `/mnt` directory is mounted to each container's filesystem, and is shared across all containers. ## Resource requests Each container in the pod requests its own amount of CPU, memory, GPU, and Inferentia resources. In addition, Cortex's dequeuer sidecar container (which is automatically added to the pod) requests 100m CPU and 100Mi memory. ## Observability See docs for [logging](../../clusters/observability/logging.md), [metrics](../../clusters/observability/metrics.md), and [alerting](../../clusters/observability/metrics.md). ## Using the Cortex CLI or client It is possible to use the Cortex CLI or client to interact with your cluster's APIs from within your API containers. All containers will have a CLI configuration file present at `/cortex/client/cli.yaml`, which is configured to connect to the cluster. In addition, the `CORTEX_CLI_CONFIG_DIR` environment variable is set to `/cortex/client` by default. Therefore, no additional configuration is required to use the CLI or Python client (which can be instantiated via `cortex.client()`). Note: your Cortex CLI or client must match the version of your cluster (available in the `CORTEX_VERSION` environment variable). ## Chaining APIs It is possible to submit Batch jobs from any Cortex API within a Cortex cluster. Jobs can be submitted to `http://ingressgateway-operator.istio-system.svc.cluster.local/batch/`, where `` is the name of the Batch API you are making a request to. For example, if there is a Batch API named `hello-world` running in the cluster, you can make a request to it from a different API in Python by using: ```python import requests job_spec = { "workers": 1, "item_list": {"items": [...], "batch_size": 10}, "config": {"my_key": "my_value"}, } response = requests.post( "http://ingressgateway-operator.istio-system.svc.cluster.local/batch/hello-world", json=job_spec, ) ``` To make requests from your Batch API to a Realtime, Task, or Async API running within the cluster, see the "Chaining APIs" docs associated with the target workload type. ================================================ FILE: docs/workloads/batch/example.md ================================================ # BatchAPI ### Define an API ```python # main.py from fastapi import FastAPI from typing import List app = FastAPI() @app.post("/") def handle_batch(batch: List[int]): print(batch) @app.post("/on-job-complete") def on_job_complete(): print("done") ``` ### Create a `Dockerfile` ```Dockerfile FROM python:3.8-slim RUN pip install --no-cache-dir fastapi uvicorn COPY main.py / CMD uvicorn --host 0.0.0.0 --port 8080 main:app ``` ### Build an image ```bash docker build . -t hello-world ``` ### Run a container locally ```bash docker run -p 8080:8080 hello-world ``` ### Make a request ```bash curl -X POST -H "Content-Type: application/json" -d '[1,2,3,4]' localhost:8080 ``` ### Login to ECR ```bash aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin .dkr.ecr.us-east-1.amazonaws.com ``` ### Create a repository ```bash aws ecr create-repository --repository-name hello-world ``` ### Tag the image ```bash docker tag hello-world .dkr.ecr.us-east-1.amazonaws.com/hello-world ``` ### Push the image ```bash docker push .dkr.ecr.us-east-1.amazonaws.com/hello-world ``` ### Configure a Cortex deployment ```yaml # cortex.yaml - name: hello-world kind: BatchAPI pod: containers: - name: api image: .dkr.ecr.us-east-1.amazonaws.com/hello-world command: ["uvicorn", "--host", "0.0.0.0", "--port", "8080", "main:app"] ``` ### Create a Cortex deployment ```bash cortex deploy ``` ### Get the API endpoint ```bash cortex get hello-world ``` ### Make a request ```bash curl -X POST -H "Content-Type: application/json" -d '{"workers": 2, "item_list": {"items": [1,2,3,4], "batch_size": 2}}' http://***.amazonaws.com/hello-world ``` ### View the logs ```bash cortex logs hello-world ``` ================================================ FILE: docs/workloads/batch/jobs.md ================================================ # BatchAPI jobs ## Get the Batch API's endpoint ```bash cortex get ``` ## Submit a Job There are three options for providing the dataset for your job: 1. [Data in the request](#data-in-the-request) 1. [List S3 file paths](#s3-file-paths) 1. [Newline delimited JSON file(s) in S3](#newline-delimited-json-files-in-s3) ### Data in the request The input data for your job can be included directly in your job submission request by specifying an `item_list` in your json request payload. Each item can be any type (object, list, string, etc.) and is treated as a single sample. `item_list.batch_size` specifies how many items to include in a single batch. __Each batch must be smaller than 256 KiB, and the total request size must be less than 10 MiB.__ If you want to submit more data, explore the other job submission methods. Submitting data in the request can be useful in the following scenarios: * the request only has a few items * each item in the request is small (e.g. urls to images/videos) * you want to avoid using S3 as an intermediate storage layer ```yaml POST : { "workers": , # the number of workers to allocate for this job (required) "timeout": , # duration in seconds since the submission of a job before it is terminated (optional) "sqs_dead_letter_queue": { # specify a queue to redirect failed batches (optional) "arn": , # arn of dead letter queue e.g. arn:aws:sqs:us-west-2:123456789:failed.fifo "max_receive_count": # number of a times a batch is allowed to be handled by a worker before it is considered to be failed and transferred to the dead letter queue (must be >= 1) }, "item_list": { "items": [ # a list items that can be of any type (required) , ], "batch_size": , # the number of items per batch (the handle_batch() function is called once per batch) (required) } "config": { # arbitrary input for this specific job (optional) "string": } } RESPONSE: { "job_id": , "api_name": , "kind": "BatchAPI", "workers": , "config": {: }, "api_id": , "sqs_url": , "timeout": , "sqs_dead_letter_queue": { "arn": , "max_receive_count": }, "created_time": } ``` The entire job specification is written to `/cortex/spec/job.json` in the API containers. ### S3 file paths If your input data is a list of files such as images/videos in an S3 directory, you can define `file_path_lister` in your submission request payload. You can use `file_path_lister.s3_paths` to specify a list of files or prefixes, and `file_path_lister.includes` and/or `file_path_lister.excludes` to remove unwanted files. The S3 file paths will be aggregated into batches of size `file_path_lister.batch_size`. To learn more about fine-grained S3 file filtering see [filtering files](#filtering-files). __The total size of a batch must be less than 256 KiB.__ This submission pattern can be useful in the following scenarios: * you have a list of images/videos in an S3 directory * each S3 file represents a single sample or a small number of samples If a single S3 file contains a lot of samples/rows, try the next submission strategy. ```yaml POST : { "workers": , # the number of workers to allocate for this job (required) "timeout": , # duration in seconds since the submission of a job before it is terminated (optional) "sqs_dead_letter_queue": { # specify a queue to redirect failed batches (optional) "arn": , # arn of dead letter queue e.g. arn:aws:sqs:us-west-2:123456789:failed.fifo "max_receive_count": # number of a times a batch is allowed to be handled by a worker before it is considered to be failed and transferred to the dead letter queue (must be >= 1) }, "file_path_lister": { "s3_paths": [], # can be S3 prefixes or complete S3 paths (required) "includes": [], # glob patterns (optional) "excludes": [], # glob patterns (optional) "batch_size": , # the number of S3 file paths per batch (the handle_batch() function is called once per batch) (required) } "config": { # arbitrary input for this specific job (optional) "string": } } RESPONSE: { "job_id": , "api_name": , "kind": "BatchAPI", "workers": , "config": {: }, "api_id": , "sqs_url": , "timeout": , "sqs_dead_letter_queue": { "arn": , "max_receive_count": }, "created_time": } ``` The entire job specification is written to `/cortex/spec/job.json` in the API containers. ### Newline delimited JSON files in S3 If your input dataset is a newline delimited json file in an S3 directory (or a list of them), you can define `delimited_files` in your request payload to break up the contents of the file into batches of size `delimited_files.batch_size`. Upon receiving `delimited_files`, your Batch API will iterate through the `delimited_files.s3_paths` to generate the set of S3 files to process. You can use `delimited_files.includes` and `delimited_files.excludes` to filter out unwanted files. Each S3 file will be parsed as a newline delimited JSON file. Each line in the file should be a JSON object, which will be treated as a single sample. The S3 file will be broken down into batches of size `delimited_files.batch_size` and submitted to your workers. To learn more about fine-grained S3 file filtering see [filtering files](#filtering-files). __The total size of a batch must be less than 256 KiB.__ This submission pattern is useful in the following scenarios: * one or more S3 files contains a large number of samples and must be broken down into batches ```yaml POST : { "workers": , # the number of workers to allocate for this job (required) "timeout": , # duration in seconds since the submission of a job before it is terminated (optional) "sqs_dead_letter_queue": { # specify a queue to redirect failed batches (optional) "arn": , # arn of dead letter queue e.g. arn:aws:sqs:us-west-2:123456789:failed.fifo "max_receive_count": # number of a times a batch is allowed to be handled by a worker before it is considered to be failed and transferred to the dead letter queue (must be >= 1) }, "delimited_files": { "s3_paths": [], # can be S3 prefixes or complete S3 paths (required) "includes": [], # glob patterns (optional) "excludes": [], # glob patterns (optional) "batch_size": , # the number of json objects per batch (the handle_batch() function is called once per batch) (required) } "config": { # arbitrary input for this specific job (optional) "string": } } RESPONSE: { "job_id": , "api_name": , "kind": "BatchAPI", "workers": , "config": {: }, "api_id": , "sqs_url": , "timeout": , "sqs_dead_letter_queue": { "arn": , "max_receive_count": }, "created_time": } ``` The entire job specification is written to `/cortex/spec/job.json` in the API containers. ## Get a job's status ```bash cortex get ``` Or make a GET request to `?jobID=`: ```yaml GET ?jobID=: RESPONSE: { "job_status": { "job_id": , "api_name": , "kind": "BatchAPI", "workers": , "config": {: }, "api_id": , "sqs_url": , "status": , "batches_in_queue": # number of batches remaining in the queue "worker_counts": { # worker counts are only available while a job is running "pending": , # number of workers that are waiting for compute resources to be provisioned "initializing": , # number of workers that are initializing "running": , # number of workers that are actively working on batches from the queue "succeeded": , # number of workers that have completed after verifying that the queue is empty "failed": , # number of workers that have failed "stalled": , # number of workers that have been stuck in pending for more than 10 minutes }, "created_time": "start_time": "end_time": (optional) }, "endpoint": "api_spec": { ... }, "metrics": { "succeeded": # number of succeeded batches "failed": int # number of failed attempts "avg_time_per_batch": (optional) # average time spent working on a batch (only considers successful attempts) } } ``` ## Stop a job ```bash cortex delete ``` Or make a DELETE request to `?jobID=`: ```yaml DELETE ?jobID=: RESPONSE: {"message":"stopped job "} ``` ## Additional Information ### Filtering files When submitting a job using `delimited_files` or `file_path_lister`, you can use `s3_paths` in conjunction with `includes` and `excludes` to precisely filter files. The Batch API will iterate through each S3 path in `s3_paths`. If the S3 path is a prefix, it iterates through each file in that prefix. For each file, if `includes` is non-empty, it will discard the S3 path if the S3 file doesn't match any of the glob patterns provided in `includes`. After passing the `includes` filter (if specified), if the `excludes` is non-empty, it will discard the S3 path if the S3 files matches any of the glob patterns provided in `excludes`. If you aren't sure which files will be processed in your request, specify the `dryRun=true` query parameter in the job submission request to see the target list. Here are a few examples of filtering for a folder structure like this: ```text ├── s3://bucket └── images ├── img_1.png ├── img_2.jpg ├── img_3.jpg └── img_4.gif ``` Select all files ```yaml { "s3_paths": ["s3://bucket/images/"] } # or { "s3_paths": ["s3://bucket/images/img"] } # Would select the following files: # s3://bucket/images/img_1.png # s3://bucket/images/img_2.jpg # s3://bucket/images/img_3.jpg # s3://bucket/images/img_4.gif ``` Select specific files ```yaml { "s3_paths": [ "s3://bucket/images/img_1.png", "s3://bucket/images/img_2.jpg" ] } # Would select the following files: # s3://bucket/images/img_1.png # s3://bucket/images/img_2.jpg ``` Only select JPG files ```yaml { "s3_paths": ["s3://bucket/images/"], "includes": ["**.jpg"] } # Would select the following files: # s3://bucket/images/img_2.jpg # s3://bucket/images/img_3.jpg ``` Select all JPG files except one specific JPG file ```yaml { "s3_paths": ["s3://bucket/images/"], "includes": ["**.jpg"], "excludes": ["**_3.jpg"] } # Would select the file: # s3://bucket/images/img_2.jpg ``` Select all files except GIFs ```yaml { "s3_paths": ["s3://bucket/images/"], "excludes": ["**.gif"] } # Would select the files: # s3://bucket/images/img_1.png # s3://bucket/images/img_2.jpg # s3://bucket/images/img_3.jpg ``` ================================================ FILE: docs/workloads/batch/statuses.md ================================================ # Job statuses | Status | Meaning | | :--- | :--- | | enqueuing | Job is being split into batches and placed into a queue | | running | Workers are retrieving batches from the queue and running inference | | succeeded | Workers completed all items in the queue without any failures | | failed while enqueuing | Failure occurred while enqueuing; check job logs for more details | | completed with failures | Workers completed all items in the queue but some of the batches weren't processed successfully and raised exceptions; check job logs for more details | | worker error | One or more workers experienced an irrecoverable error, causing the job to fail; check job logs for more details | | out of memory | One or more workers ran out of memory, causing the job to fail; check job logs for more details | | timed out | Job was terminated after the specified timeout has elapsed | | stopped | Job was stopped by the user or the Batch API was deleted | ================================================ FILE: docs/workloads/realtime/autoscaling.md ================================================ # Autoscaling Cortex autoscales each API independently based on its configuration. ## Autoscaling replicas ### Relevant pod configuration In addition to the autoscaling configuration options (described below), there are two fields in the pod configuration which are relevant to replica autoscaling: **`max_concurrency`** (default: 1): The maximum number of requests that will be concurrently sent into the container by Cortex. If your web server is designed to handle multiple concurrent requests, increasing `max_concurrency` will increase the throughput of a replica (and result in fewer total replicas for a given load).
**`max_queue_length`** (default: 100): The maximum number of requests which will be queued by the replica (beyond `max_concurrency`) before requests are rejected with HTTP error code 503. For long-running APIs, decreasing `max_queue_length` and configuring the client to retry when it receives 503 responses will improve queue fairness accross replicas by preventing requests from sitting in long queues.
### Autoscaling configuration **`min_replicas`** (default: 1): The lower bound on how many replicas can be running for an API. Scale-to-zero is supported (experimental).
**`max_replicas`** (default: 100): The upper bound on how many replicas can be running for an API.
**`target_in_flight`** (default: `max_concurrency` in the pod configuration): This is the desired number of in-flight requests per replica, and is the metric which the autoscaler uses to make scaling decisions. The number of in-flight requests is simply how many requests have been sent to a replica and have not yet been responded to. Therefore, this number includes requests which are actively being processed as well as requests which are waiting in the replica's queue. The autoscaler uses this formula to determine the number of desired replicas: `desired replicas = sum(in-flight requests accross all replicas) / target_in_flight` For example, setting `target_in_flight` to `max_concurrency` (the default) causes the cluster to adjust the number of replicas so that on average, requests are immediately processed without waiting in a queue.
**`window`** (default: 60s): The time over which to average the API's in-flight requests (which is the sum of in-flight requests in each replica). The longer the window, the slower the autoscaler will react to changes in in-flight requests, since it is averaged over the `window`. An API's in-flight requests is calculated every 10 seconds, so `window` must be a multiple of 10 seconds.
**`downscale_stabilization_period`** (default: 5m): The API will not scale below the highest recommendation made during this period. Every 10 seconds, the autoscaler makes a recommendation based on all of the other configuration parameters described here. It will then take the max of the current recommendation and all recommendations made during the `downscale_stabilization_period`, and use that to determine the final number of replicas to scale to. Increasing this value will cause the cluster to react more slowly to decreased traffic, and will reduce thrashing.
**`upscale_stabilization_period`** (default: 1m): The API will not scale above the lowest recommendation made during this period. Every 10 seconds, the autoscaler makes a recommendation based on all of the other configuration parameters described here. It will then take the min of the current recommendation and all recommendations made during the `upscale_stabilization_period`, and use that to determine the final number of replicas to scale to. Increasing this value will cause the cluster to react more slowly to increased traffic, and will reduce thrashing.
**`max_downscale_factor`** (default: 0.75): The maximum factor by which to scale down the API on a single scaling event. For example, if `max_downscale_factor` is 0.5 and there are 10 running replicas, the autoscaler will not recommend fewer than 5 replicas. Increasing this number will allow the cluster to shrink more quickly in response to dramatic dips in traffic.
**`max_upscale_factor`** (default: 1.5): The maximum factor by which to scale up the API on a single scaling event. For example, if `max_upscale_factor` is 10 and there are 5 running replicas, the autoscaler will not recommend more than 50 replicas. Increasing this number will allow the cluster to grow more quickly in response to dramatic spikes in traffic.
**`downscale_tolerance`** (default: 0.05): Any recommendation falling within this factor below the current number of replicas will not trigger a scale down event. For example, if `downscale_tolerance` is 0.1 and there are 20 running replicas, a recommendation of 18 or 19 replicas will not be acted on, and the API will remain at 20 replicas. Increasing this value will prevent thrashing, but setting it too high will prevent the cluster from maintaining it's optimal size.
**`upscale_tolerance`** (default: 0.05): Any recommendation falling within this factor above the current number of replicas will not trigger a scale-up event. For example, if `upscale_tolerance` is 0.1 and there are 20 running replicas, a recommendation of 21 or 22 replicas will not be acted on, and the API will remain at 20 replicas. Increasing this value will prevent thrashing, but setting it too high will prevent the cluster from maintaining it's optimal size.
## Autoscaling instances Cortex spins up and down instances based on the aggregate resource requests of all APIs. The number of instances will be at least `min_instances` and no more than `max_instances` for each node group (configured during installation and modifiable via `cortex cluster configure`). ## Overprovisioning The default value for `target_in_flight` is `max_concurrency`, which behaves well in many situations (see above for an explanation of how `target_in_flight` affects autoscaling). However, if your application is sensitive to spikes in traffic or if creating new replicas takes too long (see below), you may find it helpful to maintain extra capacity to handle the increased traffic while new replicas are being created. This can be accomplished by setting `target_in_flight` to a lower value relative to the expected replica's concurrency. The smaller `target_in_flight` is, the more unused capacity your API will have, and the more room it will have to handle sudden increased load. The increased request rate will still trigger the autoscaler, and your API will stabilize again (maintaining the overprovisioned capacity). For example, if you've determined that each replica in your API can efficiently handle 2 concurrent requests, you would typically set `target_in_flight` to 2. In a scenario where your API is receiving 8 concurrent requests on average, the autoscaler would maintain 4 live replicas (8/2 = 4). If you wanted to overprovision by 25%, you could set `target_in_flight` to 1.6, causing the autoscaler maintain 5 live replicas (8/1.6 = 5). ## Autoscaling responsiveness Assuming that `window` and `upscale_stabilization_period` are set to their default values (1 minute), it could take up to 2 minutes of increased traffic before an extra replica is requested. As soon as the additional replica is requested, the replica request will be visible in the output of `cortex get`, but the replica won't yet be running. If an extra instance is required to schedule the newly requested replica, it could take a few minutes for AWS to provision the instance (depending on the instance type), plus a few minutes for the newly provisioned instance to download your api image and for the api to initialize. Keep these delays in mind when considering overprovisioning (see above) and when determining appropriate values for `window` and `upscale_stabilization_period`. If you want the autoscaler to react as quickly as possible, set `upscale_stabilization_period` and `window` to their minimum values (0s and 10s respectively). ================================================ FILE: docs/workloads/realtime/configuration.md ================================================ # Configuration ```yaml - name: # name of the API (required) kind: RealtimeAPI # must be "RealtimeAPI" for realtime APIs (required) pod: # pod configuration (required) port: # port to which requests will be sent (default: 8080; exported as $CORTEX_PORT) max_concurrency: # maximum number of requests that will be concurrently sent into the container (default: 1) max_queue_length: # maximum number of requests per replica which will be queued (beyond max_concurrency) before requests are rejected with error code 503 (default: 100) containers: # configurations for the containers to run (at least one constainer must be provided) - name: # name of the container (required) image: # docker image to use for the container (required) command: # entrypoint (not executed within a shell); env vars can be used with e.g. $(CORTEX_PORT) (default: the docker image's ENTRYPOINT) args: # arguments to the entrypoint; env vars can be used with e.g. $(CORTEX_PORT) (default: the docker image's CMD) env: # dictionary of environment variables to set in the container (optional) compute: # compute resource requests (default: see below) cpu: # CPU request for the container; one unit of CPU corresponds to one virtual CPU; fractional requests are allowed, and can be specified as a floating point number or via the "m" suffix (default: 200m) gpu: # GPU request for the container; one unit of GPU corresponds to one virtual GPU (default: 0) inf: # Inferentia request for the container; one unit of inf corresponds to one virtual Inferentia chip (default: 0) mem: # memory request for the container; one unit of memory is one byte and can be expressed as an integer or by using one of these suffixes: K, M, G, T (or their power-of two counterparts: Ki, Mi, Gi, Ti) (default: Null) shm: # size of shared memory (/dev/shm) for sharing data between multiple processes, e.g. 64Mi or 1Gi (default: Null) readiness_probe: # periodic probe of container readiness; traffic will not be sent into the pod unless all containers' readiness probes are succeeding (optional) http_get: # specifies an http endpoint which must respond with status code 200 (only one of http_get, tcp_socket, and exec may be specified) port: # the port to access on the container (required) path: # the path to access on the HTTP server (default: /) tcp_socket: # specifies a port which must be ready to receive traffic (only one of http_get, tcp_socket, and exec may be specified) port: # the port to access on the container (required) exec: # specifies a command to run which must exit with code 0 (only one of http_get, tcp_socket, and exec may be specified) command: # the command to execute inside the container, which is exec'd (not run inside a shell); the working directory is root ('/') in the container's filesystem (required) initial_delay_seconds: # number of seconds after the container has started before the probe is initiated (default: 0) timeout_seconds: # number of seconds until the probe times out (default: 1) period_seconds: # how often (in seconds) to perform the probe (default: 10) success_threshold: # minimum consecutive successes for the probe to be considered successful after having failed (default: 1) failure_threshold: # minimum consecutive failures for the probe to be considered failed after having succeeded (default: 3) liveness_probe: # periodic probe of container liveness; container will be restarted if the probe fails (optional) http_get: # specifies an http endpoint which must respond with status code 200 (only one of http_get, tcp_socket, and exec may be specified) port: # the port to access on the container (required) path: # the path to access on the HTTP server (default: /) tcp_socket: # specifies a port which must be ready to receive traffic (only one of http_get, tcp_socket, and exec may be specified) port: # the port to access on the container (required) exec: # specifies a command to run which must exit with code 0 (only one of http_get, tcp_socket, and exec may be specified) command: # the command to execute inside the container, which is exec'd (not run inside a shell); the working directory is root ('/') in the container's filesystem (required) initial_delay_seconds: # number of seconds after the container has started before the probe is initiated (default: 0) timeout_seconds: # number of seconds until the probe times out (default: 1) period_seconds: # how often (in seconds) to perform the probe (default: 10) success_threshold: # minimum consecutive successes for the probe to be considered successful after having failed (default: 1) failure_threshold: # minimum consecutive failures for the probe to be considered failed after having succeeded (default: 3) pre_stop: # a pre-stop lifecycle hook for the container; will be executed before container termination (optional) http_get: # specifies an http endpoint to send a request to (only one of http_get, tcp_socket, and exec may be specified) port: # the port to access on the container (required) path: # the path to access on the HTTP server (default: /) exec: # specifies a command to run (only one of http_get, tcp_socket, and exec may be specified) command: # the command to execute inside the container, which is exec'd (not run inside a shell); the working directory is root ('/') in the container's filesystem (required) autoscaling: # autoscaling configuration (default: see below) min_replicas: # minimum number of replicas (default: 1) max_replicas: # maximum number of replicas (default: 100) init_replicas: # initial number of replicas (default: ) target_in_flight: # desired number of in-flight requests per replica (including requests actively being processed as well as queued), which the autoscaler tries to maintain (default: ) window: # duration over which to average the API's in-flight requests per replica (default: 60s) downscale_stabilization_period: # the API will not scale below the highest recommendation made during this period (default: 5m) upscale_stabilization_period: # the API will not scale above the lowest recommendation made during this period (default: 1m) max_downscale_factor: # maximum factor by which to scale down the API on a single scaling event (default: 0.75) max_upscale_factor: # maximum factor by which to scale up the API on a single scaling event (default: 1.5) downscale_tolerance: # any recommendation falling within this factor below the current number of replicas will not trigger a scale down event (default: 0.05) upscale_tolerance: # any recommendation falling within this factor above the current number of replicas will not trigger a scale-up event (default: 0.05) node_groups: # a list of node groups on which this API can run (default: all node groups are eligible) update_strategy: # deployment strategy to use when replacing existing replicas with new ones (default: see below) max_surge: # maximum number of replicas that can be scheduled above the desired number of replicas during an update; can be an absolute number, e.g. 5, or a percentage of desired replicas, e.g. 10% (default: 25%) (set to 0 to disable rolling updates) max_unavailable: # maximum number of replicas that can be unavailable during an update; can be an absolute number, e.g. 5, or a percentage of desired replicas, e.g. 10% (default: 25%) networking: # networking configuration (default: see below) endpoint: # endpoint for the API (default: ) ``` ================================================ FILE: docs/workloads/realtime/containers.md ================================================ # Containers ## Handling requests In order to handle requests to your Realtime API, one of your containers must run a web server which is listening for HTTP requests on the port which is configured in the `pod.port` field of your [API configuration](configuration.md) (default: 8080). Subpaths are supported; for example, if your API is named `hello-world`, a request to `/hello-world` will be routed to the root (`/`) of your web server, and a request to `/hello-world/subpatch` will be routed to `/subpath` on your web server. ## Readiness checks It is often important to implement a readiness check for your API. By default, as soon as your web server has bound to the port, it will start receiving traffic. In some cases, the web server may start listening on the port before its workers are ready to handle traffic (e.g. `tiangolo/uvicorn-gunicorn-fastapi` behaves this way). Readiness checks ensure that traffic is not sent into your web server before it's ready to handle them. There are three types of readiness checks which are supported: `http_get`, `tcp_socket`, and `exec` (see [API configuration](configuration.md) for usage instructions). A simple and often effective approach is to add a route to your web server (e.g. `/healthz`) which responds with status code 200, and configure your readiness probe accordingly: ```yaml readiness_probe: http_get: port: 8080 path: /healthz ``` ## Multiple containers Your API pod can contain multiple containers, only one of which can be listening for requests on the target port (it can be any of the containers). The `/mnt` directory is mounted to each container's file system, and is shared across all containers. ## Resource requests Each container in the pod requests its own amount of CPU, memory, GPU, and Inferentia resources. In addition, Cortex's proxy sidecar container (which is automatically added to the pod) requests 100m CPU and 100Mi memory. ## Observability See docs for [logging](../../clusters/observability/logging.md), [metrics](../../clusters/observability/metrics.md), and [alerting](../../clusters/observability/metrics.md). ## Using the Cortex CLI or client It is possible to use the Cortex CLI or client to interact with your cluster's APIs from within your API containers. All containers will have a CLI configuration file present at `/cortex/client/cli.yaml`, which is configured to connect to the cluster. In addition, the `CORTEX_CLI_CONFIG_DIR` environment variable is set to `/cortex/client` by default. Therefore, no additional configuration is required to use the CLI or Python client (which can be instantiated via `cortex.client()`). Note: your Cortex CLI or client must match the version of your cluster (available in the `CORTEX_VERSION` environment variable). ## Chaining APIs It is possible to make requests to Realtime APIs from any Cortex API within a Cortex cluster. Requests can be made to `http://ingressgateway-apis.istio-system.svc.cluster.local/`, where `` is the name of the Realtime API you are making a request to. For example, if there is a Realtime API named `hello-world` running in the cluster, you can make a request to it from a different API in Python by using: ```python import requests response = requests.post( "http://ingressgateway-apis.istio-system.svc.cluster.local/hello-world", json={"text": "hello world"}, ) ``` Note that if the API making the request is a Realtime API or Async API, its autoscaling configuration (i.e. `target_in_flight`) should be modified with the understanding that requests will be considered "in-flight" in the first API as the request is being fulfilled by the second API. To make requests from your Realtime API to a Batch, Async, or Task API running within the cluster, see the "Chaining APIs" docs associated with the target workload type. ================================================ FILE: docs/workloads/realtime/example.md ================================================ # RealtimeAPI ### Define an API ```python # main.py from fastapi import FastAPI from pydantic import BaseModel app = FastAPI() class Data(BaseModel): msg: str @app.post("/") def handle_post(data: Data): return data ``` ### Create a `Dockerfile` ```Dockerfile FROM python:3.8-slim RUN pip install --no-cache-dir fastapi uvicorn COPY main.py / CMD uvicorn --host 0.0.0.0 --port 8080 main:app ``` ### Build an image ```bash docker build . -t hello-world ``` ### Run a container locally ```bash docker run -p 8080:8080 hello-world ``` ### Make a request ```bash curl -X POST -H "Content-Type: application/json" -d '{"msg": "hello world"}' localhost:8080 ``` ### Login to ECR ```bash aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin .dkr.ecr.us-east-1.amazonaws.com ``` ### Create a repository ```bash aws ecr create-repository --repository-name hello-world ``` ### Tag the image ```bash docker tag hello-world .dkr.ecr.us-east-1.amazonaws.com/hello-world ``` ### Push the image ```bash docker push .dkr.ecr.us-east-1.amazonaws.com/hello-world ``` ### Configure a Cortex deployment ```yaml # cortex.yaml - name: hello-world kind: RealtimeAPI pod: containers: - name: api image: .dkr.ecr.us-east-1.amazonaws.com/hello-world ``` ### Create a Cortex deployment ```bash cortex deploy ``` ### Wait for the API to be ready ```bash cortex get --watch ``` ### Get the API endpoint ```bash cortex get hello-world ``` ### Make a request ```bash curl -X POST -H "Content-Type: application/json" -d '{"msg": "hello world"}' http://***.amazonaws.com/hello-world ``` ================================================ FILE: docs/workloads/realtime/metrics.md ================================================ # Metrics The `cortex get` and `cortex get API_NAME` commands display the request time (averaged over the past 2 weeks) and response code counts (summed over the past 2 weeks) for your APIs: ```bash cortex get env api status up-to-date requested last update avg request 2XX cortex iris-classifier live 1 1 17m 24ms 1223 cortex text-generator live 1 1 8m 180ms 433 cortex image-classifier-resnet50 live 2 2 1h 32ms 1121126 ``` The `cortex get API_NAME` command also provides a link to a Grafana dashboard: ![dashboard](https://user-images.githubusercontent.com/7456627/107253455-9c6b7b80-6a36-11eb-8600-f36a7bab6d3b.png) --- ## Metrics in the dashboard | Panel | Description | Note | |-------------------|------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------| | Request Rate | Request rate, computed over every minute, of an API | | | In Flight Request | Active in-flight requests for an API. | In-flight requests are recorded every 10 seconds, which will correspond to the minimum resolution. | | Active Replicas | Active replicas for an API | | | 2XX Responses | Request rate, computed over a minute, for responses with status code 2XX of an API | | | 4XX Responses | Request rate, computed over a minute, for responses with status code 4XX of an API | | | 5XX Responses | Request rate, computed over a minute, for responses with status code 5XX of an API | | | p99 Latency | 99th percentile latency, computed over a minute, for an API | Value might not be accurate because the histogram buckets are not dynamically set. | | p90 Latency | 90th percentile latency, computed over a minute, for an API | Value might not be accurate because the histogram buckets are not dynamically set. | | p50 Latency | 50th percentile latency, computed over a minute, for an API | Value might not be accurate because the histogram buckets are not dynamically set. | | Average Latency | Average latency, computed over a minute, for an API | | ================================================ FILE: docs/workloads/realtime/realtime.md ================================================ # Realtime Realtime APIs respond to requests synchronously and autoscale based on in-flight request volumes. Realtime APIs are a good fit for users who want to run stateless containers as a scalable microservice (for example, deploying machine learning models as APIs). **Key features** * respond to requests synchronously * autoscale based on request volume * avoid cold starts * scale to zero * perform rolling updates * automatically recover from failures and spot instance termination * perform A/B tests and canary deployments ## How it works When you deploy a Realtime API, Cortex initializes a pool of worker pods and attaches a proxy sidecar to each of the pods. The proxy is responsible for receiving incoming requests, queueing them (if necessary), and forwarding them to your pod when it is ready. Autoscaling is based on aggregate in-flight request volume, which is published by the proxy sidecars. ![](https://user-images.githubusercontent.com/808475/146854245-ed0fc153-d083-47d8-a7e2-ac5beb114ee6.png) ================================================ FILE: docs/workloads/realtime/statuses.md ================================================ # Replica states The replica states of an API can be inspected by running `cortex describe `. Here are the possible states for each replica in an API: | State | Meaning | |:---|:---| | Ready | Replica is running and it has passed the readiness checks | | ReadyOutOfDate | Replica is running and it has passed the readiness checks (for an out-of-date replica) | | NotReady | Replica is running but it's not passing the readiness checks; make sure the server is listening on the designed port of the API | | Pending | Replica is in a pending state (waiting to get scheduled onto a node) | | Creating | Replica is in the process of having its containers created | | ErrImagePull | Replica was not created because one of the specified Docker images was inaccessible at runtime; check that your API's docker images exist and are accessible via your cluster's AWS credentials | | Failed | Replica couldn't start due to an error; run `cortex logs ` to view the logs | | Killed | Replica has had one of its containers killed | | KilledOOM | Replica was terminated due to excessive memory usage; try allocating more memory to the API and re-deploy | | Stalled | Replica has been in a pending state for more than 15 minutes; see [troubleshooting](../realtime/troubleshooting.md) | | Terminating | Replica is currently in the process of being terminated | | Unknown | Replica is in an unknown state | ================================================ FILE: docs/workloads/realtime/traffic-splitter.md ================================================ # Traffic Splitter Traffic Splitters can be used to expose multiple RealtimeAPIs as a single endpoint for A/B tests, multi-armed bandits, or canary deployments. ## Configuration ```yaml - name: # name of the traffic splitter (required) kind: TrafficSplitter # must be "TrafficSplitter" for traffic splitters (required) networking: # networking configuration (default: see below) endpoint: # the endpoint for the traffic splitter (default: ) apis: # list of Realtime APIs to target (required) - name: # name of a Realtime API that is already running or is included in the same configuration file (required) weight: # percentage of traffic to route to the Realtime API (all non-shadow weights must sum to 100) (required) shadow: # duplicate incoming traffic and send fire-and-forget to this api (only one shadow per traffic splitter) (default: false) ``` ## Example This example showcases Cortex's Python client, but these steps can also be performed by using the Cortex CLI. ### Deploy a traffic splitter ```python traffic_splitter_spec = { "name": "sentiment-analyzer", "kind": "TrafficSplitter", "apis": [ {"name": "sentiment-analyzer-a", "weight": 50}, {"name": "sentiment-analyzer-b", "weight": 50}, ], } cx.deploy(traffic_splitter_spec) ``` ### Update the weights ```python new_traffic_splitter_spec = { "name": "sentiment-analyzer", "kind": "TrafficSplitter", "apis": [ {"name": "sentiment-analyzer-a", "weight": 1}, {"name": "sentiment-analyzer-b", "weight": 99}, ], } cx.deploy(new_traffic_splitter_spec) ``` ### Update the target APIs ```python new_traffic_splitter_spec = { "name": "sentiment-analyzer", "kind": "TrafficSplitter", "apis": [ {"name": "sentiment-analyzer-b", "weight": 50}, {"name": "sentiment-analyzer-c", "weight": 50}, ], } cx.deploy(new_traffic_splitter_spec) ``` ================================================ FILE: docs/workloads/realtime/troubleshooting.md ================================================ # Troubleshooting ## 503 error responses from API requests When making requests to your API, it's possible to get a `no healthy upstream` error message (with HTTP status code `503`). This means that there are currently no live replicas running for your API. This could happen for a few reasons: 1. It's possible that your API is simply not ready yet. You can check the number of ready replicas on your API with `cortex get API_NAME`, and inspect the logs in CloudWatch with the help of `cortex logs API_NAME`. 1. Your API may have errored during initialization or while responding to a previous request. `cortex describe API_NAME` will show the number of replicas that have failed to start on your API, and you can view the logs for all replicas by visiting the CloudWatch Insights URL from `cortex logs API_NAME`. If you are using API Gateway in front of your API endpoints, it is also possible to receive a `{"message":"Service Unavailable"}` error message (with HTTP status code `503`) after 29 seconds if your request exceeds API Gateway's 29 second timeout. If this is the case, you can either modify your code to take less time, run on faster hardware (e.g. GPUs), or don't use API Gateway (there is no timeout when using the API's endpoint directly). ## API is stuck updating If your API has pods stuck in the "pending" or "stalled" states (which is displayed when running `cortex describe API_NAME`), there are a few possible causes. Here are some things to check: ### Inspect API logs in CloudWatch Use `cortex logs API_NAME` for a URL to view logs for your API in CloudWatch. In addition to output from your containers, you will find logs from other parts of the Cortex infrastructure that may help your troubleshooting. ### Check `max_instances` for your cluster When you created your Cortex cluster, you configured `max_instances` for each node group that you specified (via the cluster configuration file, e.g. `cluster.yaml`). If your cluster already has `min_instances` running instances for a given node group, additional instances cannot be created and APIs may not be able to deploy, scale, or update. You can check the current value of `max_instances` for the selected node group by running `cortex cluster info --config cluster.yaml` (or `cortex cluster info --name --region ` if you have the name and region of the cluster). Once you have the name and region of the cluster, you can update the `max_instances` field by following the [instructions](../../clusters/management/update.md) to update an existing cluster. ## Check your AWS auto scaling group activity history In most cases when AWS is unable to provision additional instances, the reason will be logged in the auto scaling group's activity history. Here is how you can check these logs: 1. Log in to the AWS console and go to the EC2 service page 2. Click "Auto Scaling Groups" on the bottom of the side panel on the left 3. Select one of the "worker" autoscaling groups for your cluster (there may be two) 4. Click the "Activity" tab at the bottom half of the screen (it may also be called "Activity History" depending on which AWS console UI you're using) 5. Scroll down (if necessary) and inspect the activity history, looking for any errors and their causes 6. Repeat steps 3-5 for the other worker autoscaling group (if applicable) Here is how it looks on the new console UI: ![new ui](https://user-images.githubusercontent.com/808475/78153371-852d2c00-742a-11ea-9bde-dbad5c603f8f.png) On the old UI: ![old ui](https://user-images.githubusercontent.com/808475/78153350-7e9eb480-742a-11ea-9221-1f6559db45fd.png) The most common reason AWS is unable to provision instances is that you have reached your instance limit. There is an instance limit associated with your AWS account for each instance family in each region, for on-demand and for spot instances. You can check your current limit and request an increase [here](https://console.aws.amazon.com/servicequotas/home?#!/services/ec2/quotas) (set the region in the upper right corner to your desired region, type "on-demand" or "spot" in the search bar, and click on the quota that matches your instance type). Note that the quota values indicate the number of vCPUs available, not the number of instances; different instances have a different numbers of vCPUs, which can be seen [here](https://aws.amazon.com/ec2/instance-types). If you're using spot instances for your node group, it is also possible that AWS has run out of spot instances for your requested instance type and region. To address this, you can try adding additional alternative instance types in `instance_distribution` or changing the cluster's region to one that has a higher availability. ### Disabling rolling updates By default, cortex performs rolling updates on all APIs. This is to ensure that traffic can continue to be served during updates, and that there is no downtime if there's an error in the new version. However, this can lead to APIs getting stuck in the "updating" state when the cluster is unable to increase its instance count (e.g. for one of the reasons above). Here is an example: You set `max_instances` to 1, or your AWS account limits you to a single `g4dn.xlarge` instance (i.e. your G instance vCPU limit is 4). You have an API running which requested 1 GPU. When you update your API via `cortex deploy`, Cortex attempts to deploy the updated version, and will only take down the old version once the new one is running. In this case, since there is no GPU available on the single running instance (it's taken by the old version of your API), the new version of your API requests a new instance to run on. Normally this will be ok (it might just take a few minutes since a new instance has to spin up): the new instance will become live, the new API replica will run on it, once it starts up successfully the old replica will be terminated, and eventually the old instance will spin down. In this case, however, the new version gets stuck because the second instance cannot be created, and the first instance cannot be freed up until the new version is running. If you're running in a development environment, this rolling update behavior can be undesirable. You can disable rolling updates for your API in your API configuration: set `max_surge` to 0 in the `update_strategy` section, E.g.: ```yaml - name: hello-world kind: RealtimeAPI # ... update_strategy: max_surge: 0 ``` ================================================ FILE: docs/workloads/task/configuration.md ================================================ # Configuration ```yaml - name: # name of the API (required) kind: TaskAPI # must be "TaskAPI" for task APIs (required) pod: # pod configuration (required) containers: # configurations for the containers to run (at least one constainer must be provided) - name: # name of the container (required) image: # docker image to use for the container (required) command: # entrypoint (not executed within a shell); env vars can be used with e.g. $(CORTEX_PORT) (required) args: # arguments to the entrypoint; env vars can be used with e.g. $(CORTEX_PORT) (default: no args) env: # dictionary of environment variables to set in the container (optional) compute: # compute resource requests (default: see below) cpu: # CPU request for the container; one unit of CPU corresponds to one virtual CPU; fractional requests are allowed, and can be specified as a floating point number or via the "m" suffix (default: 200m) gpu: # GPU request for the container; one unit of GPU corresponds to one virtual GPU (default: 0) inf: # Inferentia request for the container; one unit of inf corresponds to one virtual Inferentia chip (default: 0) mem: # memory request for the container; one unit of memory is one byte and can be expressed as an integer or by using one of these suffixes: K, M, G, T (or their power-of two counterparts: Ki, Mi, Gi, Ti) (default: Null) shm: # size of shared memory (/dev/shm) for sharing data between multiple processes, e.g. 64Mi or 1Gi (default: Null) liveness_probe: # periodic probe of container liveness; container will be restarted if the probe fails (optional) http_get: # specifies an http endpoint which must respond with status code 200 (only one of http_get, tcp_socket, and exec may be specified) port: # the port to access on the container (required) path: # the path to access on the HTTP server (default: /) tcp_socket: # specifies a port which must be ready to receive traffic (only one of http_get, tcp_socket, and exec may be specified) port: # the port to access on the container (required) exec: # specifies a command to run which must exit with code 0 (only one of http_get, tcp_socket, and exec may be specified) command: # the command to execute inside the container, which is exec'd (not run inside a shell); the working directory is root ('/') in the container's filesystem (required) initial_delay_seconds: # number of seconds after the container has started before the probe is initiated (default: 0) timeout_seconds: # number of seconds until the probe times out (default: 1) period_seconds: # how often (in seconds) to perform the probe (default: 10) success_threshold: # minimum consecutive successes for the probe to be considered successful after having failed (default: 1) failure_threshold: # minimum consecutive failures for the probe to be considered failed after having succeeded (default: 3) node_groups: # a list of node groups on which this API can run (default: all node groups are eligible) networking: # networking configuration (default: see below) endpoint: # endpoint for the API (default: ) ``` ================================================ FILE: docs/workloads/task/containers.md ================================================ # Containers ## Job specification If you need access to any parameters in the job submission (e.g. `config`), the entire job specification is available at `/cortex/spec/job.json` in your API containers' filesystems. ## Multiple containers Your Task's pod can contain multiple containers. The `/mnt` directory is mounted to each container's filesystem, and is shared across all containers. ## Observability See docs for [logging](../../clusters/observability/logging.md), [metrics](../../clusters/observability/metrics.md), and [alerting](../../clusters/observability/metrics.md). ## Using the Cortex CLI or client It is possible to use the Cortex CLI or client to interact with your cluster's APIs from within your API containers. All containers will have a CLI configuration file present at `/cortex/client/cli.yaml`, which is configured to connect to the cluster. In addition, the `CORTEX_CLI_CONFIG_DIR` environment variable is set to `/cortex/client` by default. Therefore, no additional configuration is required to use the CLI or Python client (which can be instantiated via `cortex.client()`). Note: your Cortex CLI or client must match the version of your cluster (available in the `CORTEX_VERSION` environment variable). ## Chaining APIs It is possible to submit Task jobs from any Cortex API within a Cortex cluster. Jobs can be submitted to `http://ingressgateway-operator.istio-system.svc.cluster.local/tasks/`, where `` is the name of the Task API you are making a request to. For example, if there is a Task API named `hello-world` running in the cluster, you can make a request to it from a different API in Python by using: ```python import requests response = requests.post( "http://ingressgateway-operator.istio-system.svc.cluster.local/tasks/hello-world", json={"config": {"my_key": "my_value"}}, ) ``` To make requests from your Task API to a Realtime, Batch, or Async API running within the cluster, see the "Chaining APIs" docs associated with the target workload type. ================================================ FILE: docs/workloads/task/example.md ================================================ # TaskAPI ### Define an API ```python # main.py print("hello world") ``` ### Create a `Dockerfile` ```Dockerfile FROM python:3.8-slim COPY main.py / CMD exec python main.py ``` ### Build an image ```bash docker build . -t hello-world ``` ### Run a container locally ```bash docker run -it --rm hello-world ``` ### Login to ECR ```bash aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin .dkr.ecr.us-east-1.amazonaws.com ``` ### Create a repository ```bash aws ecr create-repository --repository-name hello-world ``` ### Tag the image ```bash docker tag hello-world .dkr.ecr.us-east-1.amazonaws.com/hello-world ``` ### Push the image ```bash docker push .dkr.ecr.us-east-1.amazonaws.com/hello-world ``` ### Configure a Cortex deployment ```yaml # cortex.yaml - name: hello-world kind: TaskAPI pod: containers: - name: api image: .dkr.ecr.us-east-1.amazonaws.com/hello-world command: ["python", "main.py"] ``` ### Create a Cortex deployment ```bash cortex deploy ``` ### Get the API endpoint ```bash cortex get hello-world ``` ### Make a request ```bash curl -X POST -H "Content-Type: application/json" -d '{}' http://***.amazonaws.com/hello-world ``` ### View the logs ```bash cortex logs hello-world ``` ================================================ FILE: docs/workloads/task/jobs.md ================================================ # TaskAPI jobs ## Get the Task API's endpoint ```bash cortex get ``` ## Submit a Job ```yaml POST : { "timeout": , # duration in seconds since the submission of a job before it is terminated (optional) "config": { # arbitrary input for this specific job (optional) "string": } } RESPONSE: { "job_id": , "api_name": , "kind": "TaskAPI", "workers": 1, "config": {: }, "api_id": , "timeout": , "created_time": } ``` The entire job specification is written to `/cortex/spec/job.json` in the API containers. ## Get a job's status ```bash cortex get ``` Or make a GET request to `?jobID=`: ```yaml GET ?jobID=: RESPONSE: { "job_status": { "job_id": , "api_name": , "kind": "TaskAPI", "workers": 1, "config": {: }, "api_id": , "status": , "created_time": "start_time": "end_time": (optional) }, "endpoint": "api_spec": { ... } } ``` ## Stop a job ```bash cortex delete ``` Or make a DELETE request to `?jobID=`: ```yaml DELETE ?jobID=: RESPONSE: {"message":"stopped job "} ``` ================================================ FILE: docs/workloads/task/statuses.md ================================================ # Job statuses | Status | Meaning | | :--- | :--- | | running | Task is running | | succeeded | Task has finished without errors | | worker error | The task has experienced an irrecoverable error, causing the job to fail; check job logs for more details | | out of memory | The task has ran out of memory, causing the job to fail; check job logs for more details | | timed out | Job was terminated after the specified timeout has elapsed | | stopped | Job was stopped by the user or the Task API was deleted | ================================================ FILE: docs/workloads/task/task.md ================================================ # Task Task APIs provide a lambda-style execution of containers. They are useful for running your containers on demand. Task APIs are a good fit when you need to trigger container execution via an HTTP request. They can be used to run tasks (e.g. training models), and can be configured as task runners for orchestrators (such as airflow). **Key Features** * run containers on-demand * scale to 0 (when there are no tasks) * automatically recover from failures and spot instance termination ## How it works When you deploy a Task API, an endpoint is created to receive task submissions. Upon submitting a Task, Cortex will respond with a Task ID and will asynchronously trigger the execution of a Task. Cortex will initialize a worker pod based on your API specification. After the worker pod runs to completion, the Task is marked as completed and the pod is terminated. You can make GET requests to the Task API endpoint to retrieve the status of the Task. ![](https://user-images.githubusercontent.com/808475/146854267-3785e8ee-4233-4473-a0db-37a5c5438fb4.png) ================================================ FILE: get-cli.sh ================================================ #!/bin/bash # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -e CORTEX_VERSION_BRANCH_STABLE=master CORTEX_INSTALL_PATH="${CORTEX_INSTALL_PATH:-/usr/local/bin/cortex}" # replace ~ with the home directory path CORTEX_INSTALL_PATH="${CORTEX_INSTALL_PATH/#\~/$HOME}" case "$OSTYPE" in darwin*) parsed_os="darwin" ;; linux*) parsed_os="linux" ;; *) echo -e "\nerror: only mac and linux are supported"; exit 1 ;; esac function main() { echo -e "\ndownloading cli (${CORTEX_INSTALL_PATH}) ...\n" cortex_sh_tmp_dir="$HOME/.cortex-sh-tmp" rm -rf $cortex_sh_tmp_dir && mkdir -p $cortex_sh_tmp_dir if command -v curl >/dev/null; then curl -s -w "" -o $cortex_sh_tmp_dir/cortex https://s3-us-west-2.amazonaws.com/get-cortex/$CORTEX_VERSION_BRANCH_STABLE/cli/$parsed_os/cortex elif command -v wget >/dev/null; then wget -q -O $cortex_sh_tmp_dir/cortex https://s3-us-west-2.amazonaws.com/get-cortex/$CORTEX_VERSION_BRANCH_STABLE/cli/$parsed_os/cortex else echo "error: please install \`curl\` or \`wget\`" exit 1 fi chmod +x $cortex_sh_tmp_dir/cortex if [ $(id -u) = 0 ]; then mv -f $cortex_sh_tmp_dir/cortex $CORTEX_INSTALL_PATH else ask_sudo sudo mv -f $cortex_sh_tmp_dir/cortex $CORTEX_INSTALL_PATH fi rm -rf $cortex_sh_tmp_dir echo "✓ installed cli" # prompt to update bash profile if running interactively if [ -t 1 ]; then update_bash_profile fi } function ask_sudo() { if ! sudo -n true 2>/dev/null; then echo -e "please enter your sudo password\n" fi } function get_bash_profile_path() { if [ "$parsed_os" = "darwin" ]; then if [ -f $HOME/.bash_profile ]; then echo $HOME/.bash_profile return elif [ -f $HOME/.bashrc ]; then echo $HOME/.bashrc return fi else if [ -f $HOME/.bashrc ]; then echo $HOME/.bashrc return elif [ -f $HOME/.bash_profile ]; then echo $HOME/.bash_profile return fi fi echo "" } function get_zsh_profile_path() { if [ -f $HOME/.zshrc ]; then echo $HOME/.zshrc return fi echo "" } function guess_if_bash_completion_installed() { if [ -f $HOME/.bash_profile ]; then if grep -q -e "^[^#]*bash-completion" -e "^[^#]*bash_completion" "$HOME/.bash_profile"; then echo "true" return fi fi if [ -f $HOME/.bashrc ]; then if grep -q -e "^[^#]*bash-completion" -e "^[^#]*bash_completion" "$HOME/.bashrc"; then echo "true" return fi fi echo "false" } function update_bash_profile() { bash_profile_path=$(get_bash_profile_path) zsh_profile_path=$(get_zsh_profile_path) maybe_bash_completion_installed=$(guess_if_bash_completion_installed) did_locate_shell_profile="false" if [ "$bash_profile_path" != "" ]; then did_locate_shell_profile="true" if ! grep -Fxq "source <(cortex completion bash)" "$bash_profile_path"; then echo read -p "Would you like to modify your bash profile ($bash_profile_path) to enable cortex command completion and the cx alias in bash? [y/n] " -r echo if [ "$REPLY" = "y" ] || [ "$REPLY" = "Y" ] || [ "$REPLY" = "yes" ] || [ "$REPLY" = "Yes" ] || [ "$REPLY" = "YES" ]; then echo -e "\nsource <(cortex completion bash)" >> $bash_profile_path echo -e "✓ Your bash profile has been updated" echo -e "\nCommand to update your current terminal session:" echo " source $bash_profile_path" if [ ! "$maybe_bash_completion_installed" = "true" ]; then echo -e "Note: \`bash-completion\` must be installed on your system for cortex command completion to function properly" fi else echo -e "Your bash profile has not been modified (run \`cortex completion --help\` to show how to enable bash completion manually)" fi fi fi if [ "$zsh_profile_path" != "" ]; then did_locate_shell_profile="true" if ! grep -Fxq "source <(cortex completion zsh)" "$zsh_profile_path"; then echo read -p "Would you like to modify your zsh profile ($zsh_profile_path) to enable cortex command completion and the cx alias in zsh? [y/n] " -r echo if [ "$REPLY" = "y" ] || [ "$REPLY" = "Y" ] || [ "$REPLY" = "yes" ] || [ "$REPLY" = "Yes" ] || [ "$REPLY" = "YES" ]; then echo -e "\nsource <(cortex completion zsh)" >> $zsh_profile_path echo -e "✓ Your zsh profile has been updated" echo -e "\nStart a new zsh shell for completion to take effect. If completion still doesn't work, you can try adding this line to the top of $zsh_profile_path: autoload -Uz compinit && compinit" else echo -e "Your zsh profile has not been modified (run \`cortex completion --help\` to show how to enable zsh completion manually)" fi fi fi if [ "$did_locate_shell_profile" = "false" ]; then echo -e "\nIf you would like to enable cortex bash completion and the cx alias, run \`cortex completion --help\` for instructions" fi } main ================================================ FILE: go.mod ================================================ module github.com/cortexlabs/cortex go 1.17 require ( github.com/DataDog/datadog-go v4.8.0+incompatible github.com/aws/amazon-vpc-cni-k8s v1.11.3 github.com/aws/aws-sdk-go v1.43.29 github.com/cortexlabs/go-input v0.0.0-20200503032952-8b67a7a7b28d github.com/cortexlabs/yaml v0.0.0-20210628201654-31e52ba8433b github.com/davecgh/go-spew v1.1.1 github.com/denormal/go-gitignore v0.0.0-20180930084346-ae8ad1d07817 github.com/docker/docker v20.10.7+incompatible github.com/fatih/color v1.12.0 github.com/getsentry/sentry-go v0.11.0 github.com/go-logr/logr v0.4.0 github.com/gobwas/glob v0.2.3 github.com/google/uuid v1.2.0 github.com/gorilla/handlers v1.5.1 github.com/gorilla/mux v1.8.0 github.com/gorilla/websocket v1.4.2 github.com/mitchellh/go-homedir v1.1.0 github.com/onsi/ginkgo v1.16.4 github.com/onsi/gomega v1.15.0 github.com/ory/dockertest/v3 v3.7.0 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.11.0 github.com/prometheus/common v0.29.0 github.com/shirou/gopsutil v3.21.6+incompatible github.com/spf13/cobra v1.5.0 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.8.0 github.com/ugorji/go/codec v1.2.6 github.com/xlab/treeprint v1.0.0 github.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c go.uber.org/atomic v1.8.0 go.uber.org/zap v1.19.0 gopkg.in/segmentio/analytics-go.v3 v3.1.0 istio.io/api v0.0.0-20220304045306-249321c725a9 istio.io/client-go v1.10.6 k8s.io/api v0.22.11 k8s.io/apimachinery v0.22.11 k8s.io/client-go v0.22.11 k8s.io/metrics v0.22.11 sigs.k8s.io/aws-iam-authenticator v0.5.3 sigs.k8s.io/controller-runtime v0.10.3 ) require ( cloud.google.com/go v0.81.0 // indirect github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect github.com/Azure/go-autorest/autorest v0.11.18 // indirect github.com/Azure/go-autorest/autorest/adal v0.9.13 // indirect github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect github.com/Azure/go-autorest/logger v0.2.1 // indirect github.com/Azure/go-autorest/tracing v0.6.0 // indirect github.com/Microsoft/go-winio v0.5.0 // indirect github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect github.com/StackExchange/wmi v0.0.0-20210224194228-fe8f1750fd46 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v4 v4.1.1 // indirect github.com/cespare/xxhash/v2 v2.1.1 // indirect github.com/containerd/containerd v1.5.4 // indirect github.com/containerd/continuity v0.1.0 // indirect github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 // indirect github.com/docker/cli v20.10.7+incompatible // indirect github.com/docker/distribution v2.7.1+incompatible // indirect github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.4.0 // indirect github.com/evanphx/json-patch v4.11.0+incompatible // indirect github.com/felixge/httpsnoop v1.0.2 // indirect github.com/form3tech-oss/jwt-go v3.2.3+incompatible // indirect github.com/fsnotify/fsnotify v1.5.4 // indirect github.com/go-logr/zapr v0.4.0 // indirect github.com/go-ole/go-ole v1.2.5 // indirect github.com/gofrs/flock v0.7.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/google/go-cmp v0.5.6 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/googleapis/gnostic v0.5.5 // indirect github.com/imdario/mergo v0.3.12 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/json-iterator/go v1.1.11 // indirect github.com/kr/pretty v0.3.0 // indirect github.com/mattn/go-colorable v0.1.12 // indirect github.com/mattn/go-isatty v0.0.14 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect github.com/mitchellh/mapstructure v1.4.1 // indirect github.com/moby/spdystream v0.2.0 // indirect github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.1 // indirect github.com/morikuni/aec v1.0.0 // indirect github.com/nxadm/tail v1.4.8 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.0.1 // indirect github.com/opencontainers/runc v1.0.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/procfs v0.6.0 // indirect github.com/segmentio/backo-go v0.0.0-20200129164019-23eae7c10bd3 // indirect github.com/sirupsen/logrus v1.8.1 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/xeipuuv/gojsonschema v1.2.0 // indirect go.uber.org/multierr v1.7.0 // indirect golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914 // indirect golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e // indirect golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect golang.org/x/text v0.3.7 // indirect golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect golang.org/x/tools v0.1.11 // indirect gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20210701133433-6b8dcf568a95 // indirect google.golang.org/grpc v1.39.0 // indirect google.golang.org/protobuf v1.27.1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect istio.io/gogo-genproto v0.0.0-20210113155706-4daf5697332f // indirect k8s.io/apiextensions-apiserver v0.22.2 // indirect k8s.io/component-base v0.22.2 // indirect k8s.io/klog/v2 v2.9.0 // indirect k8s.io/kube-openapi v0.0.0-20211110013926-83f114cd0513 // indirect k8s.io/utils v0.0.0-20211116205334-6203023598ed // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect sigs.k8s.io/yaml v1.2.0 // indirect ) replace github.com/docker/docker => github.com/docker/engine v17.12.0-ce-rc1.0.20200916142827-bd33bbf0497b+incompatible ================================================ FILE: go.sum ================================================ bazil.org/fuse v0.0.0-20160811212531-371fbbdaa898/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= cloud.google.com/go v0.81.0 h1:at8Tk2zUz63cLPR0JPWm5vp77pEZmzxEQBEfRKn1VV8= cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-ansiterm v0.0.0-20210608223527-2377c96fe795/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Azure/go-autorest v10.8.1+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= github.com/Azure/go-autorest/autorest v0.11.1/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw= github.com/Azure/go-autorest/autorest v0.11.18 h1:90Y4srNYrwOtAgVo3ndrQkTYn6kf1Eg/AjTFJ8Is2aM= github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg= github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= github.com/Azure/go-autorest/autorest/adal v0.9.13 h1:Mp5hbtOePIzM8pJVRa3YLrWWmZtoxRXqUEzCfJt3+/Q= github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= github.com/Azure/go-autorest/autorest/mocks v0.4.1 h1:K0laFcLE6VLTOwNgSxaGbUcLPuGXlNkbVvq4cW4nIHk= github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+ZtXWSmf4Tg= github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53/go.mod h1:+3IMCy2vIlbG1XG/0ggNQv0SvxCAIpPM5b1nCz56Xno= github.com/CloudyKit/jet/v3 v3.0.0/go.mod h1:HKQPgSJmdK8hdoAbKUUWajkHyHo4RaU5rMdUywE7VMo= github.com/DataDog/datadog-go v4.8.0+incompatible h1:vjzonG+3XzZgYrumNmdrA4QpXju/ZXrwb0mRjpYYbuo= github.com/DataDog/datadog-go v4.8.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw= github.com/Microsoft/go-winio v0.4.16-0.20201130162521-d1ffc52c7331/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= github.com/Microsoft/go-winio v0.4.17-0.20210211115548-6eac466e5fa3/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Microsoft/go-winio v0.4.17-0.20210324224401-5516f17a5958/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Microsoft/go-winio v0.4.17/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Microsoft/go-winio v0.5.0 h1:Elr9Wn+sGKPlkaBvwu4mTrxtmOp3F3yV9qhaHbXGjwU= github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg= github.com/Microsoft/hcsshim v0.8.7-0.20190325164909-8abdbb8205e4/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg= github.com/Microsoft/hcsshim v0.8.7/go.mod h1:OHd7sQqRFrYd3RmSgbgji+ctCwkbq2wbEYNSzOYtcBQ= github.com/Microsoft/hcsshim v0.8.9/go.mod h1:5692vkUqntj1idxauYlpoINNKeqCiG6Sg38RRsjT5y8= github.com/Microsoft/hcsshim v0.8.14/go.mod h1:NtVKoYxQuTLx6gEq0L96c9Ju4JbRJ4nY2ow3VK6a9Lg= github.com/Microsoft/hcsshim v0.8.15/go.mod h1:x38A4YbHbdxJtc0sF6oIz+RG0npwSCAvn69iY6URG00= github.com/Microsoft/hcsshim v0.8.16/go.mod h1:o5/SZqmR7x9JNKsW3pu+nqHm0MF8vbA+VxGOoXdC600= github.com/Microsoft/hcsshim v0.8.18/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwTOcER2fw4I4= github.com/Microsoft/hcsshim/test v0.0.0-20201218223536-d3e5debf77da/go.mod h1:5hlzMzRKMLyo42nCZ9oml8AdTlq/0cvIaBv6tK1RehU= github.com/Microsoft/hcsshim/test v0.0.0-20210227013316-43a75bb4edd3/go.mod h1:mw7qgWloBUl75W/gVH3cQszUg1+gUITj7D6NY7ywVnY= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw= github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0= github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ= github.com/StackExchange/wmi v0.0.0-20210224194228-fe8f1750fd46 h1:5sXbqlSomvdjlRbWyNqkPsJ3Fg+tQZCbgeX1VGljbQY= github.com/StackExchange/wmi v0.0.0-20210224194228-fe8f1750fd46/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/amazon-vpc-cni-k8s v1.11.3 h1:VlxS3LrwMbs6/wjZiOphj+9cjYaBN4SgQxmhwwMormY= github.com/aws/amazon-vpc-cni-k8s v1.11.3/go.mod h1:w8f1LLPue3xzRw7F9NPNWZt1B/yj7YNpypIuFMBmnyU= github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= github.com/aws/aws-sdk-go v1.37.1/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/aws/aws-sdk-go v1.43.29 h1:P6tBpMLwVLS/QwPkaBxfDIF3SmPouoacIk+/7NKnDxY= github.com/aws/aws-sdk-go v1.43.29/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/bshuster-repo/logrus-logstash-hook v0.4.1/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk= github.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8= github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b/go.mod h1:obH5gd0BsqsP2LwDJ9aOkm/6J86V6lyAXCoQWGw3K50= github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE= github.com/cenkalti/backoff/v4 v4.1.0/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/cenkalti/backoff/v4 v4.1.1 h1:G2HAfAmvm/GcKan2oOQpBXOd2tT2G57ZnZGWa1PxPBQ= github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/checkpoint-restore/go-criu/v4 v4.1.0/go.mod h1:xUQBLp4RLc5zJtWY++yjOoMoB5lihDt7fai+75m+rGw= github.com/checkpoint-restore/go-criu/v5 v5.0.0/go.mod h1:cfwC0EG7HMUenopBsUf9d89JlCLQIfgVcNsNN0t6T2M= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/cilium/ebpf v0.0.0-20200110133405-4032b1d8aae3/go.mod h1:MA5e5Lr8slmEg9bt0VpxxWqJlO4iwu3FBdHUzV7wQVg= github.com/cilium/ebpf v0.0.0-20200702112145-1c8d4c9ef775/go.mod h1:7cR51M8ViRLIdUjrmSXlK9pkrsDlLHbO8jiB8X8JnOc= github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs= github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= github.com/cilium/ebpf v0.6.1/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo= github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA= github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM= github.com/containerd/aufs v0.0.0-20200908144142-dab0cbea06f4/go.mod h1:nukgQABAEopAHvB6j7cnP5zJ+/3aVcE7hCYqvIwAHyE= github.com/containerd/aufs v0.0.0-20201003224125-76a6863f2989/go.mod h1:AkGGQs9NM2vtYHaUen+NljV0/baGCAPELGm2q9ZXpWU= github.com/containerd/aufs v0.0.0-20210316121734-20793ff83c97/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU= github.com/containerd/aufs v1.0.0/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU= github.com/containerd/btrfs v0.0.0-20201111183144-404b9149801e/go.mod h1:jg2QkJcsabfHugurUvvPhS3E08Oxiuh5W/g1ybB4e0E= github.com/containerd/btrfs v0.0.0-20210316141732-918d888fb676/go.mod h1:zMcX3qkXTAi9GI50+0HOeuV8LU2ryCE/V2vG/ZBiTss= github.com/containerd/btrfs v1.0.0/go.mod h1:zMcX3qkXTAi9GI50+0HOeuV8LU2ryCE/V2vG/ZBiTss= github.com/containerd/cgroups v0.0.0-20190717030353-c4b9ac5c7601/go.mod h1:X9rLEHIqSf/wfK8NsPqxJmeZgW4pcfzdXITDrUSJ6uI= github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko= github.com/containerd/cgroups v0.0.0-20200531161412-0dbf7f05ba59/go.mod h1:pA0z1pT8KYB3TCXK/ocprsh7MAkoW8bZVzPdih9snmM= github.com/containerd/cgroups v0.0.0-20200710171044-318312a37340/go.mod h1:s5q4SojHctfxANBDvMeIaIovkq29IP48TKAxnhYRxvo= github.com/containerd/cgroups v0.0.0-20200824123100-0b889c03f102/go.mod h1:s5q4SojHctfxANBDvMeIaIovkq29IP48TKAxnhYRxvo= github.com/containerd/cgroups v0.0.0-20210114181951-8a68de567b68/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE= github.com/containerd/cgroups v1.0.1/go.mod h1:0SJrPIenamHDcZhEcJMNBB85rHcUsw4f25ZfBiPYRkU= github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= github.com/containerd/console v0.0.0-20181022165439-0650fd9eeb50/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= github.com/containerd/console v0.0.0-20191206165004-02ecf6a7291e/go.mod h1:8Pf4gM6VEbTNRIT26AyyU7hxdQU3MvAvxVI0sc00XBE= github.com/containerd/console v1.0.1/go.mod h1:XUsP6YE/mKtz6bxc+I8UiKKTP04qjQL4qcS3XoQ5xkw= github.com/containerd/console v1.0.2/go.mod h1:ytZPjGgY2oeTkAONYafi2kSj0aYggsf8acV1PGKCbzQ= github.com/containerd/containerd v1.2.10/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/containerd v1.3.0-beta.2.0.20190828155532-0293cbd26c69/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/containerd v1.3.0/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/containerd v1.3.1-0.20191213020239-082f7e3aed57/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/containerd v1.3.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/containerd v1.4.0-beta.2.0.20200729163537-40b22ef07410/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/containerd v1.4.1/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/containerd v1.4.3/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/containerd v1.5.0-beta.1/go.mod h1:5HfvG1V2FsKesEGQ17k5/T7V960Tmcumvqn8Mc+pCYQ= github.com/containerd/containerd v1.5.0-beta.3/go.mod h1:/wr9AVtEM7x9c+n0+stptlo/uBBoBORwEx6ardVcmKU= github.com/containerd/containerd v1.5.0-beta.4/go.mod h1:GmdgZd2zA2GYIBZ0w09ZvgqEq8EfBp/m3lcVZIvPHhI= github.com/containerd/containerd v1.5.0-rc.0/go.mod h1:V/IXoMqNGgBlabz3tHD2TWDoTJseu1FGOKuoA4nNb2s= github.com/containerd/containerd v1.5.1/go.mod h1:0DOxVqwDy2iZvrZp2JUx/E+hS0UNTVn7dJnIOwtYR4g= github.com/containerd/containerd v1.5.4 h1:uPF0og3ByFzDnaStfiQj3fVGTEtaSNyU+bW7GR/nqGA= github.com/containerd/containerd v1.5.4/go.mod h1:sx18RgvW6ABJ4iYUw7Q5x7bgFOAB9B6G7+yO0XBc4zw= github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/continuity v0.0.0-20190815185530-f2a389ac0a02/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/continuity v0.0.0-20190827140505-75bee3e2ccb6/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/continuity v0.0.0-20191127005431-f65d91d395eb/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/continuity v0.0.0-20200710164510-efbc4488d8fe/go.mod h1:cECdGN1O8G9bgKTlLhuPJimka6Xb/Gg7vYzCTNVxhvo= github.com/containerd/continuity v0.0.0-20201208142359-180525291bb7/go.mod h1:kR3BEg7bDFaEddKm54WSmrol1fKWDU1nKYkgrcgZT7Y= github.com/containerd/continuity v0.0.0-20210208174643-50096c924a4e/go.mod h1:EXlVlkqNba9rJe3j7w3Xa924itAMLgZH4UD/Q4PExuQ= github.com/containerd/continuity v0.1.0 h1:UFRRY5JemiAhPZrr/uE0n8fMTLcZsUvySPr1+D7pgr8= github.com/containerd/continuity v0.1.0/go.mod h1:ICJu0PwR54nI0yPEnJ6jcS+J7CZAUXrLh8lPo2knzsM= github.com/containerd/fifo v0.0.0-20180307165137-3d5202aec260/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= github.com/containerd/fifo v0.0.0-20200410184934-f15a3290365b/go.mod h1:jPQ2IAeZRCYxpS/Cm1495vGFww6ecHmMk1YJH2Q5ln0= github.com/containerd/fifo v0.0.0-20201026212402-0724c46b320c/go.mod h1:jPQ2IAeZRCYxpS/Cm1495vGFww6ecHmMk1YJH2Q5ln0= github.com/containerd/fifo v0.0.0-20210316144830-115abcc95a1d/go.mod h1:ocF/ME1SX5b1AOlWi9r677YJmCPSwwWnQ9O123vzpE4= github.com/containerd/fifo v1.0.0/go.mod h1:ocF/ME1SX5b1AOlWi9r677YJmCPSwwWnQ9O123vzpE4= github.com/containerd/go-cni v1.0.1/go.mod h1:+vUpYxKvAF72G9i1WoDOiPGRtQpqsNW/ZHtSlv++smU= github.com/containerd/go-cni v1.0.2/go.mod h1:nrNABBHzu0ZwCug9Ije8hL2xBCYh/pjfMb1aZGrrohk= github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= github.com/containerd/go-runc v0.0.0-20190911050354-e029b79d8cda/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= github.com/containerd/go-runc v0.0.0-20200220073739-7016d3ce2328/go.mod h1:PpyHrqVs8FTi9vpyHwPwiNEGaACDxT/N/pLcvMSRA9g= github.com/containerd/go-runc v0.0.0-20201020171139-16b287bc67d0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok= github.com/containerd/go-runc v1.0.0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok= github.com/containerd/imgcrypt v1.0.1/go.mod h1:mdd8cEPW7TPgNG4FpuP3sGBiQ7Yi/zak9TYCG3juvb0= github.com/containerd/imgcrypt v1.0.4-0.20210301171431-0ae5c75f59ba/go.mod h1:6TNsg0ctmizkrOgXRNQjAPFWpMYRWuiB6dSF4Pfa5SA= github.com/containerd/imgcrypt v1.1.1-0.20210312161619-7ed62a527887/go.mod h1:5AZJNI6sLHJljKuI9IHnw1pWqo/F0nGDOuR9zgTs7ow= github.com/containerd/imgcrypt v1.1.1/go.mod h1:xpLnwiQmEUJPvQoAapeb2SNCxz7Xr6PJrXQb0Dpc4ms= github.com/containerd/nri v0.0.0-20201007170849-eb1350a75164/go.mod h1:+2wGSDGFYfE5+So4M5syatU0N0f0LbWpuqyMi4/BE8c= github.com/containerd/nri v0.0.0-20210316161719-dbaa18c31c14/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY= github.com/containerd/nri v0.1.0/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY= github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= github.com/containerd/ttrpc v0.0.0-20190828172938-92c8520ef9f8/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= github.com/containerd/ttrpc v0.0.0-20191028202541-4f1b8fe65a5c/go.mod h1:LPm1u0xBw8r8NOKoOdNMeVHSawSsltak+Ihv+etqsE8= github.com/containerd/ttrpc v1.0.1/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y= github.com/containerd/ttrpc v1.0.2/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y= github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc= github.com/containerd/typeurl v0.0.0-20190911142611-5eb25027c9fd/go.mod h1:GeKYzf2pQcqv7tJ0AoCuuhtnqhva5LNU3U+OyKxxJpk= github.com/containerd/typeurl v1.0.1/go.mod h1:TB1hUtrpaiO88KEK56ijojHS1+NeF0izUACaJW2mdXg= github.com/containerd/typeurl v1.0.2/go.mod h1:9trJWW2sRlGub4wZJRTW83VtbOLS6hwcDZXTn6oPz9s= github.com/containerd/zfs v0.0.0-20200918131355-0a33824f23a2/go.mod h1:8IgZOBdv8fAgXddBT4dBXJPtxyRsejFIpXoklgxgEjw= github.com/containerd/zfs v0.0.0-20210301145711-11e8f1707f62/go.mod h1:A9zfAbMlQwE+/is6hi0Xw8ktpL+6glmqZYtevJgaB8Y= github.com/containerd/zfs v0.0.0-20210315114300-dde8f0fda960/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY= github.com/containerd/zfs v0.0.0-20210324211415-d5c4544f0433/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY= github.com/containerd/zfs v1.0.0/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY= github.com/containernetworking/cni v0.7.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= github.com/containernetworking/cni v0.8.0/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= github.com/containernetworking/cni v0.8.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= github.com/containernetworking/plugins v0.8.6/go.mod h1:qnw5mN19D8fIwkqW7oHHYDHVlzhJpcY6TQxn/fUyDDM= github.com/containernetworking/plugins v0.9.0/go.mod h1:dbWv4dI0QrBGuVgj+TuVQ6wJRZVOhrCQj91YyC92sxg= github.com/containernetworking/plugins v0.9.1/go.mod h1:xP/idU2ldlzN6m4p5LmGiwRDjeJr6FLK6vuiUwoH7P8= github.com/containers/ocicrypt v1.0.1/go.mod h1:MeJDzk1RJHv89LjsH0Sp5KTY3ZYkjXO/C+bKAeWFIrc= github.com/containers/ocicrypt v1.1.0/go.mod h1:b8AOe0YR67uU8OqfVNcznfFpAzu3rdgUV4GP9qXPfu4= github.com/containers/ocicrypt v1.1.1/go.mod h1:Dm55fwWm1YZAjYRaJ94z2mfZikIyIN4B0oB3dj3jFxY= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-iptables v0.4.5/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= github.com/coreos/go-iptables v0.5.0/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20161114122254-48702e0da86b/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cortexlabs/go-input v0.0.0-20200503032952-8b67a7a7b28d h1:2SmODObcy2ekPA0eLFlR/+Vu5Yo2hoVbNMJ+vWzinpo= github.com/cortexlabs/go-input v0.0.0-20200503032952-8b67a7a7b28d/go.mod h1:rxijm42+fHjyPbFGqUTzUUhioEVg5LUi+w53dlY0WUM= github.com/cortexlabs/yaml v0.0.0-20210628201654-31e52ba8433b h1:jS+jWCjvtnZJ9f9GGR9eA6EgPBhKZ3ILUvTmMglYzyw= github.com/cortexlabs/yaml v0.0.0-20210628201654-31e52ba8433b/go.mod h1:qYZ6ij3BG8ss8YpnSPcpdl9BCWIWyStg8OcyVOFlJKI= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4= github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c/go.mod h1:Ct2BUK8SB0YC1SMSibvLzxjeJLnrYEVLULFNiHY9YfQ= github.com/d2g/dhcp4client v1.0.0/go.mod h1:j0hNfjhrt2SxUOw55nL0ATM/z4Yt3t2Kd1mW34z5W5s= github.com/d2g/dhcp4server v0.0.0-20181031114812-7d4a0a7f59a5/go.mod h1:Eo87+Kg/IX2hfWJfwxMzLyuSZyxSoAug2nGa1G2QAi8= github.com/d2g/hardwareaddr v0.0.0-20190221164911-e7d9fbe030e4/go.mod h1:bMl4RjIciD2oAxI7DmWRx6gbeqrkoLqv3MV0vzNad+I= github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 h1:y5HC9v93H5EPKqaS1UYVg1uYah5Xf51mBfIoWehClUQ= github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964/go.mod h1:Xd9hchkHSWYkEqJwUGisez3G1QY8Ryz0sdWrLPMGjLk= github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/denormal/go-gitignore v0.0.0-20180930084346-ae8ad1d07817 h1:0nsrg//Dc7xC74H/TZ5sYR8uk4UQRNjsw8zejqH5a4Q= github.com/denormal/go-gitignore v0.0.0-20180930084346-ae8ad1d07817/go.mod h1:C/+sI4IFnEpCn6VQ3GIPEp+FrQnQw+YQP3+n+GdGq7o= github.com/denverdino/aliyungo v0.0.0-20190125010748-a747050bb1ba/go.mod h1:dV8lFg6daOBZbT6/BDGIz6Y3WFGn8juu6G+CQ6LHtl0= github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= github.com/dgrijalva/jwt-go v0.0.0-20170104182250-a601269ab70c/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= github.com/docker/cli v20.10.7+incompatible h1:pv/3NqibQKphWZiAskMzdz8w0PRbtTaEB+f6NwdU7Is= github.com/docker/cli v20.10.7+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/distribution v0.0.0-20190905152932-14b96e55d84c/go.mod h1:0+TTO4EOBfRPhZXAeF1Vu+W3hHZ8eLp8PgKVZlcvtFY= github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/engine v17.12.0-ce-rc1.0.20200916142827-bd33bbf0497b+incompatible h1:nnCzIfwUkdP7f5ZYf8el5qKoWSwuQxEeTcDwWHLKsKg= github.com/docker/engine v17.12.0-ce-rc1.0.20200916142827-bd33bbf0497b+incompatible/go.mod h1:3CPr2caMgTHxxIAZgEMd3uLYPDlRvPqCpyeRf6ncPcY= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-events v0.0.0-20170721190031-9461782956ad/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= github.com/docker/go-metrics v0.0.0-20180209012529-399ea8c73916/go.mod h1:/u0gXw0Gay3ceNrsHubL3BtdOL2fHf93USgMTe0W5dI= github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw= github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.11.0+incompatible h1:glyUF9yIYtMHzn8xaKw5rMhdWcwsYV8dZHIq5567/xs= github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/color v1.12.0 h1:mRhaKNwANqRgUBGKmnI5ZxEk7QXmjQeCcuYFMX2bfcc= github.com/fatih/color v1.12.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/felixge/httpsnoop v1.0.2 h1:+nS9g82KMXccJ/wp0zyRW9ZBHFETmMGtkk+2CTTrW4o= github.com/felixge/httpsnoop v1.0.2/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/form3tech-oss/jwt-go v3.2.3+incompatible h1:7ZaBxOI7TMoYBfyA3cQHErNNyAWIKUMIwqxEtgHOs5c= github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa/go.mod h1:KnogPXtdwXqoenmZCw6S+25EAm2MkxbG0deNDu4cbSA= github.com/garyburd/redigo v0.0.0-20150301180006-535138d7bcd7/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc= github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= github.com/getsentry/sentry-go v0.11.0 h1:qro8uttJGvNAMr5CLcFI9CHR0aDzXl0Vs3Pmw/oTPg8= github.com/getsentry/sentry-go v0.11.0/go.mod h1:KBQIxiZAetw62Cj8Ri964vAEWVdgfaUCn30Q3bCvANo= github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM= github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v0.3.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc= github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/zapr v0.2.0/go.mod h1:qhKdvif7YF5GI9NWEpyxTSSBdGmzkNguibrdCNVPunU= github.com/go-logr/zapr v0.4.0 h1:uc1uML3hRYL9/ZZPdgHS/n8Nzo+eaYL/Efxkkamf7OM= github.com/go-logr/zapr v0.4.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk= github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8= github.com/go-ole/go-ole v1.2.5 h1:t4MGB5xEDZvXI+0rMjjsfBsD7yAgp/s9ZDkL1JndXwY= github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= github.com/go-openapi/spec v0.19.2/go.mod h1:sCxk3jxKgioEJikev4fgkNmwS+3kuYdJtcsZsD5zxMY= github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= github.com/godbus/dbus v0.0.0-20151105175453-c7fdd8b5cd55/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofrs/flock v0.7.0 h1:pGFUjl501gafK9HBt1VGL1KCOd/YhIooID+xgyJCf3g= github.com/gofrs/flock v0.7.0/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gogo/googleapis v1.2.0/go.mod h1:Njal3psf3qN6dwBtQfUmBZh2ybovJ0tlu3o/AC7HYjU= github.com/gogo/googleapis v1.4.0/go.mod h1:5YRNX2z1oM5gXdAkurHa942MDgEJyk02w4OecKY87+c= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/mock v1.5.0 h1:jlYHihg//f7RRwuPfptm04yp4s7O6Kw8EZiVYIGcH0g= github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/gomodule/redigo v1.7.1-0.20190724094224-574c33c3df38/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-jsonnet v0.16.0/go.mod h1:sOcuej3UW1vpPTZOr8L7RQimqai1a57bt5j22LzGZCw= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= github.com/googleapis/gnostic v0.5.5 h1:9fHAtK0uDfpveeqqo1hkEZJcFvYXAiCN3UutL8F9xHw= github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/handlers v0.0.0-20150720190736-60c7bfde3e33/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= github.com/gorilla/handlers v1.5.1 h1:9lRY6j8DEeeBT10CvO9hGW0gmky0BprnvDI5vfhUHH4= github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q= github.com/gorilla/mux v1.7.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/iris-contrib/blackfriday v2.0.0+incompatible/go.mod h1:UzZ2bDEoaSGPbkg6SAB4att1aAwTmVIx/5gCVqeyUdI= github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/+fafWORmlnuysV2EMP8MW+qe0= github.com/iris-contrib/jade v1.1.3/go.mod h1:H/geBymxJhShH5kecoiOCSssPX7QWYH7UaeZTSWddIk= github.com/iris-contrib/pongo2 v0.0.1/go.mod h1:Ssh+00+3GAZqSQb30AvBRNxBx7rf0GqwkjqxNd0u65g= github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw= github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= github.com/kataras/golog v0.0.10/go.mod h1:yJ8YKCmyL+nWjERB90Qwn+bdyBZsaQwU3bTVFgkFIp8= github.com/kataras/iris/v12 v12.1.8/go.mod h1:LMYy4VlP67TQ3Zgriz8RE2h2kMZV2SgMYbq3UhfoFmE= github.com/kataras/neffos v0.0.14/go.mod h1:8lqADm8PnbeFfL7CLXh1WHw53dG27MC3pgi2R1rmoTE= github.com/kataras/pio v0.0.2/go.mod h1:hAoW0t9UmXi4R5Oyq5Z4irTbaTsOemSrDGUtaTl7Dro= github.com/kataras/sitemap v0.0.5/go.mod h1:KY2eugMKiPwsJgx7+U103YZehfvNGOXURubcGyk0Bz8= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.9.7/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/labstack/echo/v4 v4.1.11/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvfxNnFqi74g= github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= github.com/lib/pq v0.0.0-20180327071824-d34b9ff171c2 h1:hRGSmZu7j271trc9sneMrpOW7GN5ngLm8YUZIPzf394= github.com/lib/pq v0.0.0-20180327071824-d34b9ff171c2/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/marstr/guid v1.1.0/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/mediocregopher/radix/v3 v3.4.2/go.mod h1:8FL3F6UQRXHXIBSPUs5h0RybMF8i4n7wVopoX3x7Bv8= github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A= github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc= github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/moby/sys/mountinfo v0.4.0/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= github.com/moby/sys/mountinfo v0.4.1/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= github.com/moby/sys/symlink v0.1.0/go.mod h1:GGDODQmbFOjFsXvfLVn3+ZRxkch54RkSiGqsZeMYowQ= github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc= github.com/moby/term v0.0.0-20210610120745-9d4ed1856297/go.mod h1:vgPCkQMyxTZ7IDy8SXRufE172gr8+K/JE/7hHFxHW3A= github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 h1:dcztxKSvZ4Id8iPpHERQBbIJfabdt4wUm5qy3wOL2Zc= github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180320133207-05fbef0ca5da/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ= github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onsi/ginkgo v0.0.0-20151202141238-7f8ab55aaf3b/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.14.1/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/gomega v0.0.0-20151007035656-2152b45fa28a/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.10.2/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc= github.com/onsi/gomega v1.15.0 h1:WjP/FQ/sk43MRmnEcT+MlDw2TFvkrXlprrPST/IudjU= github.com/onsi/gomega v1.15.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0= github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v1.0.0-rc1.0.20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.0.0/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/opencontainers/runc v1.0.0-rc8.0.20190926000215-3e425f80a8c9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/opencontainers/runc v1.0.0-rc9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/opencontainers/runc v1.0.0-rc93/go.mod h1:3NOsor4w32B2tC0Zbl8Knk4Wg84SM2ImC1fxBuqJ/H0= github.com/opencontainers/runc v1.0.0 h1:QOhAQAYUlKeofuyeKdR6ITvOnXLPbEAjPMjz9wCUXcU= github.com/opencontainers/runc v1.0.0/go.mod h1:MU2S3KEB2ZExnhnAQYbwjdYV6HwKtDlNbA2Z2OeNDeA= github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.0.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.0.2-0.20190207185410-29686dbc5559/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.0.3-0.20200929063507-e6143ca7d51d/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-tools v0.0.0-20181011054405-1d69bd0f9c39/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs= github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqiriPsEqVhc+svHE= github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo= github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/ory/dockertest/v3 v3.7.0 h1:Bijzonc69Ont3OU0a3TWKJ1Rzlh3TsDXP1JrTAkSmsM= github.com/ory/dockertest/v3 v3.7.0/go.mod h1:PvCCgnP7AfBZeVrzwiUTjZx/IUXlGLC1zQlUQrLIlUE= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= github.com/prometheus/client_golang v0.0.0-20180209125602-c332b6f63c06/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20180110214958-89604d197083/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.29.0 h1:3jqPBvKT4OHAbje2Ql7KeaaSicDBCxMYwEJU1zRJceE= github.com/prometheus/common v0.29.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.0-20190522114515-bc1a522cf7b1/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/schollz/closestmatch v2.1.0+incompatible/go.mod h1:RtP1ddjLong6gTkbtmuhtR2uUrrJOpYzYRvbcPAid+g= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= github.com/segmentio/backo-go v0.0.0-20200129164019-23eae7c10bd3 h1:ZuhckGJ10ulaKkdvJtiAqsLTiPrLaXSdnVgXJKJkTxE= github.com/segmentio/backo-go v0.0.0-20200129164019-23eae7c10bd3/go.mod h1:9/Rh6yILuLysoQnZ2oNooD2g7aBnvM7r/fNVxRNWfBc= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/shirou/gopsutil v3.21.6+incompatible h1:mmZtAlWSd8U2HeRTjswbnDLPxqsEoK01NK+GZ1P+nEM= github.com/shirou/gopsutil v3.21.6+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= github.com/spf13/cobra v1.5.0 h1:X+jTBEBqF0bHN+9cSMgmfuvv2VHJ9ezmFNf9Y/XstYU= github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.1-0.20171106142849-4c012f6dcd95/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980/go.mod h1:AO3tvPzVZ/ayst6UlUKUv6rcPQInYe3IknH3jYhAKu8= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.0.0-20180129172003-8a3f7159479f/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.4.0 h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v0.0.0-20180303142811-b89eecf5ca5d/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/tchap/go-patricia v2.2.6+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go v1.2.6 h1:tGiWC9HENWE2tqYycIqFTNorMmFRVhNwCpDOpWqnk8E= github.com/ugorji/go v1.2.6/go.mod h1:anCg0y61KIhDlPZmnH+so+RQbysYVyDko0IMgJv0Nn0= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= github.com/ugorji/go/codec v1.2.6 h1:7kbGefxLoDBuYXOms4yD7223OpNMMPNPZxXk5TvFcyQ= github.com/ugorji/go/codec v1.2.6/go.mod h1:V6TCNZ4PHqoHGFZuSG1W8nrCzzdgA2DozYxWFFpvxTw= github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w= github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI= github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xlab/treeprint v1.0.0 h1:J0TkWtiuYgtdlrkkrDLISYBQ92M+X5m4LrIIMKrbDTs= github.com/xlab/treeprint v1.0.0/go.mod h1:IoImgRak9i3zJyuxOKUP1v4UZd1tMoKkq/Cimt1uhCg= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c h1:3lbZUMbMiGUW/LMkfsEABsc5zNT9+b1CvsJx47JzJ8g= github.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c/go.mod h1:UrdRz5enIKZ63MEE3IF9l2/ebyx59GyGgPi+tICQdmM= github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI= github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg= github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM= github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs= github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50/go.mod h1:NUSPSUX/bi6SeDMUh6brw0nXpxHnc96TguQh0+r/ssA= github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f/go.mod h1:GlGEuHIJweS1mbCqG+7vt2nvWLzLLnRHbXz5JKd/Qbg= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= go.etcd.io/etcd/client/v3 v3.5.0/go.mod h1:AIKXXVX/DQXtfTEqBryiLTUXwON+GuvO6Z7lLS/oTh0= go.etcd.io/etcd/pkg/v3 v3.5.0/go.mod h1:UzJGatBQ1lXChBkQF0AuAtkRQMYnHubxAEYIrC3MSsE= go.etcd.io/etcd/raft/v3 v3.5.0/go.mod h1:UFOHSIvO/nKwd4lhkwabrTD3cqW5yVyYYf/KlD00Szc= go.etcd.io/etcd/server/v3 v3.5.0/go.mod h1:3Ah5ruV+M+7RZr0+Y/5mNLwC+eQlni+mQmOVdCRJoS4= go.hein.dev/go-version v0.1.0/go.mod h1:WOEm7DWMroRe5GdUgHMvx+Pji5WWIpMuXmK/3foylXs= go.mozilla.org/pkcs7 v0.0.0-20200128120323-432b2356ecb1/go.mod h1:SNgMg+EgDFwmvSmLRTNKC5fegJjB7v23qTQ0XLGUNHk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0/go.mod h1:oVGt1LRbBOBq1A5BQLlUg9UaU/54aiHw8cgjV3aWZ/E= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0/go.mod h1:2AboqHi0CiIZU0qwhtUfCYD1GeUzvvIXWNkhDt7ZMG4= go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo= go.opentelemetry.io/otel/exporters/otlp v0.20.0/go.mod h1:YIieizyaN77rtLJra0buKiNBOm9XQfkPEKBeuhoMwAM= go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw= go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc= go.opentelemetry.io/otel/sdk/export/metric v0.20.0/go.mod h1:h7RBNMsDJ5pmI1zExLi+bJK+Dr8NQCh0qGhm1KDnNlE= go.opentelemetry.io/otel/sdk/metric v0.20.0/go.mod h1:knxiS8Xd4E/N+ZqKmUPf3gTTZ4/0TjTXukfxjzSTpHE= go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.8.0 h1:CUhrE4N1rqSE6FM9ecihEjRkLQu8cDfgDyoOs83mEY4= go.uber.org/atomic v1.8.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.7.0 h1:zaiO/rmgFjbmCXdSYJWQcdvOCsthmdaHfr3Gm2Kx4Ec= go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.8.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go.uber.org/zap v1.19.0 h1:mZQZefskPPCMIBCSEH0v2/iUqqLrYtaeqwD6FUGUnFE= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181009213950-7c1a557ab941/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190312203227-4b39c73a6495/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190327091125-710a502c58a2/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190619014844-b5b0513f8c1b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914 h1:3B43BWw0xEBsLZ/NO1VALz6fppU3481pik+2Ksv45z8= golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190514135907-3a4b5fb9f71f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190522044717-8097e1b27ff5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190602015325-4c4f7f33c9ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190812073006-9eafafc0a87e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190812172437-4e8604ab3aff/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200120151820-655fe14d7479/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200817155316-9781c653f443/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200916030750-2334cc1a136f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200922070232-aee5d888a860/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201117170446-d9b008d0a637/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201202213521-69691e467435/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210817190340-bfb29a6856f2/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e h1:CsOuNlbOuf0mzxJIefr6Q4uAUetRUwZE4qt7VfzP+xo= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs= golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181221001348-537d06c36207/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190327201419-c70d86f8b7cf/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190812233024-afc3694995b6/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200616133436-c1934b75d054/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/tools v0.1.11 h1:loJ25fNOEhSXfHrpoGj91eCUThwdNX6u24rO1xnNteY= golang.org/x/tools v0.1.11/go.mod h1:SgwaegtQh8clINPpECJMqnxLv9I09HLqnW3RMqW0CA4= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df h1:5Pf6pFKu98ODmgnpvkJ3kFUOQGGLIzLIkbzUHp47618= golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= gomodules.xyz/jsonpatch/v2 v2.1.0/go.mod h1:IhYNNY4jnS53ZnfE4PAmpKtDpTCj1JFXc+3mwe7XcUU= gomodules.xyz/jsonpatch/v2 v2.2.0 h1:4pT439QV83L+G9FkcCriY6EkpcK6r6bK+A5FBUMI7qY= gomodules.xyz/jsonpatch/v2 v2.2.0/go.mod h1:WXp+iVDkoLQqPudfQ9GBlwB2eZ5DKOnjQZCYdOS8GPY= gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485/go.mod h1:2ltnJ7xHfj0zHS40VVPYEAAMTa3ZGguvHGBSJeRWqE0= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e/go.mod h1:kS+toOQn6AQKjmKJ7gzohV1XkqsFehRA2FbsbkopSuQ= google.golang.org/api v0.0.0-20160322025152-9bf6e6e569ff/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/cloud v0.0.0-20151119220103-975617b05ea8/go.mod h1:0H1ncTHf11KCFhTc/+EFRbzSCOZx+VUbRMk55Yv5MYk= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190522204451-c2c4e71fbf69/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200117163144-32f20d992d24/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= google.golang.org/genproto v0.0.0-20210701133433-6b8dcf568a95 h1:xyRjacsGcaSoZ2fTcaLCSzh2JEceLLOT4X8k32Q0xAQ= google.golang.org/genproto v0.0.0-20210701133433-6b8dcf568a95/go.mod h1:yiaVoXHpRzHGyxV3o4DktVWY4mSUErTKaeEOq6C3t3U= google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= google.golang.org/grpc v1.29.0/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.39.0 h1:Klz8I9kdtkIN6EpHHUOMLCYhTn/2WAe5a0s1hcBkdTI= google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20141024133853-64131543e789/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.51.1/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/segmentio/analytics-go.v3 v3.1.0 h1:UzxH1uaGZRpMKDhJyBz0pexz6yUoBU3x8bJsRk/HV6U= gopkg.in/segmentio/analytics-go.v3 v3.1.0/go.mod h1:4QqqlTlSSpVlWA9/9nDcPw+FkM2yv1NQoYjUbL9/JAw= gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/square/go-jose.v2 v2.5.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20191120175047-4206685974f2/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= istio.io/api v0.0.0-20211015181651-ddbde26ea264/go.mod h1:nsSFw1LIMmGL7r/+6fJI6FxeG/UGlLxRK8bkojIvBVs= istio.io/api v0.0.0-20220304045306-249321c725a9 h1:CYmLAMT4hKCQ4m3LgwlbP/VEHAZK2lfUCEYN1oe2K0E= istio.io/api v0.0.0-20220304045306-249321c725a9/go.mod h1:nsSFw1LIMmGL7r/+6fJI6FxeG/UGlLxRK8bkojIvBVs= istio.io/client-go v1.10.6 h1:DMcNQOnsx/D5P24U/8IcCNIfyaJgqmOwVW8mnbf2j8Q= istio.io/client-go v1.10.6/go.mod h1:VXhAHzCaFpMKDL3Kul+hCz+dX3FGjM3GuTXGoi7tTv4= istio.io/gogo-genproto v0.0.0-20210113155706-4daf5697332f h1:9710FpGLvIJ1GGEbpuTh1smVBv+r8cJfR3G82ouSxIQ= istio.io/gogo-genproto v0.0.0-20210113155706-4daf5697332f/go.mod h1:6BwTZRNbWS570wHX/uR1Wqk5e0157TofTAUMzT7N4+s= k8s.io/api v0.16.8/go.mod h1:a8EOdYHO8en+YHhPBLiW5q+3RfHTr7wxTqqp7emJ7PM= k8s.io/api v0.20.1/go.mod h1:KqwcCVogGxQY3nBlRpwt+wpAMF/KjaCc7RpywacvqUo= k8s.io/api v0.20.2/go.mod h1:d7n6Ehyzx+S+cE3VhTGfVNNqtGc/oL9DCdYYahlurV8= k8s.io/api v0.20.4/go.mod h1:++lNL1AJMkDymriNniQsWRkMDzRaX2Y/POTUi8yvqYQ= k8s.io/api v0.20.6/go.mod h1:X9e8Qag6JV/bL5G6bU8sdVRltWKmdHsFUGS3eVndqE8= k8s.io/api v0.22.2/go.mod h1:y3ydYpLJAaDI+BbSe2xmGcqxiWHmWjkEeIbiwHvnPR8= k8s.io/api v0.22.11 h1:1CGTTsEy2VCZK9uDXyEauDN7VRiMCyVOFeI0Bj3JLts= k8s.io/api v0.22.11/go.mod h1:l2IQ0P1TtTvSeWnjqTrFQ7sX5LR1H9a9Knb0N2ILJKE= k8s.io/apiextensions-apiserver v0.20.1/go.mod h1:ntnrZV+6a3dB504qwC5PN/Yg9PBiDNt1EVqbW2kORVk= k8s.io/apiextensions-apiserver v0.22.2 h1:zK7qI8Ery7j2CaN23UCFaC1hj7dMiI87n01+nKuewd4= k8s.io/apiextensions-apiserver v0.22.2/go.mod h1:2E0Ve/isxNl7tWLSUDgi6+cmwHi5fQRdwGVCxbC+KFA= k8s.io/apimachinery v0.16.8/go.mod h1:Xk2vD2TRRpuWYLQNM6lT9R7DSFZUYG03SarNkbGrnKE= k8s.io/apimachinery v0.20.1/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= k8s.io/apimachinery v0.20.2/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= k8s.io/apimachinery v0.20.4/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= k8s.io/apimachinery v0.20.6/go.mod h1:ejZXtW1Ra6V1O5H8xPBGz+T3+4gfkTCeExAHKU57MAc= k8s.io/apimachinery v0.22.2/go.mod h1:O3oNtNadZdeOMxHFVxOreoznohCpy0z6mocxbZr7oJ0= k8s.io/apimachinery v0.22.11 h1:wOAkOC3Vzzf2s2Bqmmk7fmh5jIl2xH8kxom6qAjQqO4= k8s.io/apimachinery v0.22.11/go.mod h1:ZvVLP5iLhwVFg2Yx9Gh5W0um0DUauExbRhe+2Z8I1EU= k8s.io/apiserver v0.20.1/go.mod h1:ro5QHeQkgMS7ZGpvf4tSMx6bBOgPfE+f52KwvXfScaU= k8s.io/apiserver v0.20.4/go.mod h1:Mc80thBKOyy7tbvFtB4kJv1kbdD0eIH8k8vianJcbFM= k8s.io/apiserver v0.20.6/go.mod h1:QIJXNt6i6JB+0YQRNcS0hdRHJlMhflFmsBDeSgT1r8Q= k8s.io/apiserver v0.22.2/go.mod h1:vrpMmbyjWrgdyOvZTSpsusQq5iigKNWv9o9KlDAbBHI= k8s.io/client-go v0.16.8/go.mod h1:WmPuN0yJTKHXoklExKxzo3jSXmr3EnN+65uaTb5VuNs= k8s.io/client-go v0.20.1/go.mod h1:/zcHdt1TeWSd5HoUe6elJmHSQ6uLLgp4bIJHVEuy+/Y= k8s.io/client-go v0.20.2/go.mod h1:kH5brqWqp7HDxUFKoEgiI4v8G1xzbe9giaCenUWJzgE= k8s.io/client-go v0.20.4/go.mod h1:LiMv25ND1gLUdBeYxBIwKpkSC5IsozMMmOOeSJboP+k= k8s.io/client-go v0.20.6/go.mod h1:nNQMnOvEUEsOzRRFIIkdmYOjAZrC8bgq0ExboWSU1I0= k8s.io/client-go v0.22.2/go.mod h1:sAlhrkVDf50ZHx6z4K0S40wISNTarf1r800F+RlCF6U= k8s.io/client-go v0.22.11 h1:2lQhSQqqnNRVOHzK0i9rG4p+jYKjct7t8lXl++GhjrY= k8s.io/client-go v0.22.11/go.mod h1:zNje4oH4oU3DchJmlmU71VeOgPxiJ+r/Rh8VRq7Jq6E= k8s.io/code-generator v0.16.8/go.mod h1:wFdrXdVi/UC+xIfLi+4l9elsTT/uEF61IfcN2wOLULQ= k8s.io/code-generator v0.20.1/go.mod h1:UsqdF+VX4PU2g46NC2JRs4gc+IfrctnwHb76RNbWHJg= k8s.io/code-generator v0.22.2/go.mod h1:eV77Y09IopzeXOJzndrDyCI88UBok2h6WxAlBwpxa+o= k8s.io/code-generator v0.22.11/go.mod h1:iOZwYADSgFPNGWfqHFfg1V0TNJnl1t0WyZluQp4baqU= k8s.io/component-base v0.16.8/go.mod h1:Q8UWOWShpP3MZZny4n/15gOncfaaVtc9SbCdkM5MhUE= k8s.io/component-base v0.20.1/go.mod h1:guxkoJnNoh8LNrbtiQOlyp2Y2XFCZQmrcg2n/DeYNLk= k8s.io/component-base v0.20.2/go.mod h1:pzFtCiwe/ASD0iV7ySMu8SYVJjCapNM9bjvk7ptpKh0= k8s.io/component-base v0.20.4/go.mod h1:t4p9EdiagbVCJKrQ1RsA5/V4rFQNDfRlevJajlGwgjI= k8s.io/component-base v0.20.6/go.mod h1:6f1MPBAeI+mvuts3sIdtpjljHWBQ2cIy38oBIWMYnrM= k8s.io/component-base v0.22.2 h1:vNIvE0AIrLhjX8drH0BgCNJcR4QZxMXcJzBsDplDx9M= k8s.io/component-base v0.22.2/go.mod h1:5Br2QhI9OTe79p+TzPe9JKNQYvEKbq9rTJDWllunGug= k8s.io/cri-api v0.0.0-20191107035106-03d130a7dc28/go.mod h1:9a7E6pmKLfuq8ZL31k2PDpgvSdyZfUOH9czlEmpblFk= k8s.io/cri-api v0.17.3/go.mod h1:X1sbHmuXhwaHs9xxYffLqJogVsnI+f6cPRcgPel7ywM= k8s.io/cri-api v0.20.1/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI= k8s.io/cri-api v0.20.4/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI= k8s.io/cri-api v0.20.6/go.mod h1:ew44AjNXwyn1s0U4xCKGodU7J1HzBeZ1MpGrpa5r8Yc= k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20190822140433-26a664648505/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20201113003025-83324d819ded/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.9.0 h1:D7HV+n1V57XeZ0m6tdRkfknthUaM06VFbWldOFh8kzM= k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= k8s.io/kube-openapi v0.0.0-20190816220812-743ec37842bf/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E= k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= k8s.io/kube-openapi v0.0.0-20211109043538-20434351676c/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= k8s.io/kube-openapi v0.0.0-20211110013926-83f114cd0513 h1:pbudjNtv90nOgR0/DUhPwKHnQ55Khz8+sNhJBIK7A5M= k8s.io/kube-openapi v0.0.0-20211110013926-83f114cd0513/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk= k8s.io/metrics v0.22.11 h1:XjuWs2rh0/WrT0Zi+ZS9k3utawKCHRtMFK8PjLKixSA= k8s.io/metrics v0.22.11/go.mod h1:zTDGPesMSDXU5HznhD3r/jrZn2L/RGrc95vw3vUtB6w= k8s.io/sample-controller v0.16.8/go.mod h1:aXlORS1ekU77qhGybB5t3JORDurzDpWgvMYxmCsiuos= k8s.io/utils v0.0.0-20190801114015-581e00157fb1/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20210111153108-fddb29f9d009/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20211116205334-6203023598ed h1:ck1fRPWPJWsMd8ZRFsWc6mh/zHp5fZ/shhbrgPUxDAE= k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw= modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk= modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= modernc.org/xc v1.0.0/go.mod h1:mRNCo0bvLjGhHO9WsyuKVU4q0ceiDDDoEeWDJHrNx8I= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.22/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= sigs.k8s.io/aws-iam-authenticator v0.5.3 h1:EyqQ/uxzbe2mDETZZmuMnv0xHITnyLhZfPlGb6Mma20= sigs.k8s.io/aws-iam-authenticator v0.5.3/go.mod h1:DIq7gy0lvnyaG88AgFyJzUVeix+ia5msHEp4RL0102I= sigs.k8s.io/controller-runtime v0.8.3/go.mod h1:U/l+DUopBc1ecfRZ5aviA9JDmGFQKvLf5YkZNx2e0sU= sigs.k8s.io/controller-runtime v0.10.3 h1:s5Ttmw/B4AuIbwrXD3sfBkXwnPMMWrqpVj4WRt1dano= sigs.k8s.io/controller-runtime v0.10.3/go.mod h1:CQp8eyUQZ/Q7PJvnIrB6/hgfTC1kBkGylwsLgOQi1WY= sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e h1:4Z09Hglb792X0kfOBBJUPFEyvVfQWrYT/l8h5EKA6JQ= sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.0.3/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= sigs.k8s.io/structured-merge-diff/v4 v4.2.1 h1:bKCqE9GvQ5tiVHn5rfn1r+yao3aLQEaLzkkmAkf+A6Y= sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= ================================================ FILE: images/activator/Dockerfile ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. FROM golang:1.17.3 as builder WORKDIR /workspace COPY go.mod go.mod COPY go.sum go.sum RUN go mod download COPY pkg pkg COPY cmd/activator cmd/activator WORKDIR /workspace/cmd/activator RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -o /workspace/bin/activator main.go FROM gcr.io/distroless/static:nonroot WORKDIR / COPY --from=builder /workspace/bin/activator . USER 65532:65532 ENTRYPOINT ["/activator"] ================================================ FILE: images/async-gateway/Dockerfile ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ARG TARGETARCH, TARGETOS FROM golang:1.17.3 as builder COPY go.mod go.sum /workspace/ WORKDIR /workspace RUN go mod download COPY pkg/consts pkg/consts COPY pkg/lib pkg/lib COPY pkg/async-gateway pkg/async-gateway COPY pkg/types pkg/types COPY cmd/async-gateway cmd/async-gateway RUN GO111MODULE=on CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -installsuffix cgo -o async-gateway ./cmd/async-gateway FROM alpine:3.15 RUN apk update && apk add ca-certificates COPY --from=builder /workspace/async-gateway /root/ RUN chmod +x /root/async-gateway ENTRYPOINT ["/root/async-gateway"] ================================================ FILE: images/autoscaler/Dockerfile ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. FROM golang:1.17.3 as builder WORKDIR /workspace COPY go.mod go.mod COPY go.sum go.sum RUN go mod download COPY pkg pkg COPY cmd/autoscaler cmd/autoscaler WORKDIR /workspace/cmd/autoscaler RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -o /workspace/bin/autoscaler main.go FROM gcr.io/distroless/static:nonroot WORKDIR / COPY --from=builder /workspace/bin/autoscaler . USER 65532:65532 ENTRYPOINT ["/autoscaler"] ================================================ FILE: images/cluster-autoscaler/Dockerfile ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ARG TARGETARCH, TARGETOS FROM golang:1.17.5 AS builder RUN git clone -b cluster-autoscaler-1.22.2-cortex --depth 1 https://github.com/cortexlabs/autoscaler /k8s.io/autoscaler WORKDIR /k8s.io/autoscaler/cluster-autoscaler RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build --installsuffix cgo -o cluster-autoscaler k8s.io/autoscaler/cluster-autoscaler \ && cp cluster-autoscaler /usr/local/bin FROM alpine:3.8 RUN apk add -U --no-cache ca-certificates && rm -rf /var/cache/apk/* COPY --from=builder /usr/local/bin/cluster-autoscaler . ================================================ FILE: images/controller-manager/Dockerfile ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. FROM golang:1.17.3 as builder WORKDIR /workspace COPY go.mod go.mod COPY go.sum go.sum RUN go mod download COPY pkg/config pkg/config COPY pkg/consts pkg/consts COPY pkg/crds pkg/crds COPY pkg/lib pkg/lib COPY pkg/types pkg/types COPY pkg/workloads pkg/workloads WORKDIR /workspace/pkg/crds RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -o /workspace/bin/manager main.go FROM gcr.io/distroless/static:nonroot WORKDIR / COPY --from=builder /workspace/bin/manager . USER 65532:65532 ENTRYPOINT ["/manager"] ================================================ FILE: images/dequeuer/Dockerfile ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ARG TARGETARCH, TARGETOS FROM golang:1.17.3 as builder COPY go.mod go.sum /workspace/ WORKDIR /workspace RUN go mod download COPY pkg/config pkg/config COPY pkg/consts pkg/consts COPY pkg/lib pkg/lib COPY pkg/dequeuer pkg/dequeuer COPY pkg/probe pkg/probe COPY pkg/types pkg/types COPY pkg/crds pkg/crds COPY pkg/workloads pkg/workloads COPY cmd/dequeuer cmd/dequeuer RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} GO111MODULE=on go build -o dequeuer ./cmd/dequeuer FROM gcr.io/distroless/static:nonroot WORKDIR / COPY --from=builder /workspace/dequeuer . USER nonroot:nonroot ENTRYPOINT ["/dequeuer"] ================================================ FILE: images/enqueuer/Dockerfile ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ARG TARGETARCH, TARGETOS FROM golang:1.17.3 as builder COPY go.mod go.sum /workspace/ WORKDIR /workspace RUN go mod download COPY pkg/config pkg/config COPY pkg/consts pkg/consts COPY pkg/lib pkg/lib COPY pkg/enqueuer pkg/enqueuer COPY pkg/types pkg/types COPY pkg/crds pkg/crds COPY pkg/workloads pkg/workloads COPY cmd/enqueuer cmd/enqueuer RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} GO111MODULE=on go build -o enqueuer ./cmd/enqueuer FROM gcr.io/distroless/static:nonroot WORKDIR / COPY --from=builder /workspace/enqueuer . USER nonroot:nonroot ENTRYPOINT ["/enqueuer"] ================================================ FILE: images/event-exporter/Dockerfile ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. FROM ghcr.io/opsgenie/kubernetes-event-exporter:v0.11 ================================================ FILE: images/fluent-bit/Dockerfile ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. FROM amazon/aws-for-fluent-bit:2.23.3 ================================================ FILE: images/grafana/Dockerfile ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. FROM grafana/grafana:8.0.4 ================================================ FILE: images/istio-pilot/Dockerfile ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. FROM docker.io/istio/pilot:1.11.8 ================================================ FILE: images/istio-proxy/Dockerfile ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. FROM docker.io/istio/proxyv2:1.11.8 ================================================ FILE: images/kube-rbac-proxy/Dockerfile ================================================ # Copyright 2021 Kube RBAC Proxy Authors rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Modifications Copyright 2022 Cortex Labs, Inc. ARG TARGETARCH, TARGETOS FROM golang:1.18 AS builder RUN git clone -b v0.13.0 --depth 1 https://github.com/brancz/kube-rbac-proxy /go/src/github.com/brancz/kube-rbac-proxy WORKDIR /go/src/github.com/brancz/kube-rbac-proxy RUN GO111MODULE=on CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build --installsuffix cgo -o kube-rbac-proxy github.com/brancz/kube-rbac-proxy \ && cp kube-rbac-proxy /usr/local/bin FROM alpine:3.8 RUN apk add -U --no-cache ca-certificates && rm -rf /var/cache/apk/* COPY --from=builder /usr/local/bin/kube-rbac-proxy . ENTRYPOINT ["./kube-rbac-proxy"] EXPOSE 8080 ================================================ FILE: images/kubexit/Dockerfile ================================================ # Copyright 2020 Karl Isenberg (https://github.com/karlkfi/kubexit) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Modifications Copyright 2022 Cortex Labs, Inc. ARG TARGETARCH, TARGETOS FROM golang:1.14 AS builder RUN mkdir /tmp/kubexit RUN git clone -b v0.1.0-cortex --depth 1 https://github.com/cortexlabs/kubexit.git /tmp/kubexit WORKDIR /tmp/kubexit RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -o kubexit ./cmd/kubexit FROM alpine:3.11 RUN apk --no-cache add ca-certificates tzdata COPY --from=builder /tmp/kubexit/kubexit /bin/ ENTRYPOINT ["kubexit"] ================================================ FILE: images/manager/Dockerfile ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. FROM python:3.7-alpine3.16 WORKDIR /root ENV PATH /root/.local/bin:$PATH ENV AWS_RETRY_MODE standard ENV AWS_MAX_ATTEMPTS 10 COPY manager/requirements.txt /root/requirements.txt RUN pip install --upgrade pip && \ pip install awscli --upgrade --user && \ pip install -r /root/requirements.txt && \ rm -rf /root/.cache/pip* RUN apk add --no-cache bash curl gettext jq openssl RUN curl --location "https://github.com/weaveworks/eksctl/releases/download/v0.107.0/eksctl_$(uname -s)_amd64.tar.gz" | tar xz -C /tmp && \ mv /tmp/eksctl /usr/local/bin RUN curl -o aws-iam-authenticator https://s3.us-west-2.amazonaws.com/amazon-eks/1.21.2/2021-07-05/bin/linux/amd64/aws-iam-authenticator && \ chmod +x ./aws-iam-authenticator && \ mv ./aws-iam-authenticator /usr/local/bin/aws-iam-authenticator RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.23.6/bin/linux/amd64/kubectl && \ chmod +x ./kubectl && \ mv ./kubectl /usr/local/bin/kubectl RUN curl -L "https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2Fv4.1.2/kustomize_v4.1.2_linux_amd64.tar.gz" | tar xz -C /tmp && \ mv /tmp/kustomize /usr/local/bin ENV ISTIO_VERSION 1.11.8 RUN curl -L https://istio.io/downloadIstio | sh - COPY manager /root COPY pkg/crds/config /root/config ENTRYPOINT ["/bin/bash"] ================================================ FILE: images/metrics-server/Dockerfile ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. FROM k8s.gcr.io/metrics-server/metrics-server:v0.6.1 ================================================ FILE: images/neuron-device-plugin/Dockerfile ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. FROM public.ecr.aws/neuron/neuron-device-plugin:1.8.2.0 ================================================ FILE: images/neuron-scheduler/Dockerfile ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. FROM public.ecr.aws/neuron/neuron-scheduler:1.8.2.0 ================================================ FILE: images/nvidia-device-plugin/Dockerfile ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. FROM nvidia/k8s-device-plugin:v0.11.0 ================================================ FILE: images/operator/Dockerfile ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. FROM golang:1.17.3 as builder RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.23.6/bin/linux/amd64/kubectl && \ mv ./kubectl /tmp/kubectl COPY go.mod go.sum /workspace/ WORKDIR /workspace RUN go mod download COPY pkg/config pkg/config COPY pkg/consts pkg/consts COPY pkg/lib pkg/lib COPY pkg/operator pkg/operator COPY pkg/types pkg/types COPY pkg/crds pkg/crds COPY pkg/workloads pkg/workloads COPY cmd/operator cmd/operator RUN GO111MODULE=on CGO_ENABLED=0 GOOS=linux go build -installsuffix cgo -o operator ./cmd/operator FROM alpine:3.15 COPY --from=builder /tmp/kubectl /usr/local/bin/kubectl RUN chmod +x /usr/local/bin/kubectl RUN apk --no-cache add ca-certificates bash COPY --from=builder /workspace/operator /root/ RUN chmod +x /root/operator EXPOSE 8888 ENTRYPOINT ["/root/operator"] ================================================ FILE: images/prometheus/Dockerfile ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. FROM quay.io/prometheus/prometheus:v2.28.0 ================================================ FILE: images/prometheus-config-reloader/Dockerfile ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. FROM quay.io/prometheus-operator/prometheus-config-reloader:v0.48.1 ================================================ FILE: images/prometheus-dcgm-exporter/Dockerfile ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. FROM nvcr.io/nvidia/k8s/dcgm-exporter:2.1.4-2.3.1-ubuntu18.04 # uncomment DCGM_FI_DEV_GPU_UTIL metric RUN sed -i '/DCGM_FI_DEV_GPU_UTIL/s/^# //g' /etc/dcgm-exporter/default-counters.csv ================================================ FILE: images/prometheus-kube-state-metrics/Dockerfile ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. FROM k8s.gcr.io/kube-state-metrics/kube-state-metrics:v2.1.0 ================================================ FILE: images/prometheus-node-exporter/Dockerfile ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. FROM quay.io/prometheus/node-exporter:v1.3.1 ================================================ FILE: images/prometheus-operator/Dockerfile ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. FROM quay.io/prometheus-operator/prometheus-operator:v0.48.1 ================================================ FILE: images/prometheus-statsd-exporter/Dockerfile ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. FROM prom/statsd-exporter:v0.22.4 ================================================ FILE: images/proxy/Dockerfile ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ARG TARGETARCH, TARGETOS FROM golang:1.17.3 as builder WORKDIR /workspace COPY go.mod go.mod COPY go.sum go.sum RUN go mod download COPY pkg pkg COPY cmd/proxy cmd/proxy WORKDIR /workspace/cmd/proxy RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} GO111MODULE=on go build -a -o /workspace/bin/proxy main.go FROM gcr.io/distroless/static:nonroot WORKDIR / COPY --from=builder /workspace/bin/proxy . USER 65532:65532 ENTRYPOINT ["/proxy"] ================================================ FILE: manager/check_cortex_version.sh ================================================ #!/bin/bash # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -e CORTEX_VERSION=master if [ "$CORTEX_VERSION" != "$CORTEX_CLI_VERSION" ]; then echo "error: your CLI version ($CORTEX_CLI_VERSION) doesn't match your Cortex manager image version ($CORTEX_VERSION); please update your CLI (pip install cortex==$CORTEX_VERSION) to match the version of your Cortex manager image" exit 1 fi ================================================ FILE: manager/cluster_config_env.py ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import sys import yaml import json from copy import deepcopy def export(base_key, value): if base_key.lower() == "cortex_tags": exportTags(value, "CORTEX_TAGS") exportTags( value, "CORTEX_OPERATOR_LOAD_BALANCER_TAGS", {"cortex.dev/load-balancer": "operator"} ) exportTags(value, "CORTEX_API_LOAD_BALANCER_TAGS", {"cortex.dev/load-balancer": "api"}) return if value is None: return elif type(value) is list: print(f'export {base_key.upper()}="{yaml.dump(value, default_flow_style=True).strip()}"') elif type(value) is dict: for key, child in value.items(): export(base_key + "_" + key, child) else: print(f'export {base_key.upper()}="{value}"') if base_key.lower().startswith("cortex_image_"): hub = value.rsplit("/", 1)[0] suffix = value.rsplit("/", 1)[1] if ":" in suffix: image = suffix.split(":")[0] tag = suffix.split(":")[1] else: image = suffix tag = "latest" print(f'export {base_key.upper()}_HUB="{hub}"') print(f'export {base_key.upper()}_IMAGE="{image}"') print(f'export {base_key.upper()}_TAG="{tag}"') def exportTags(tags, env_var_name, tag_overrides={}): tags = deepcopy(tags) for k, v in tag_overrides.items(): tags[k] = v inlined_tags = ",".join([f"{k}={v}" for k, v in tags.items()]) print(f"export {env_var_name}='{inlined_tags}'") print(f"export {env_var_name}_JSON='{json.dumps(tags)}'") for config_path in sys.argv[1:]: with open(config_path, "r") as f: config = yaml.safe_load(f) export("CORTEX", config) ================================================ FILE: manager/debug.sh ================================================ #!/bin/bash # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set +e CORTEX_VERSION_MINOR=master debug_out_path="$1" mkdir -p "$(dirname "$debug_out_path")" if ! eksctl utils describe-stacks --cluster=$CORTEX_CLUSTER_NAME --region=$CORTEX_REGION >/dev/null 2>&1; then echo "error: there is no cluster named \"$CORTEX_CLUSTER_NAME\" in $CORTEX_REGION; please update your configuration to point to an existing cortex cluster or create a cortex cluster with \`cortex cluster up\`" exit 1 fi eksctl utils write-kubeconfig --cluster=$CORTEX_CLUSTER_NAME --region=$CORTEX_REGION --verbose=0 | (grep -v "saved kubeconfig as" || true) out=$(kubectl get pods 2>&1 || true); if [[ "$out" == *"must be logged in to the server"* ]]; then echo "error: your aws iam user does not have access to this cluster; to grant access, see https://docs.cortexlabs.com/v/${CORTEX_VERSION_MINOR}/"; exit 1; fi echo -n "gathering cluster data" mkdir -p /cortex-debug/k8s for resource in pods pods.metrics nodes nodes.metrics daemonsets deployments hpa services virtualservices gateways ingresses configmaps jobs replicasets events; do kubectl describe $resource --all-namespaces > "/cortex-debug/k8s/${resource}" 2>&1 kubectl get $resource --all-namespaces > "/cortex-debug/k8s/${resource}-list" 2>&1 echo -n "." done mkdir -p /cortex-debug/logs kubectl get pods --all-namespaces -o json | jq '.items[] | . as $parent | $parent.spec.containers[]? | "kubectl logs -n \($parent.metadata.namespace) \($parent.metadata.name) \(.name) --timestamps --tail=10000 > /cortex-debug/logs/\($parent.metadata.namespace).\($parent.metadata.name).\(.name) 2>&1; echo -n ."' | xargs -n 1 bash -c kubectl get pods --all-namespaces -o json | jq '.items[] | . as $parent | $parent.spec.containers[]? | "kubectl logs -n \($parent.metadata.namespace) \($parent.metadata.name) \(.name) --previous --timestamps --tail=10000 > /cortex-debug/logs/\($parent.metadata.namespace).\($parent.metadata.name).\(.name).previous 2>&1; if [ $? -ne 0 ]; then rm /cortex-debug/logs/\($parent.metadata.namespace).\($parent.metadata.name).\(.name).previous; fi; echo -n ."' | xargs -n 1 bash -c echo -n "." kubectl get pods --all-namespaces -o json | jq '.items[] | . as $parent | $parent.spec.initContainers[]? | "kubectl logs -n \($parent.metadata.namespace) \($parent.metadata.name) \(.name) --timestamps --tail=10000 > /cortex-debug/logs/\($parent.metadata.namespace).\($parent.metadata.name).init.\(.name) 2>&1; echo -n ."' | xargs -n 1 bash -c kubectl get pods --all-namespaces -o json | jq '.items[] | . as $parent | $parent.spec.initContainers[]? | "kubectl logs -n \($parent.metadata.namespace) \($parent.metadata.name) \(.name) --previous --timestamps --tail=10000 > /cortex-debug/logs/\($parent.metadata.namespace).\($parent.metadata.name).init.\(.name).previous 2>&1; if [ $? -ne 0 ]; then rm /cortex-debug/logs/\($parent.metadata.namespace).\($parent.metadata.name).init.\(.name).previous; fi; echo -n ."' | xargs -n 1 bash -c echo -n "." kubectl top pods --all-namespaces --containers=true > "/cortex-debug/k8s/top_pods" 2>&1 echo -n "." kubectl top nodes > "/cortex-debug/k8s/top_nodes" 2>&1 echo -n "." mkdir -p /cortex-debug/aws/amis aws autoscaling describe-auto-scaling-groups --region=$CORTEX_REGION --output json > "/cortex-debug/aws/asgs" 2>&1 echo -n "." aws autoscaling describe-scaling-activities --max-items 1000 --region=$CORTEX_REGION --output json > "/cortex-debug/aws/asg-activities" 2>&1 echo -n "." aws ec2 describe-instances --filters Name=tag:cortex.dev/cluster-name,Values=$CORTEX_CLUSTER_NAME --region=$CORTEX_REGION --output json > "/cortex-debug/aws/instances" 2>&1 echo -n "." aws ec2 describe-instance-status --include-all-instances --region=$CORTEX_REGION --output json > "/cortex-debug/aws/instance-statuses" 2>&1 echo -n "." aws ec2 describe-instances --filters Name=tag:cortex.dev/cluster-name,Values=$CORTEX_CLUSTER_NAME --region=$CORTEX_REGION --output json | jq "[.Reservations[].Instances[].ImageId] | unique | .[] | \"aws ec2 describe-images --image-ids \(.) --region=$CORTEX_REGION --output json > /cortex-debug/aws/amis/\(.) 2>&1\"" | xargs -n 1 bash -c echo -n "." python get_operator_load_balancer_state.py > "/cortex-debug/aws/operator_load_balancer_state" 2>&1 echo -n "." python get_api_load_balancer_state.py > "/cortex-debug/aws/api_load_balancer_state" 2>&1 echo -n "." python get_operator_target_group_status.py > "/cortex-debug/aws/operator_load_balancer_target_group_status" 2>&1 echo -n "." mkdir -p /cortex-debug/misc operator_endpoint=$(kubectl -n=istio-system get service ingressgateway-operator -o json 2>/dev/null | tr -d '[:space:]' | sed 's/.*{\"hostname\":\"\(.*\)\".*/\1/') echo "$operator_endpoint" > /cortex-debug/misc/operator_endpoint if [ "$operator_endpoint" == "" ]; then echo "unable to get operator endpoint" > /cortex-debug/misc/operator_curl else curl -sv --max-time 5 "${operator_endpoint}/verifycortex" > /cortex-debug/misc/operator_curl 2>&1 fi echo -n "." (cd / && tar -czf cortex-debug.tgz cortex-debug) mv /cortex-debug.tgz $debug_out_path echo " ✓" ================================================ FILE: manager/generate_eks.py ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import json import click from collections import namedtuple import re import yaml K8S_VERSION = "1.22" ParsedInstanceType = namedtuple( "ParsedInstanceType", ["family", "generation", "capabilities", "size"] ) def parse_instance_type(instance_type: str) -> ParsedInstanceType: parts = instance_type.split(".") if len(parts) != 2: raise ValueError(f"unexpected invalid instance type: {instance_type}") prefix = parts[0] size = parts[1] family = re.search("[a-z]*", prefix.lower()).group() generation = re.sub("\D", "", prefix.lower()) capabilities = prefix[len(family) + len(generation) :] return ParsedInstanceType(family, generation, capabilities, size) # kubelet config schema: # https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/kubelet/config/v1beta1/types.go def default_nodegroup(cluster_config): partition = "aws" if "us-gov" in cluster_config["region"]: partition = "aws-us-gov" return { "iam": { "withAddonPolicies": {"autoScaler": True}, "attachPolicyARNs": [ f"arn:{partition}:iam::aws:policy/AmazonEKSWorkerNodePolicy", f"arn:{partition}:iam::aws:policy/AmazonEKS_CNI_Policy", f"arn:{partition}:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly", f"arn:{partition}:iam::aws:policy/ElasticLoadBalancingFullAccess", cluster_config["cortex_policy_arn"], ] + cluster_config.get("iam_policy_arns", []), }, "privateNetworking": cluster_config.get("subnet_visibility", "public") != "public", "kubeletExtraConfig": { "kubeReserved": {"cpu": "150m", "memory": "300Mi", "ephemeral-storage": "1Gi"}, "kubeReservedCgroup": "/kube-reserved", "systemReserved": {"cpu": "150m", "memory": "300Mi", "ephemeral-storage": "1Gi"}, "evictionHard": {"memory.available": "200Mi", "nodefs.available": "5%"}, "registryPullQPS": 10, }, "preBootstrapCommands": [ "sudo yum install -y ipvsadm", "sudo modprobe ip_vs", # IP virtual server "sudo modprobe ip_vs_rr", # round robing load balancer "sudo modprobe ip_vs_lc", # least connected load balancer "sudo modprobe ip_vs_wrr", # weighted round robin load balancer "sudo modprobe ip_vs_sh", # source-hashing load balancer "sudo modprobe nf_conntrack_ipv4", ], "overrideBootstrapCommand": "\n".join( [ "#!/bin/bash", "source /var/lib/cloud/scripts/eksctl/bootstrap.helper.sh", f"/etc/eks/bootstrap.sh {cluster_config['cluster_name']} --container-runtime dockerd --kubelet-extra-args \"--node-labels=${{NODE_LABELS}} --register-with-taints=${{NODE_TAINTS}}\"", ] ), } def merge_override(a, b): "merges b into a" for key in b: if key in a: if isinstance(a[key], dict) and isinstance(b[key], dict): merge_override(a[key], b[key]) elif isinstance(a[key], list) and isinstance(b[key], list): a[key] += b[key] else: a[key] = b[key] else: a[key] = b[key] return a def apply_worker_settings(nodegroup, config): worker_settings = { "name": "cx-wd-" + config["name"], "asgSuspendProcesses": ["AZRebalance"], "labels": {"workload": "true"}, "taints": [ { "key": "workload", "value": "true", "effect": "NoSchedule", }, ], "tags": { "k8s.io/cluster-autoscaler/enabled": "true", "k8s.io/cluster-autoscaler/node-template/label/workload": "true", }, } return merge_override(nodegroup, worker_settings) def apply_clusterconfig(nodegroup, config): clusterconfig_settings = { "instanceType": config["instance_type"], "volumeSize": config["instance_volume_size"], "minSize": config["min_instances"], "maxSize": config["max_instances"], "volumeType": config["instance_volume_type"], "desiredCapacity": 1 if config["min_instances"] == 0 else config["min_instances"], } # add iops to settings if volume_type is io1/gp3 if config["instance_volume_type"] in ["io1", "gp3"]: clusterconfig_settings["volumeIOPS"] = config["instance_volume_iops"] if config["instance_volume_type"] == "gp3": clusterconfig_settings["volumeThroughput"] = config["instance_volume_throughput"] return merge_override(nodegroup, clusterconfig_settings) def apply_spot_settings(nodegroup, config): spot_settings = { "name": "cx-ws-" + config["name"], "instanceType": "mixed", "instancesDistribution": { "instanceTypes": config["spot_config"]["instance_distribution"], "onDemandBaseCapacity": config["spot_config"]["on_demand_base_capacity"], "onDemandPercentageAboveBaseCapacity": config["spot_config"][ "on_demand_percentage_above_base_capacity" ], "maxPrice": config["spot_config"]["max_price"], "spotInstancePools": config["spot_config"]["instance_pools"], }, "labels": {"lifecycle": "Ec2Spot"}, } return merge_override(nodegroup, spot_settings) def apply_gpu_settings(nodegroup): gpu_settings = { "tags": { "k8s.io/cluster-autoscaler/node-template/label/nvidia.com/gpu": "true", "k8s.io/cluster-autoscaler/node-template/taint/dedicated": "nvidia.com/gpu=true", "k8s.io/cluster-autoscaler/node-template/label/k8s.amazonaws.com/accelerator": "true", # accepted values are GPU type such as nvidia-tesla-k80 but using "true" as a placeholder for now because the value doesn't matter for AWS cluster autoscaler }, "labels": { "nvidia.com/gpu": "true", "k8s.amazonaws.com/accelerator": "true", # accepted values are GPU type such as nvidia-tesla-k80 but using "true" as a placeholder for now because the value doesn't matter for AWS cluster autoscaler }, "taints": [ { "key": "nvidia.com/gpu", "value": "true", "effect": "NoSchedule", }, ], } return merge_override(nodegroup, gpu_settings) def is_gpu(instance_type): parsed_instance_type = parse_instance_type(instance_type) return parsed_instance_type.family in ["g", "p"] def apply_inf_settings(nodegroup, config): instance_type = config["instance_type"] num_chips, num_hugepages = get_inf_resources(instance_type) inf_settings = { "tags": { "k8s.io/cluster-autoscaler/node-template/label/aws.amazon.com/neuron": "true", "k8s.io/cluster-autoscaler/node-template/taint/dedicated": "aws.amazon.com/neuron=true", "k8s.io/cluster-autoscaler/node-template/resources/aws.amazon.com/neuron": str( num_chips ), "k8s.io/cluster-autoscaler/node-template/resources/hugepages-2Mi": num_hugepages, }, "labels": {"aws.amazon.com/neuron": "true"}, "taints": [ { "key": "aws.amazon.com/neuron", "value": "true", "effect": "NoSchedule", }, ], } return merge_override(nodegroup, inf_settings) def is_inf(instance_type): parsed_instance_type = parse_instance_type(instance_type) return parsed_instance_type.family == "inf" def get_inf_resources(instance_type): num_chips = 0 if instance_type in ["inf1.xlarge", "inf1.2xlarge"]: num_chips = 1 elif instance_type == "inf1.6xlarge": num_chips = 4 elif instance_type == "inf1.24xlarge": num_chips = 16 return num_chips, f"{128 * num_chips}Mi" def is_arm64(instance_type: str): parsed_instance_type = parse_instance_type(instance_type) return parsed_instance_type.family == "a" or "g" in parsed_instance_type.capabilities def get_all_worker_nodegroups(ami_map: dict, cluster_config: dict) -> list: """ Gets all node groups in EKS-dict format. """ worker_nodegroups = [] for ng in cluster_config["node_groups"]: worker_nodegroups.append(get_worker_nodegroup(ami_map, ng, cluster_config)) return worker_nodegroups def get_worker_nodegroup(ami_map: dict, nodegroup_config: dict, cluster_config: dict) -> dict: """ Converts Cortex-dict nodegroup config to EKS-dict format. """ worker_nodegroup = default_nodegroup(cluster_config) worker_nodegroup["ami"] = get_ami(ami_map, nodegroup_config["instance_type"]) apply_worker_settings(worker_nodegroup, nodegroup_config) apply_clusterconfig(worker_nodegroup, nodegroup_config) if nodegroup_config["spot"]: apply_spot_settings(worker_nodegroup, nodegroup_config) if is_gpu(nodegroup_config["instance_type"]): apply_gpu_settings(worker_nodegroup) if is_inf(nodegroup_config["instance_type"]): apply_inf_settings(worker_nodegroup, nodegroup_config) return worker_nodegroup def get_nodegroup_config_by_name(cluster_config: dict, ng_name: str) -> dict: """ Gets a nodegroup in Cortex-dict format from Cortex cluster config. """ for ng in cluster_config["node_groups"]: if ng["name"] == ng_name: return ng def get_empty_eks_nodegroup(name: str) -> dict: """ Gets an empty nodegroup in EKS-dict format that only has the Cortex nodegroup name filled out. """ return {"name": name} def get_ami(ami_map: dict, instance_type: str) -> str: if is_gpu(instance_type) or is_inf(instance_type): return ami_map["accelerated_amd64"] if is_arm64(instance_type): return ami_map["cpu_arm64"] return ami_map["cpu_amd64"] @click.command() @click.argument("cluster-config_file", type=click.File("r")) @click.argument("ami-json-file", type=click.File("r")) @click.option( "--add-cortex-node-groups", type=str, help="specific cortex nodegroups to add to the generated eks file; use this for existing clusters", ) @click.option( "--remove-eks-node-groups", type=str, help="specific eks nodegroup stacks to add to the generated eks file; use this for existing clusters", ) def generate_eks( cluster_config_file, ami_json_file, add_cortex_node_groups: str, remove_eks_node_groups: str ): cluster_config = yaml.safe_load(cluster_config_file) region = cluster_config["region"] name = cluster_config["cluster_name"] prometheus_instance_type = cluster_config["prometheus_instance_type"] ami_map = json.load(ami_json_file)[K8S_VERSION][region] eks = { "apiVersion": "eksctl.io/v1alpha5", "kind": "ClusterConfig", "metadata": { "name": name, "region": region, "version": K8S_VERSION, }, } if add_cortex_node_groups: node_group_names = add_cortex_node_groups.split(",") eks["nodeGroups"] = [] for node_group_name in node_group_names: nodegroup_config = get_nodegroup_config_by_name(cluster_config, node_group_name) eks["nodeGroups"].append( get_worker_nodegroup(ami_map, nodegroup_config, cluster_config) ) click.echo(yaml.dump(eks, Dumper=IgnoreAliases, default_flow_style=False, default_style="")) return if remove_eks_node_groups: stacks_names = remove_eks_node_groups.split(",") eks["nodeGroups"] = [] for stack_name in stacks_names: eks["nodeGroups"].append(get_empty_eks_nodegroup(stack_name)) click.echo(yaml.dump(eks, Dumper=IgnoreAliases, default_flow_style=False, default_style="")) return operator_nodegroup = default_nodegroup(cluster_config) operator_settings = { "ami": get_ami(ami_map, "t3.medium"), "name": "cx-operator", "instanceType": "t3.medium", "minSize": 2, "maxSize": 25, "desiredCapacity": 2, "volumeType": "gp3", "volumeSize": 20, "volumeIOPS": 3000, "volumeThroughput": 125, "labels": {"operator": "true"}, } operator_nodegroup = merge_override(operator_nodegroup, operator_settings) prometheus_nodegroup = default_nodegroup(cluster_config) prometheus_settings = { "ami": get_ami(ami_map, prometheus_instance_type), "name": "cx-prometheus", "instanceType": prometheus_instance_type, "minSize": 1, "maxSize": 1, "desiredCapacity": 1, "volumeType": "gp3", "volumeSize": 20, "volumeIOPS": 3000, "volumeThroughput": 125, "labels": {"prometheus": "true"}, "taints": [ { "key": "prometheus", "value": "true", "effect": "NoSchedule", }, ], } prometheus_nodegroup = merge_override(prometheus_nodegroup, prometheus_settings) worker_nodegroups = get_all_worker_nodegroups(ami_map, cluster_config) nat_gateway = "Disable" if cluster_config["nat_gateway"] == "single": nat_gateway = "Single" elif cluster_config["nat_gateway"] == "highly_available": nat_gateway = "HighlyAvailable" eks = { "apiVersion": "eksctl.io/v1alpha5", "kind": "ClusterConfig", "metadata": { "name": name, "region": region, "version": K8S_VERSION, "tags": cluster_config["tags"], }, "vpc": {"nat": {"gateway": nat_gateway}}, "nodeGroups": [operator_nodegroup, prometheus_nodegroup] + worker_nodegroups, "addons": [ { "name": "vpc-cni", "version": "1.11.3", }, ], } if ( len(cluster_config.get("availability_zones", [])) > 0 and len(cluster_config.get("subnets", [])) == 0 ): eks["availabilityZones"] = cluster_config["availability_zones"] if len(cluster_config.get("subnets", [])) > 0: eks_subnet_configs = {} for subnet_config in cluster_config["subnets"]: eks_subnet_configs[subnet_config["availability_zone"]] = { "id": subnet_config["subnet_id"] } if cluster_config.get("subnet_visibility", "public") == "private": eks["vpc"]["subnets"] = {"private": eks_subnet_configs} else: eks["vpc"]["subnets"] = {"public": eks_subnet_configs} if cluster_config.get("vpc_cidr", "") != "": eks["vpc"]["cidr"] = cluster_config["vpc_cidr"] click.echo(yaml.dump(eks, Dumper=IgnoreAliases, default_flow_style=False, default_style="")) class IgnoreAliases(yaml.Dumper): """By default, yaml dumper tries to compress yaml by annotating collections (lists and maps) and replacing subsequent identical collections with aliases. This class overrides the default behaviour to preserve the duplication of arrays. """ def ignore_aliases(self, data): return True if __name__ == "__main__": generate_eks() ================================================ FILE: manager/get_api_load_balancer_state.py ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import boto3 import os from helpers import get_api_load_balancer_v2, get_api_load_balancer, get_api_load_balancer_health def get_api_load_balancer_state(): cluster_name = os.environ["CORTEX_CLUSTER_NAME"] region = os.environ["CORTEX_REGION"] load_balancer_type = os.environ["CORTEX_API_LOAD_BALANCER_TYPE"] if load_balancer_type == "nlb": client_elbv2 = boto3.client("elbv2", region_name=region) load_balancer = get_api_load_balancer_v2(cluster_name, client_elbv2) return load_balancer["State"]["Code"] else: client_elb = boto3.client("elb", region_name=region) load_balancer = get_api_load_balancer(cluster_name, client_elb) return get_api_load_balancer_health(load_balancer["LoadBalancerName"], client_elb) if __name__ == "__main__": print(get_api_load_balancer_state(), end="") ================================================ FILE: manager/get_operator_load_balancer_state.py ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import boto3 import os from helpers import get_operator_load_balancer_v2 def get_operator_load_balancer_state(): cluster_name = os.environ["CORTEX_CLUSTER_NAME"] region = os.environ["CORTEX_REGION"] client_elbv2 = boto3.client("elbv2", region_name=region) load_balancer = get_operator_load_balancer_v2(cluster_name, client_elbv2) return load_balancer["State"]["Code"] if __name__ == "__main__": print(get_operator_load_balancer_state(), end="") ================================================ FILE: manager/get_operator_target_group_status.py ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import boto3 import os import json from helpers import get_operator_load_balancer_v2 def get_operator_target_group_status(): cluster_name = os.environ["CORTEX_CLUSTER_NAME"] region = os.environ["CORTEX_REGION"] client_elbv2 = boto3.client("elbv2", region_name=region) load_balancer_arn = get_operator_load_balancer_v2(cluster_name, client_elbv2)["LoadBalancerArn"] target_group_arn = get_load_balancer_https_target_group_arn(load_balancer_arn, client_elbv2) return get_target_health(target_group_arn, client_elbv2) def get_load_balancer_https_target_group_arn(load_balancer_arn, client_elbv2): paginator = client_elbv2.get_paginator("describe_listeners") for listener_page in paginator.paginate(LoadBalancerArn=load_balancer_arn): for listener in listener_page["Listeners"]: if listener["Port"] == 443: return listener["DefaultActions"][0]["TargetGroupArn"] raise Exception( f"unable to find https target group for operator load balancer ({load_balancer_arn})" ) def get_target_health(target_group_arn, client_elbv2): response = client_elbv2.describe_target_health(TargetGroupArn=target_group_arn) for health_description in response["TargetHealthDescriptions"]: if health_description["TargetHealth"]["State"] == "healthy": return "healthy" return json.dumps(response["TargetHealthDescriptions"]) if __name__ == "__main__": print(get_operator_target_group_status(), end="") ================================================ FILE: manager/helpers.py ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. def get_operator_load_balancer_v2(cluster_name, client_elbv2): return _get_load_balancer_v2("operator", cluster_name, client_elbv2) def get_api_load_balancer_v2(cluster_name, client_elbv2): return _get_load_balancer_v2("api", cluster_name, client_elbv2) def get_api_load_balancer(cluster_name, client_elb): return _get_load_balancer("api", cluster_name, client_elb) def get_api_load_balancer_health(load_balancer_name, client_elb): instance_health = client_elb.describe_instance_health( LoadBalancerName=load_balancer_name, ) for instance_state in instance_health["InstanceStates"]: if instance_state["State"] != "InService": return "inactive" return "active" def _get_load_balancer_v2(load_balancer_tag, cluster_name, client_elbv2): paginator = client_elbv2.get_paginator("describe_load_balancers") for load_balancer_page in paginator.paginate(PaginationConfig={"PageSize": 20}): load_balancers = { load_balancer["LoadBalancerArn"]: load_balancer for load_balancer in load_balancer_page["LoadBalancers"] } tag_descriptions = client_elbv2.describe_tags(ResourceArns=list(load_balancers.keys()))[ "TagDescriptions" ] for tag_description in tag_descriptions: foundClusterNameTag = False foundLoadBalancerTag = False for tags in tag_description["Tags"]: if tags["Key"] == "cortex.dev/cluster-name" and tags["Value"] == cluster_name: foundClusterNameTag = True if tags["Key"] == "cortex.dev/load-balancer" and tags["Value"] == load_balancer_tag: foundLoadBalancerTag = True if foundClusterNameTag and foundLoadBalancerTag: return load_balancers[tag_description["ResourceArn"]] raise Exception(f"unable to find {load_balancer_tag} load balancer") def _get_load_balancer(load_balancer_tag, cluster_name, client_elb): paginator = client_elb.get_paginator("describe_load_balancers") for load_balancer_page in paginator.paginate(PaginationConfig={"PageSize": 20}): load_balancers = { load_balancer["LoadBalancerName"]: load_balancer for load_balancer in load_balancer_page["LoadBalancerDescriptions"] } tag_descriptions = client_elb.describe_tags(LoadBalancerNames=list(load_balancers.keys()))[ "TagDescriptions" ] for tag_description in tag_descriptions: foundClusterNameTag = False foundLoadBalancerTag = False for tags in tag_description["Tags"]: if tags["Key"] == "cortex.dev/cluster-name" and tags["Value"] == cluster_name: foundClusterNameTag = True if tags["Key"] == "cortex.dev/load-balancer" and tags["Value"] == load_balancer_tag: foundLoadBalancerTag = True if foundClusterNameTag and foundLoadBalancerTag: return load_balancers[tag_description["LoadBalancerName"]] raise Exception(f"unable to find {load_balancer_tag} load balancer") ================================================ FILE: manager/install.sh ================================================ #!/bin/bash # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eo pipefail export CORTEX_VERSION=master export CORTEX_VERSION_MINOR=master EKSCTL_CLUSTER_TIMEOUT=45m EKSCTL_NODEGROUP_TIMEOUT=30m mkdir /workspace arg1="$1" function main() { if [ "$arg1" = "--configure" ]; then cluster_configure else cluster_up fi } function cluster_up() { create_eks echo -n "￮ updating cluster configuration " setup_namespaces setup_configmap echo "✓" echo -n "￮ configuring networking (this will take a few minutes) " setup_ipvs setup_istio python render_template.py $CORTEX_CLUSTER_CONFIG_FILE manifests/apis.yaml.j2 | kubectl apply -f - >/dev/null echo "✓" echo -n "￮ configuring autoscaling " python render_template.py $CORTEX_CLUSTER_CONFIG_FILE manifests/autoscaler.yaml.j2 | kubectl apply -f - >/dev/null python render_template.py $CORTEX_CLUSTER_CONFIG_FILE manifests/activator.yaml.j2 | kubectl apply -f - >/dev/null python render_template.py $CORTEX_CLUSTER_CONFIG_FILE manifests/cluster-autoscaler.yaml.j2 | kubectl apply -f - >/dev/null echo "✓" echo -n "￮ configuring async gateway " python render_template.py $CORTEX_CLUSTER_CONFIG_FILE manifests/async-gateway.yaml.j2 | kubectl apply -f - >/dev/null echo "✓" echo -n "￮ configuring logging " python render_template.py $CORTEX_CLUSTER_CONFIG_FILE manifests/fluent-bit.yaml.j2 | kubectl apply -f - >/dev/null envsubst < manifests/event-exporter.yaml | kubectl apply -f - >/dev/null echo "✓" echo -n "￮ configuring metrics " envsubst < manifests/metrics-server.yaml | kubectl apply -f - >/dev/null setup_prometheus setup_grafana echo "✓" echo -n "￮ configuring gpu support (for nodegroups that may require it) " envsubst < manifests/nvidia.yaml | kubectl apply -f - >/dev/null NVIDIA_COM_GPU_VALUE=true envsubst < manifests/prometheus-dcgm-exporter.yaml | kubectl apply -f - >/dev/null echo "✓" echo -n "￮ configuring inf support (for nodegroups that may require it) " envsubst < manifests/inferentia.yaml | kubectl apply -f - >/dev/null echo "✓" restart_operator start_controller_manager validate_cortex echo -e "\ncortex is ready!" if [ "$CORTEX_OPERATOR_LOAD_BALANCER_SCHEME" == "internal" ]; then echo -e "\nnote: you will need to configure VPC Peering to connect to your cluster: https://docs.cortexlabs.com/v/${CORTEX_VERSION_MINOR}/" fi print_endpoints } function cluster_configure() { check_eks resize_nodegroups add_nodegroups remove_nodegroups update_networking echo -n "￮ updating cluster configuration " setup_configmap echo "✓" # this is necessary since max_instances may have been updated echo -n "￮ configuring autoscaling " python render_template.py $CORTEX_CLUSTER_CONFIG_FILE manifests/cluster-autoscaler.yaml.j2 | kubectl apply -f - >/dev/null echo "✓" restart_controller_manager restart_operator validate_cortex echo -e "\ncortex is ready!" print_endpoints } # creates the eks cluster and configures kubectl function create_eks() { set +e cluster_info=$(eksctl get cluster --name=$CORTEX_CLUSTER_NAME --region=$CORTEX_REGION -o json 2> /dev/null) cluster_info_exit_code=$? set -e # cluster already exists if [ $cluster_info_exit_code -eq 0 ]; then set +e # cluster statuses: https://github.com/aws/aws-sdk-go/blob/master/service/eks/api.go#L6883 cluster_status=$(echo "$cluster_info" | jq -r 'first | .Status') set -e if [ "$cluster_status" == "ACTIVE" ]; then echo "error: there is already a cluster named \"$CORTEX_CLUSTER_NAME\" in $CORTEX_REGION" exit 1 elif [ "$cluster_status" == "DELETING" ]; then echo "error: your cortex cluster named \"$CORTEX_CLUSTER_NAME\" in $CORTEX_REGION is currently spinning down; please try again once it is completely deleted (may take a few minutes)" exit 1 elif [ "$cluster_status" == "CREATING" ]; then echo "error: your cortex cluster named \"$CORTEX_CLUSTER_NAME\" in $CORTEX_REGION is currently spinning up; please try again once it is ready" exit 1 elif [ "$cluster_status" == "UPDATING" ]; then echo "error: your cortex cluster named \"$CORTEX_CLUSTER_NAME\" in $CORTEX_REGION is currently updating; please try again once it is ready" exit 1 elif [ "$cluster_status" == "FAILED" ]; then echo "error: your cortex cluster named \"$CORTEX_CLUSTER_NAME\" in $CORTEX_REGION is failed; delete it with \`eksctl delete cluster --name=$CORTEX_CLUSTER_NAME --region=$CORTEX_REGION --disable-nodegroup-eviction\` and try again" exit 1 else # cluster exists, but is has an unknown status (unexpected) echo "error: there is already a cluster named \"$CORTEX_CLUSTER_NAME\" in $CORTEX_REGION (status: ${cluster_status})" exit 1 fi fi echo -e "￮ spinning up the cluster (this will take about 30 minutes) ...\n" python generate_eks.py $CORTEX_CLUSTER_CONFIG_FILE manifests/ami.json > /workspace/eks.yaml eksctl create cluster --timeout=$EKSCTL_CLUSTER_TIMEOUT --install-neuron-plugin=false --install-nvidia-plugin=false -f /workspace/eks.yaml echo write_kubeconfig } # checks that the eks cluster is active and configures kubectl function check_eks() { set +e cluster_info=$(eksctl get cluster --name=$CORTEX_CLUSTER_NAME --region=$CORTEX_REGION -o json 2> /dev/null) cluster_info_exit_code=$? set -e # no cluster if [ $cluster_info_exit_code -ne 0 ]; then echo "error: there is no cluster named \"$CORTEX_CLUSTER_NAME\" in $CORTEX_REGION; please update your configuration to point to an existing cortex cluster or create a cortex cluster with \`cortex cluster up\`" exit 1 fi set +e # cluster statuses: https://github.com/aws/aws-sdk-go/blob/master/service/eks/api.go#L6883 cluster_status=$(echo "$cluster_info" | jq -r 'first | .Status') set -e if [ "$cluster_status" == "DELETING" ]; then echo "error: your cortex cluster named \"$CORTEX_CLUSTER_NAME\" in $CORTEX_REGION is currently spinning down; please try again once it is completely deleted (may take a few minutes)" exit 1 elif [ "$cluster_status" == "CREATING" ]; then echo "error: your cortex cluster named \"$CORTEX_CLUSTER_NAME\" in $CORTEX_REGION is currently spinning up; please try again once it is ready" exit 1 elif [ "$cluster_status" == "UPDATING" ]; then echo "error: your cortex cluster named \"$CORTEX_CLUSTER_NAME\" in $CORTEX_REGION is currently updating; please try again once it is ready" exit 1 elif [ "$cluster_status" == "FAILED" ]; then echo "error: your cortex cluster named \"$CORTEX_CLUSTER_NAME\" in $CORTEX_REGION is failed; delete it with \`eksctl delete cluster --name=$CORTEX_CLUSTER_NAME --region=$CORTEX_REGION --disable-nodegroup-eviction\` and try again" exit 1 fi # cluster status is ACTIVE or unknown (in which case we'll assume things are ok instead of erroring) write_kubeconfig } function write_kubeconfig() { eksctl utils write-kubeconfig --cluster=$CORTEX_CLUSTER_NAME --region=$CORTEX_REGION --verbose=0 | (grep -v "saved kubeconfig as" || true) out=$(kubectl get pods 2>&1 || true); if [[ "$out" == *"must be logged in to the server"* ]]; then echo "error: your aws iam user does not have access to this cluster; to grant access, see https://docs.cortexlabs.com/v/${CORTEX_VERSION_MINOR}/"; exit 1; fi } function setup_namespaces() { # doing a patch to prevent getting the kubectl.kubernetes.io/last-applied-configuration annotation warning kubectl patch namespace default -p '{"metadata": {"labels": {"istio-discovery": "enabled"}}}' >/dev/null kubectl apply -f manifests/namespaces.yaml >/dev/null } function setup_configmap() { envsubst < manifests/default_cortex_cli_config.yaml > tmp_cli_config.yaml kubectl -n=default create configmap 'client-config' \ --from-file='cli.yaml'=tmp_cli_config.yaml \ -o yaml --dry-run=client | kubectl apply -f - >/dev/null rm tmp_cli_config.yaml kubectl -n=default create configmap 'cluster-config' \ --from-file='cluster.yaml'=$CORTEX_CLUSTER_CONFIG_FILE \ -o yaml --dry-run=client | kubectl apply -f - >/dev/null kubectl -n=default create configmap 'env-vars' \ --from-literal='CORTEX_VERSION'=$CORTEX_VERSION \ --from-literal='CORTEX_REGION'=$CORTEX_REGION \ --from-literal='AWS_DEFAULT_REGION'=$CORTEX_REGION \ --from-literal='AWS_RETRY_MODE'="standard" \ --from-literal='AWS_MAX_ATTEMPTS'="5" \ --from-literal='CORTEX_TELEMETRY_DISABLE'=$CORTEX_TELEMETRY_DISABLE \ --from-literal='CORTEX_TELEMETRY_SENTRY_DSN'=$CORTEX_TELEMETRY_SENTRY_DSN \ --from-literal='CORTEX_TELEMETRY_SEGMENT_WRITE_KEY'=$CORTEX_TELEMETRY_SEGMENT_WRITE_KEY \ --from-literal='CORTEX_DEV_DEFAULT_IMAGE_REGISTRY'=$CORTEX_DEV_DEFAULT_IMAGE_REGISTRY \ -o yaml --dry-run=client | kubectl apply -f - >/dev/null } function setup_prometheus() { envsubst < manifests/prometheus-operator.yaml | kubectl apply -f - >/dev/null envsubst < manifests/prometheus-statsd-exporter.yaml | kubectl apply -f - >/dev/null envsubst < manifests/prometheus-kubelet-exporter.yaml | kubectl apply -f - >/dev/null envsubst < manifests/prometheus-kube-state-metrics.yaml | kubectl apply -f - >/dev/null envsubst < manifests/prometheus-node-exporter.yaml | kubectl apply -f - >/dev/null envsubst < manifests/prometheus-monitoring.yaml | kubectl apply -f - >/dev/null python render_template.py $CORTEX_CLUSTER_CONFIG_FILE manifests/prometheus-additional-scrape-configs.yaml.j2 > prometheus-additional-scrape-configs.yaml if ! kubectl get secret -n prometheus additional-scrape-configs >/dev/null 2>&1; then kubectl create secret generic -n prometheus additional-scrape-configs --from-file=prometheus-additional-scrape-configs.yaml > /dev/null fi } function setup_grafana() { kubectl apply -f manifests/grafana/grafana-dashboard-realtime.yaml >/dev/null kubectl apply -f manifests/grafana/grafana-dashboard-async.yaml >/dev/null kubectl apply -f manifests/grafana/grafana-dashboard-batch.yaml >/dev/null kubectl apply -f manifests/grafana/grafana-dashboard-task.yaml >/dev/null kubectl apply -f manifests/grafana/grafana-dashboard-cluster.yaml >/dev/null kubectl apply -f manifests/grafana/grafana-dashboard-nodes.yaml >/dev/null if [ "$CORTEX_DEV_ADD_CONTROL_PLANE_DASHBOARD" = "true" ]; then kubectl apply -f manifests/grafana/grafana-dashboard-control-plane.yaml >/dev/null fi python render_template.py $CORTEX_CLUSTER_CONFIG_FILE manifests/grafana/grafana.yaml.j2 | kubectl apply -f - >/dev/null } function restart_operator() { echo -n "￮ starting operator " kubectl -n=default delete --ignore-not-found=true --grace-period=10 deployment operator >/dev/null 2>&1 printed_dot="false" until [ "$(kubectl -n=default get pods -l workloadID=operator -o json | jq -j '.items | length')" -eq "0" ]; do echo -n "."; printed_dot="true"; sleep 2; done python render_template.py $CORTEX_CLUSTER_CONFIG_FILE manifests/operator.yaml.j2 > /workspace/operator.yaml kubectl apply -f /workspace/operator.yaml >/dev/null if [ "$printed_dot" == "true" ]; then echo " ✓"; else echo "✓"; fi } function start_controller_manager() { echo -n "￮ starting controller manager " kustomize build config/default | kubectl delete --ignore-not-found=true -f - >/dev/null cd config/manager \ && kustomize edit set image controller=${CORTEX_IMAGE_CONTROLLER_MANAGER} \ && cd ../.. > /dev/null kustomize build config/default | kubectl apply -f - >/dev/null echo "✓" } function restart_controller_manager() { echo -n "￮ restarting controller manager " kubectl rollout restart deployments/operator-controller-manager >/dev/null echo "✓" } function resize_nodegroups() { if [ -z "$CORTEX_NODEGROUP_NAMES_TO_UPDATE" ]; then return fi eksctl get nodegroup --cluster=$CORTEX_CLUSTER_NAME --region=$CORTEX_REGION --verbose=0 -o json > nodegroups.json eks_ng_len=$(cat nodegroups.json | jq -r length) cfg_ng_len=$(cat $CORTEX_CLUSTER_CONFIG_FILE | yq -r .node_groups | yq -r length) for cfg_ng_name in $CORTEX_NODEGROUP_NAMES_TO_UPDATE; do has_ng="false" for eks_idx in $(seq 0 $(($eks_ng_len-1))); do stack_ng=$(cat nodegroups.json | jq -r .[$eks_idx].Name) if [ "$stack_ng" = "cx-operator" ]; then continue fi if [[ "$stack_ng" == *"$cfg_ng_name" ]]; then has_ng="true" break fi done if [ "$has_ng" == "false" ]; then echo -e "error: \"$cfg_ng_name\" nodegroup (\"cx-*-$cfg_ng_name\" on aws) couldn't be scaled because stack couldn't be found\n" exit 1 fi for cfg_idx in $(seq 0 $(($cfg_ng_len-1))); do cfg_ng=$(cat $CORTEX_CLUSTER_CONFIG_FILE | yq -r .node_groups[$cfg_idx].name) if [ "$cfg_ng" = "$cfg_ng_name" ]; then break fi done desired=$(cat nodegroups.json | jq -r .[$eks_idx].DesiredCapacity) existing_min=$(cat nodegroups.json | jq -r .[$eks_idx].MinSize) existing_max=$(cat nodegroups.json | jq -r .[$eks_idx].MaxSize) updating_min=$(cat $CORTEX_CLUSTER_CONFIG_FILE | yq -r .node_groups[$cfg_idx].min_instances) updating_max=$(cat $CORTEX_CLUSTER_CONFIG_FILE | yq -r .node_groups[$cfg_idx].max_instances) if [ "$desired" -lt $updating_min ]; then desired=$updating_min fi if [ "$desired" -gt $updating_max ]; then desired=$updating_max fi if [ "$existing_min" != "$updating_min" ] && [ "$existing_max" != "$updating_max" ]; then echo "￮ nodegroup $cfg_ng_name: updating min instances to $updating_min and max instances to $updating_max" eksctl scale nodegroup --cluster=$CORTEX_CLUSTER_NAME --region=$CORTEX_REGION $stack_ng --nodes $desired --nodes-min $updating_min --nodes-max $updating_max --timeout "60m" echo elif [ "$existing_min" != "$updating_min" ]; then echo "￮ nodegroup $cfg_ng_name: updating min instances to $updating_min" eksctl scale nodegroup --cluster=$CORTEX_CLUSTER_NAME --region=$CORTEX_REGION $stack_ng --nodes $desired --nodes-min $updating_min --timeout "60m" echo elif [ "$existing_max" != "$updating_max" ]; then echo "￮ nodegroup $cfg_ng_name: updating max instances to $updating_max" eksctl scale nodegroup --cluster=$CORTEX_CLUSTER_NAME --region=$CORTEX_REGION $stack_ng --nodes $desired --nodes-max $updating_max --timeout "60m" echo fi done rm nodegroups.json } function add_nodegroups() { if [ -z "$CORTEX_NODEGROUP_NAMES_TO_ADD" ]; then return fi nodegroup_names="$(join_by , $CORTEX_NODEGROUP_NAMES_TO_ADD)" echo "￮ adding new nodegroup(s) to the cluster ..." python generate_eks.py $CORTEX_CLUSTER_CONFIG_FILE manifests/ami.json --add-cortex-node-groups="$nodegroup_names" > /workspace/nodegroups.yaml eksctl create nodegroup --timeout=$EKSCTL_NODEGROUP_TIMEOUT --install-neuron-plugin=false --install-nvidia-plugin=false --skip-outdated-addons-check -f /workspace/nodegroups.yaml rm /workspace/nodegroups.yaml echo } function remove_nodegroups() { if [ -z "$CORTEX_EKS_NODEGROUP_NAMES_TO_REMOVE" ]; then return fi eks_nodegroup_names="$(join_by , $CORTEX_EKS_NODEGROUP_NAMES_TO_REMOVE)" echo "￮ removing nodegroup(s) from the cluster ..." python generate_eks.py $CORTEX_CLUSTER_CONFIG_FILE manifests/ami.json --remove-eks-node-groups="$eks_nodegroup_names" > /workspace/nodegroups.yaml eksctl delete nodegroup --timeout=$EKSCTL_NODEGROUP_TIMEOUT --approve -f /workspace/nodegroups.yaml rm /workspace/nodegroups.yaml echo } function setup_ipvs() { # get a random kube-proxy pod kubectl rollout status daemonset kube-proxy -n kube-system --timeout 30m >/dev/null kube_proxy_pod=$(kubectl get pod -n kube-system -l k8s-app=kube-proxy -o jsonpath='{.items[*].metadata.name}' | cut -d " " -f1) # export kube-proxy's current config kubectl exec -it -n kube-system ${kube_proxy_pod} -- cat /var/lib/kube-proxy-config/config > proxy_config.yaml # upgrade proxy mode from the exported kube-proxy config python upgrade_kube_proxy_mode.py proxy_config.yaml > upgraded_proxy_config.yaml # update kube-proxy's configmap to include the updated configuration kubectl get configmap -n kube-system kube-proxy -o yaml | yq --arg replace "`cat upgraded_proxy_config.yaml`" '.data.config=$replace' | kubectl apply -f - >/dev/null # patch the kube-proxy daemonset kubectl patch ds -n kube-system kube-proxy --patch "$(cat manifests/kube-proxy.patch.yaml)" >/dev/null kubectl rollout status daemonset kube-proxy -n kube-system --timeout 30m >/dev/null } function setup_istio() { if ! grep -q "istio-customgateway-certs" <<< $(kubectl get secret -n istio-system); then WEBSITE=localhost openssl req -subj "/C=US/CN=$WEBSITE" -newkey rsa:2048 -nodes -keyout $WEBSITE.key -x509 -days 3650 -out $WEBSITE.crt >/dev/null 2>&1 kubectl create -n istio-system secret tls istio-customgateway-certs --key $WEBSITE.key --cert $WEBSITE.crt >/dev/null fi python render_template.py $CORTEX_CLUSTER_CONFIG_FILE manifests/istio.yaml.j2 > /workspace/istio.yaml output_if_error istio-${ISTIO_VERSION}/bin/istioctl install --skip-confirmation --filename /workspace/istio.yaml } function update_networking() { prev_ssl_certificate_arn=$(kubectl get svc ingressgateway-apis -n=istio-system -o json | jq -r '.metadata.annotations."service.beta.kubernetes.io/aws-load-balancer-ssl-cert"') if [ "$prev_ssl_certificate_arn" = "null" ]; then prev_ssl_certificate_arn="" fi new_ssl_certificate_arn=$(cat $CORTEX_CLUSTER_CONFIG_FILE | yq -r .ssl_certificate_arn) if [ "$new_ssl_certificate_arn" = "null" ]; then new_ssl_certificate_arn="" fi prev_api_whitelist_ip_address=$(kubectl get svc ingressgateway-apis -n=istio-system -o yaml | yq -r -c ".spec.loadBalancerSourceRanges") prev_operator_whitelist_ip_address=$(kubectl get svc ingressgateway-operator -n=istio-system -o yaml | yq -r -c ".spec.loadBalancerSourceRanges") new_api_whitelist_ip_address=$(cat $CORTEX_CLUSTER_CONFIG_FILE | yq -r -c ".api_load_balancer_cidr_white_list") new_operator_whitelist_ip_address=$(cat $CORTEX_CLUSTER_CONFIG_FILE | yq -r -c ".operator_load_balancer_cidr_white_list") if [ "$prev_ssl_certificate_arn" = "$new_ssl_certificate_arn" ] && [ "$prev_api_whitelist_ip_address" = "$new_api_whitelist_ip_address" ] && [ "$prev_operator_whitelist_ip_address" = "$new_operator_whitelist_ip_address" ] ; then return fi echo -n "￮ updating networking configuration " if [ "$new_ssl_certificate_arn" != "$prev_ssl_certificate_arn" ] ; then # there is a bug where changing the certificate annotation will not cause the HTTPS listener in the NLB to update # the current workaround is to delete the HTTPS listener and have it recreated with istioctl if [ "$prev_ssl_certificate_arn" != "" ] ; then kubectl patch svc ingressgateway-apis -n=istio-system --type=json -p="[{'op': 'remove', 'path': '/metadata/annotations/service.beta.kubernetes.io~1aws-load-balancer-ssl-cert'}]" >/dev/null fi https_index=$(kubectl get svc ingressgateway-apis -n=istio-system -o json | jq '.spec.ports | map(.name == "https") | index(true)') if [ "$https_index" != "null" ] ; then kubectl patch svc ingressgateway-apis -n=istio-system --type=json -p="[{'op': 'remove', 'path': '/spec/ports/$https_index'}]" >/dev/null fi fi python render_template.py $CORTEX_CLUSTER_CONFIG_FILE manifests/istio.yaml.j2 > /workspace/istio.yaml output_if_error istio-${ISTIO_VERSION}/bin/istioctl install --skip-confirmation --filename /workspace/istio.yaml python render_template.py $CORTEX_CLUSTER_CONFIG_FILE manifests/apis.yaml.j2 > /workspace/apis.yaml kubectl apply -f /workspace/apis.yaml >/dev/null echo "✓" } function validate_cortex() { set +e validation_start_time="$(date +%s)" echo -n "￮ waiting for load balancers " operator_pod_name="" operator_pod_is_ready="" operator_pod_status="" operator_endpoint="" api_load_balancer_endpoint="" operator_load_balancer_state="" api_load_balancer_state="" operator_target_group_status="" operator_endpoint_reachable="" prometheus_ready="" success_cycles=0 while true; do # 30 minute timeout now="$(date +%s)" if [ "$now" -ge "$(($validation_start_time+1800))" ]; then echo -e "\n\ntimeout has occurred when validating your cortex cluster" echo -e "\ndebugging info:" if [ "$operator_pod_name" != "" ]; then echo "operator pod name: $operator_pod_name" fi if [ "$operator_pod_is_ready" != "" ]; then echo "operator pod is ready: $operator_pod_is_ready" fi if [ "$operator_pod_status" != "" ]; then echo "operator pod status: $operator_pod_status" fi if [ "$operator_endpoint" != "" ]; then echo "operator endpoint: $operator_endpoint" fi if [ "${prometheus_ready}" != "" ]; then echo "prometheus is ready: $prometheus_ready" fi if [ "$api_load_balancer_endpoint" != "" ]; then echo "api load balancer endpoint: $api_load_balancer_endpoint" fi if [ "$operator_load_balancer_state" != "" ]; then echo "operator load balancer state: $operator_load_balancer_state" fi if [ "$api_load_balancer_state" != "" ]; then echo "api load balancer state: $api_load_balancer_state" fi if [ "$operator_target_group_status" != "" ]; then echo "operator target group status: $operator_target_group_status" fi if [ "$CORTEX_OPERATOR_LOAD_BALANCER_SCHEME" == "internet-facing" ] && [ "$operator_endpoint_reachable" != "" ]; then echo "operator endpoint reachable: $operator_endpoint_reachable" fi if [ "$operator_endpoint" != "" ]; then echo "operator curl response:" curl --max-time 3 "${operator_endpoint}/verifycortex" fi echo "additional networking events:" kubectl get events -n=istio-system --field-selector involvedObject.kind=Service --sort-by=".metadata.managedFields[0].time" | tail -10 kubectl get events -n=istio-system --field-selector involvedObject.kind=Pod --sort-by=".metadata.managedFields[0].time" | tail -10 echo exit 1 fi echo -n "." sleep 5 operator_pod_name=$(kubectl -n=default get pods -o=name --sort-by=.metadata.creationTimestamp | (grep "^pod/operator-" || true) | tail -1) if [ "$operator_pod_name" == "" ]; then success_cycles=0 continue fi operator_pod_is_ready=$(kubectl -n=default get "$operator_pod_name" -o jsonpath='{.status.containerStatuses[0].ready}') if [ "$operator_pod_is_ready" != "true" ]; then operator_pod_status=$(kubectl -n=default get "$operator_pod_name" -o jsonpath='{.status.containerStatuses[0]}') if [[ "$operator_pod_status" == *"ImagePullBackOff"* ]]; then echo -e "\nerror: the operator image you specified could not be pulled:" echo $operator_pod_status echo exit 1 fi num_restarts=$(kubectl -n=default get "$operator_pod_name" -o jsonpath='{.status.containerStatuses[0].restartCount}') if [[ $num_restarts -ge 2 ]]; then echo -e "\n\nan error occurred when starting the cortex operator" echo -e "\noperator logs (currently running container):\n" kubectl -n=default logs "$operator_pod_name" echo -e "\noperator logs (previous container):\n" kubectl -n=default logs "$operator_pod_name" --previous echo exit 1 fi success_cycles=0 continue fi operator_pod_status="" # reset operator_pod_status since now the operator is active if [ "$operator_endpoint" == "" ]; then out=$(kubectl -n=istio-system get service ingressgateway-operator -o json | tr -d '[:space:]') if [[ $out != *'"loadBalancer":{"ingress":[{"'* ]]; then success_cycles=0 continue fi operator_endpoint=$(kubectl -n=istio-system get service ingressgateway-operator -o json | tr -d '[:space:]' | sed 's/.*{\"hostname\":\"\(.*\)\".*/\1/') fi if [ "$prometheus_ready" == "" ]; then readyReplicas=$(kubectl get statefulset -n prometheus prometheus-prometheus -o jsonpath='{.status.readyReplicas}' 2> /dev/null) desiredReplicas=$(kubectl get statefulset -n prometheus prometheus-prometheus -o jsonpath='{.status.replicas}' 2> /dev/null) if [ "$readyReplicas" != "" ] && [ "$desiredReplicas" != "" ]; then if [ "$readyReplicas" == "$desiredReplicas" ]; then prometheus_ready="true" else prometheus_ready="false" fi fi if [ "$prometheus_ready" != "true" ]; then success_cycles=0 continue fi fi if [ "$api_load_balancer_endpoint" == "" ]; then out=$(kubectl -n=istio-system get service ingressgateway-apis -o json | tr -d '[:space:]') if [[ $out != *'"loadBalancer":{"ingress":[{"'* ]]; then success_cycles=0 continue fi api_load_balancer_endpoint=$(kubectl -n=istio-system get service ingressgateway-apis -o json | tr -d '[:space:]' | sed 's/.*{\"hostname\":\"\(.*\)\".*/\1/') fi operator_load_balancer_state="$(python get_operator_load_balancer_state.py)" # don't cache this result if [ "$operator_load_balancer_state" != "active" ]; then success_cycles=0 continue fi api_load_balancer_state="$(python get_api_load_balancer_state.py)" # don't cache this result if [ "$api_load_balancer_state" != "active" ]; then success_cycles=0 continue fi operator_target_group_status="$(python get_operator_target_group_status.py)" # don't cache this result if [ "$operator_target_group_status" != "healthy" ]; then success_cycles=0 continue fi if [ "$CORTEX_OPERATOR_LOAD_BALANCER_SCHEME" == "internet-facing" ]; then operator_endpoint_reachable="false" # don't cache this result if ! curl --max-time 3 "${operator_endpoint}/verifycortex" >/dev/null 2>&1; then success_cycles=0 continue fi operator_endpoint_reachable="true" fi if [[ $success_cycles -lt 5 ]]; then ((success_cycles++)) continue fi break done echo " ✓" } function print_endpoints() { echo "" operator_endpoint=$(get_operator_endpoint) api_load_balancer_endpoint=$(get_api_load_balancer_endpoint) echo "operator: $operator_endpoint" # before modifying this, search for this prefix echo "api load balancer: $api_load_balancer_endpoint" } function get_operator_endpoint() { kubectl -n=istio-system get service ingressgateway-operator -o json | tr -d '[:space:]' | sed 's/.*{\"hostname\":\"\(.*\)\".*/\1/' } function get_api_load_balancer_endpoint() { kubectl -n=istio-system get service ingressgateway-apis -o json | tr -d '[:space:]' | sed 's/.*{\"hostname\":\"\(.*\)\".*/\1/' } function output_if_error() { set +e rm --force /tmp/suppress.out 2> /dev/null ${1+"$@"} > /tmp/suppress.out 2>&1 if [ "$?" != "0" ]; then echo cat /tmp/suppress.out exit 1 fi rm --force /tmp/suppress.out 2> /dev/null set -e } function join_by { local IFS="$1" shift echo "$*" } main ================================================ FILE: manager/manifests/activator.yaml.j2 ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. apiVersion: v1 kind: ServiceAccount metadata: name: activator namespace: default --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: activator-role rules: - apiGroups: - "networking.istio.io" resources: - virtualservices verbs: - list - watch - apiGroups: - "apps" resources: - deployments verbs: - list - watch --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: activator-rolebinding roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: activator-role subjects: - kind: ServiceAccount name: activator namespace: default --- apiVersion: apps/v1 kind: Deployment metadata: name: activator spec: selector: matchLabels: app: activator template: metadata: name: activator labels: app: activator spec: serviceAccountName: activator containers: - name: activator imagePullPolicy: Always image: {{ config['image_activator'] }} args: - "--in-cluster" - "--port=8000" - "--autoscaler-url=http://autoscaler.default:8000" - "--namespace=default" ports: - name: http containerPort: 8000 - name: admin containerPort: 15000 livenessProbe: httpGet: port: 8000 path: / httpHeaders: - name: X-Cortex-Probe value: "true" readinessProbe: httpGet: port: 8000 path: / httpHeaders: - name: X-Cortex-Probe value: "true" resources: requests: cpu: 100m memory: 100Mi limits: cpu: 250m memory: 300Mi envFrom: - configMapRef: name: env-vars --- apiVersion: v1 kind: Service metadata: name: activator spec: type: ClusterIP selector: app: activator ports: - port: 8000 --- apiVersion: autoscaling/v2beta2 kind: HorizontalPodAutoscaler metadata: name: activator-hpa spec: scaleTargetRef: apiVersion: apps/v1 kind: Deployment name: activator minReplicas: 1 maxReplicas: 10 metrics: - type: Resource resource: name: cpu target: type: Utilization averageUtilization: 70 - type: Resource resource: name: memory target: type: Utilization averageUtilization: 70 ================================================ FILE: manager/manifests/ami.json ================================================ { "1.22": { "af-south-1": { "accelerated_amd64": "ami-060db0c99048ca916", "cpu_amd64": "ami-05b1996f962e1a840", "cpu_arm64": "ami-03156c2c7ecbe55ca" }, "ap-east-1": { "accelerated_amd64": "ami-0ac2ce8b81291a5f1", "cpu_amd64": "ami-0bcbff72f7d2bb056", "cpu_arm64": "ami-07e2412bfc3c401b5" }, "ap-northeast-1": { "accelerated_amd64": "ami-0daf96135f91047cf", "cpu_amd64": "ami-05955794c71f6cde9", "cpu_arm64": "ami-035ae428df1978df6" }, "ap-northeast-2": { "accelerated_amd64": "ami-040df236ed16db21b", "cpu_amd64": "ami-0f2bcb5a524a100a4", "cpu_arm64": "ami-0c4d775e231b413f5" }, "ap-northeast-3": { "accelerated_amd64": "ami-0bbe7685e20d96d41", "cpu_amd64": "ami-09b5be21ab02366ed", "cpu_arm64": "ami-03412067ed8e25694" }, "ap-south-1": { "accelerated_amd64": "ami-0a373620de38ec67d", "cpu_amd64": "ami-02c0a86da73ff476d", "cpu_arm64": "ami-055c892f309c8766d" }, "ap-southeast-1": { "accelerated_amd64": "ami-006821bdff8e4bcb6", "cpu_amd64": "ami-09d066ada84539c8b", "cpu_arm64": "ami-06b35c1ca0ddc31c2" }, "ap-southeast-2": { "accelerated_amd64": "ami-04ab748d420d0c991", "cpu_amd64": "ami-0471c8c731aff64c4", "cpu_arm64": "ami-0bf89cf1628a1835f" }, "ca-central-1": { "accelerated_amd64": "ami-0cf22a8efb72853d5", "cpu_amd64": "ami-03dc7e20ad86871af", "cpu_arm64": "ami-0cc7b246694710794" }, "eu-central-1": { "accelerated_amd64": "ami-0f6759a0eb110c0b1", "cpu_amd64": "ami-0f0bee6c186e51fd3", "cpu_arm64": "ami-024f7b2f337648f01" }, "eu-north-1": { "accelerated_amd64": "ami-0b00a6e259d3bb3f0", "cpu_amd64": "ami-0cb92e7e44afc2fe2", "cpu_arm64": "ami-0559329495f812c1d" }, "eu-south-1": { "accelerated_amd64": "ami-0a07fabae484c3acb", "cpu_amd64": "ami-076e155512c7ccd51", "cpu_arm64": "ami-0e21d55220a1ce340" }, "eu-west-1": { "accelerated_amd64": "ami-0ce94487e6828454f", "cpu_amd64": "ami-03ba20706dab77da5", "cpu_arm64": "ami-04f28ffa44ceadaa6" }, "eu-west-2": { "accelerated_amd64": "ami-09ba6f26d6f9bafb1", "cpu_amd64": "ami-08b4681872300c3d1", "cpu_arm64": "ami-07dd0d8548066837c" }, "eu-west-3": { "accelerated_amd64": "ami-018ab5c8ff8b25e33", "cpu_amd64": "ami-02ec360db9d1c78aa", "cpu_arm64": "ami-030f5a65291bee0cf" }, "me-south-1": { "accelerated_amd64": "ami-04af060694de1bde4", "cpu_amd64": "ami-0da6a4f209e783387", "cpu_arm64": "ami-037082471a912a223" }, "sa-east-1": { "accelerated_amd64": "ami-022752810a1fe5dbb", "cpu_amd64": "ami-0ce71a979799fcfe1", "cpu_arm64": "ami-04ba15cc4ce9eec0c" }, "us-east-1": { "accelerated_amd64": "ami-05ed50f0d1f4bcf18", "cpu_amd64": "ami-0d5cbb67678bc879c", "cpu_arm64": "ami-013845c4dfca498f6" }, "us-east-2": { "accelerated_amd64": "ami-00bc47f895d9384a0", "cpu_amd64": "ami-0c0c07bda9db344ec", "cpu_arm64": "ami-01c1233f0498cd796" }, "us-gov-east-1": { "accelerated_amd64": "ami-0a1f4110c37454db8", "cpu_amd64": "ami-03a40d00b52a4b7d4", "cpu_arm64": "ami-01d8133657b1e33e5" }, "us-gov-west-1": { "accelerated_amd64": "ami-03d3d9405fd000a27", "cpu_amd64": "ami-05067e9caa5a2b27b", "cpu_arm64": "ami-0c775d2df44eafd24" }, "us-west-1": { "accelerated_amd64": "ami-0dff3ce23d6001ee4", "cpu_amd64": "ami-0c68a0cdb2f0ae2b7", "cpu_arm64": "ami-0fe9484a6f1e7b2f1" }, "us-west-2": { "accelerated_amd64": "ami-077be0d48d8abb41b", "cpu_amd64": "ami-045759229c0add028", "cpu_arm64": "ami-08850c88dbabfc43b" } } } ================================================ FILE: manager/manifests/apis.yaml.j2 ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. apiVersion: networking.istio.io/v1beta1 kind: Gateway metadata: name: apis-gateway namespace: default spec: selector: istio: ingressgateway-apis servers: - port: number: 80 name: http protocol: HTTP hosts: - "*" {% if config.get('ssl_certificate_arn', '') == '' %} - port: number: 443 name: https protocol: HTTPS hosts: - "*" tls: mode: SIMPLE serverCertificate: /etc/istio/customgateway-certs/tls.crt privateKey: /etc/istio/customgateway-certs/tls.key {% else %} - port: number: 443 name: https protocol: HTTP hosts: - "*" {% endif %} ================================================ FILE: manager/manifests/async-gateway.yaml.j2 ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. apiVersion: v1 kind: ServiceAccount metadata: name: async-gateway namespace: default --- apiVersion: apps/v1 kind: Deployment metadata: name: async-gateway namespace: default spec: selector: matchLabels: app: async-gateway strategy: rollingUpdate: maxSurge: 25% maxUnavailable: 25% type: RollingUpdate template: metadata: name: async-gateway labels: app: async-gateway spec: serviceAccountName: async-gateway containers: - name: gateway image: {{ config["image_async_gateway"] }} imagePullPolicy: Always args: - --port - "8888" - --cluster-uid - "{{ config["cluster_uid"] }}" - --bucket - "{{ config["bucket"] }}" envFrom: - configMapRef: name: env-vars ports: - containerPort: 8888 readinessProbe: httpGet: path: /healthz port: 8888 scheme: HTTP livenessProbe: httpGet: path: /healthz port: 8888 scheme: HTTP resources: requests: cpu: 400m memory: 512Mi limits: cpu: 400m --- apiVersion: v1 kind: Service metadata: name: async-gateway spec: type: ClusterIP selector: app: async-gateway ports: - port: 8888 --- apiVersion: autoscaling/v2beta2 kind: HorizontalPodAutoscaler metadata: name: async-gateway spec: maxReplicas: 20 minReplicas: 1 scaleTargetRef: apiVersion: apps/v1 kind: Deployment name: async-gateway metrics: - type: Resource resource: name: cpu target: type: Utilization averageUtilization: 90 - type: Resource resource: name: memory target: type: Utilization averageUtilization: 90 ================================================ FILE: manager/manifests/autoscaler.yaml.j2 ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. apiVersion: v1 kind: ServiceAccount metadata: name: autoscaler namespace: default --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: autoscaler-role rules: - apiGroups: - "networking.istio.io" resources: - virtualservices verbs: - get - list - watch - update - apiGroups: - "apps" resources: - deployments verbs: - get - update - watch --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: autoscaler-rolebinding roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: autoscaler-role subjects: - kind: ServiceAccount name: autoscaler namespace: default --- apiVersion: apps/v1 kind: Deployment metadata: name: autoscaler spec: selector: matchLabels: app: autoscaler template: metadata: name: autoscaler labels: app: autoscaler spec: serviceAccountName: autoscaler containers: - name: autoscaler imagePullPolicy: Always image: {{ config['image_autoscaler'] }} args: - "--in-cluster" - "--port=8000" - "--prometheus-url=http://prometheus.prometheus:9090" - "--namespace=default" ports: - containerPort: 8000 livenessProbe: httpGet: port: 8000 path: /healthz readinessProbe: httpGet: port: 8000 path: /healthz resources: requests: cpu: 100m memory: 100Mi envFrom: - configMapRef: name: env-vars --- apiVersion: v1 kind: Service metadata: name: autoscaler spec: type: ClusterIP selector: app: autoscaler ports: - port: 8000 ================================================ FILE: manager/manifests/cluster-autoscaler.yaml.j2 ================================================ # Copyright 2016 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Modifications Copyright 2022 Cortex Labs, Inc. # Source: https://github.com/kubernetes/autoscaler/blob/cluster-autoscaler-1.22.2/cluster-autoscaler/cloudprovider/aws/examples/cluster-autoscaler-autodiscover.yaml apiVersion: v1 kind: ServiceAccount metadata: labels: k8s-addon: cluster-autoscaler.addons.k8s.io k8s-app: cluster-autoscaler name: cluster-autoscaler namespace: kube-system --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: cluster-autoscaler labels: k8s-addon: cluster-autoscaler.addons.k8s.io k8s-app: cluster-autoscaler rules: - apiGroups: [""] resources: ["events", "endpoints"] verbs: ["create", "patch"] - apiGroups: [""] resources: ["pods/eviction"] verbs: ["create"] - apiGroups: [""] resources: ["pods/status"] verbs: ["update"] - apiGroups: [""] resources: ["endpoints"] resourceNames: ["cluster-autoscaler"] verbs: ["get", "update"] - apiGroups: [""] resources: ["nodes"] verbs: ["watch", "list", "get", "update"] - apiGroups: [""] resources: - "pods" - "services" - "replicationcontrollers" - "persistentvolumeclaims" - "persistentvolumes" verbs: ["watch", "list", "get"] - apiGroups: ["extensions"] resources: ["replicasets", "daemonsets"] verbs: ["watch", "list", "get"] - apiGroups: ["policy"] resources: ["poddisruptionbudgets"] verbs: ["watch", "list"] - apiGroups: ["apps"] resources: ["statefulsets", "replicasets", "daemonsets"] verbs: ["watch", "list", "get"] - apiGroups: ["storage.k8s.io"] resources: ["storageclasses", "csinodes"] verbs: ["watch", "list", "get"] - apiGroups: ["batch", "extensions"] resources: ["jobs"] verbs: ["get", "list", "watch", "patch"] - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["create"] - apiGroups: ["coordination.k8s.io"] resourceNames: ["cluster-autoscaler"] resources: ["leases"] verbs: ["get", "update"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: cluster-autoscaler namespace: kube-system labels: k8s-addon: cluster-autoscaler.addons.k8s.io k8s-app: cluster-autoscaler rules: - apiGroups: [""] resources: ["configmaps"] verbs: ["create","list","watch"] - apiGroups: [""] resources: ["configmaps"] resourceNames: ["cluster-autoscaler-status", "cluster-autoscaler-priority-expander"] verbs: ["delete", "get", "update", "watch"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: cluster-autoscaler labels: k8s-addon: cluster-autoscaler.addons.k8s.io k8s-app: cluster-autoscaler roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: cluster-autoscaler subjects: - kind: ServiceAccount name: cluster-autoscaler namespace: kube-system --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: cluster-autoscaler namespace: kube-system labels: k8s-addon: cluster-autoscaler.addons.k8s.io k8s-app: cluster-autoscaler roleRef: apiGroup: rbac.authorization.k8s.io kind: Role name: cluster-autoscaler subjects: - kind: ServiceAccount name: cluster-autoscaler namespace: kube-system --- apiVersion: v1 kind: ConfigMap metadata: name: cluster-autoscaler-priority-expander namespace: kube-system data: priorities: |- {% for p in range(1, 101) %} {% set found = {'priority': False} %} {% for ng in config['node_groups'] %} {% if ng['priority'] == p %} {%- if found.update({'priority':True}) %} {%- endif %} {% endif %} {% endfor %} {% if found['priority'] %} {{ p }}: {% endif %} {% for ng in config['node_groups'] %} {% if ng['priority'] == p %} {% if ng['spot'] %} - .*{{ 'cx-ws-' + ng['name'] }}.* {% else %} - .*{{ 'cx-wd-' + ng['name'] }}.* {% endif %} {% endif %} {% endfor %} {% endfor %} --- apiVersion: apps/v1 kind: Deployment metadata: name: cluster-autoscaler namespace: kube-system labels: app: cluster-autoscaler spec: replicas: 1 selector: matchLabels: app: cluster-autoscaler template: metadata: labels: app: cluster-autoscaler spec: serviceAccountName: cluster-autoscaler priorityClassName: system-cluster-critical containers: - image: {{ config['image_cluster_autoscaler'] }} name: cluster-autoscaler resources: limits: cpu: 300m requests: cpu: 100m memory: 400Mi command: - ./cluster-autoscaler - --v=4 - --stderrthreshold=info - --cloud-provider=aws - --skip-nodes-with-local-storage=false - --expander=priority - --max-total-unready-percentage=5 - --ok-total-unready-count=30 - --max-node-provision-time=8m - --scan-interval=20s - --scale-up-rate-limit-enabled=true - --scale-up-max-number-nodes-per-min=50 - --scale-up-burst-number-nodes-per-min=75 - --node-group-auto-discovery=asg:tag=k8s.io/cluster-autoscaler/enabled,k8s.io/cluster-autoscaler/{{ config['cluster_name'] }} volumeMounts: - name: ssl-certs mountPath: /etc/ssl/certs/ca-certificates.crt #/etc/ssl/certs/ca-bundle.crt for Amazon Linux Worker Nodes readOnly: true imagePullPolicy: "Always" volumes: - name: ssl-certs hostPath: path: "/etc/ssl/certs/ca-bundle.crt" strategy: type: RollingUpdate rollingUpdate: maxSurge: 0 # necessary because there may not be enough room on the operator node for a rolling update ================================================ FILE: manager/manifests/default_cortex_cli_config.yaml ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. default_environment: default environments: - name: default operator_endpoint: http://operator.default.svc.cluster.local:8888 ================================================ FILE: manager/manifests/event-exporter.yaml ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. apiVersion: v1 kind: ServiceAccount metadata: namespace: logging name: event-exporter --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: event-exporter roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: view subjects: - kind: ServiceAccount namespace: logging name: event-exporter --- apiVersion: v1 kind: ConfigMap metadata: name: event-exporter-config namespace: logging data: config.yaml: | logLevel: error logFormat: json route: routes: - match: - receiver: "stdout" labels: cortex.dev/api: true receivers: - name: "stdout" file: path: "/dev/stdout" --- apiVersion: apps/v1 kind: Deployment metadata: name: event-exporter namespace: logging spec: replicas: 1 selector: matchLabels: app: event-exporter template: metadata: labels: app: event-exporter spec: serviceAccountName: event-exporter containers: - name: event-exporter image: $CORTEX_IMAGE_EVENT_EXPORTER imagePullPolicy: IfNotPresent args: - -conf=/data/config.yaml volumeMounts: - mountPath: /data name: event-exporter-config resources: requests: cpu: 20m memory: 50Mi volumes: - name: event-exporter-config configMap: name: event-exporter-config ================================================ FILE: manager/manifests/fluent-bit.yaml.j2 ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. apiVersion: v1 kind: ServiceAccount metadata: name: fluent-bit namespace: logging --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: fluent-bit-read rules: - apiGroups: [""] resources: - namespaces - pods verbs: ["get", "list", "watch"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: fluent-bit-read roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: fluent-bit-read subjects: - kind: ServiceAccount name: fluent-bit namespace: logging --- apiVersion: v1 kind: ConfigMap metadata: name: fluent-bit-config namespace: logging labels: k8s-app: fluent-bit data: # Configuration files: server, input, filters and output # ====================================================== fluent-bit.conf: | [SERVICE] Flush 1 Grace 30 Log_Level info Daemon off Parsers_File parsers.conf HTTP_Server Off Config_Watch Off @INCLUDE input-kubernetes.conf @INCLUDE filter-kubernetes.conf @INCLUDE filter-k8s-events.conf @INCLUDE filter-stackdriver-format.conf @INCLUDE output.conf input-kubernetes.conf: | [INPUT] Name tail Tag kube.* Path /var/log/containers/*.log Parser docker DB /var/log/flb_kube.db Mem_Buf_Limit 5MB Skip_Long_Lines On Refresh_Interval 10 filter-kubernetes.conf: | [FILTER] Name kubernetes Match kube.var.log.containers.* Kube_URL https://kubernetes.default.svc:443 Kube_Tag_Prefix kube.var.log.containers. Merge_Log On # this retagging helps stackdriver and it doesn't matter for cloudwatch # https://docs.fluentbit.io/manual/pipeline/outputs/stackdriver#configuration-file [FILTER] Name rewrite_tag Match kube.var.log.containers.* Rule $log ^(.*)$ k8s_container.$kubernetes['namespace_name'].$kubernetes['pod_name'].$kubernetes['container_name'] false [FILTER] Name modify Match k8s_container.* Condition Key_Exists message Hard_rename message log [FILTER] Name modify Match k8s_container.* Condition Key_Exists msg Hard_rename msg log [FILTER] Name nest Match k8s_container.* Operation lift Nested_under kubernetes Add_prefix k8s. [FILTER] Name modify Match k8s_container.* Condition Key_Does_Not_Exist cortex.labels Rename k8s.labels cortex.labels [FILTER] Name modify Match k8s_container.* Remove_wildcard k8s. filter-k8s-events.conf: | [FILTER] Name nest Match k8s_container.*.event-exporter-* Operation lift Nested_under involvedObject Add_prefix involvedObject. [FILTER] Name modify Match k8s_container.*.event-exporter-* Condition Key_exists labels Rename labels k8s.labels [FILTER] Name modify Match k8s_container.*.event-exporter-* Condition Key_exists involvedObject.labels Hard_copy involvedObject.labels cortex.labels [FILTER] Name nest Match k8s_container.*.event-exporter-* Operation nest Wildcard involvedObject.* Nest_under involvedObject Remove_prefix involvedObject. filter-stackdriver-format.conf: | [FILTER] Name modify Match k8s_container.* Condition Key_exists log Rename log message [FILTER] Name modify Match k8s_container.* Condition Key_exists levelname Rename levelname level output.conf: | [OUTPUT] Name cloudwatch Match k8s_container.* region {{ config["region"] }} log_group_name {{ config["cluster_name"] }} log_stream_prefix kube. auto_create_group true parsers.conf: | [PARSER] Name docker Format json Time_Key time Time_Format %Y-%m-%dT%H:%M:%S.%L Time_Keep On --- apiVersion: apps/v1 kind: DaemonSet metadata: name: fluent-bit namespace: logging spec: selector: matchLabels: k8s-app: fluent-bit-logging template: metadata: labels: app: fluent-bit k8s-app: fluent-bit-logging version: v1 kubernetes.io/cluster-service: "true" spec: containers: - name: fluent-bit image: {{ config["image_fluent_bit"] }} imagePullPolicy: Always resources: requests: cpu: 100m memory: 150Mi limits: cpu: 150m memory: 150Mi ports: - containerPort: 2020 volumeMounts: - name: varlog mountPath: /var/log - name: varlibdockercontainers mountPath: /var/lib/docker/containers readOnly: true - name: fluent-bit-config mountPath: /fluent-bit/etc/ terminationGracePeriodSeconds: 60 volumes: - name: varlog hostPath: path: /var/log - name: varlibdockercontainers hostPath: path: /var/lib/docker/containers - name: fluent-bit-config configMap: name: fluent-bit-config serviceAccountName: fluent-bit tolerations: - key: node-role.kubernetes.io/master operator: Exists effect: NoSchedule - operator: "Exists" effect: "NoExecute" - operator: "Exists" effect: "NoSchedule" - key: aws.amazon.com/neuron operator: Exists effect: NoSchedule - key: nvidia.com/gpu operator: Exists effect: NoSchedule - key: workload operator: Exists effect: NoSchedule - key: prometheus operator: Exists effect: NoSchedule ================================================ FILE: manager/manifests/grafana/grafana-dashboard-async.yaml ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. apiVersion: v1 kind: ConfigMap metadata: name: grafana-dashboard-async namespace: default data: async.json: |- { "annotations": { "list": [ { "builtIn": 1, "datasource": "prometheus", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "type": "dashboard" } ] }, "editable": true, "gnetId": null, "graphTooltip": 0, "iteration": 1625805144458, "links": [], "panels": [ { "datasource": null, "gridPos": { "h": 2, "w": 24, "x": 0, "y": 0 }, "id": 15, "options": { "content": "

AsyncAPI

", "mode": "markdown" }, "pluginVersion": "8.0.4", "timeFrom": null, "timeShift": null, "transparent": true, "type": "text" }, { "collapsed": false, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 2 }, "id": 22, "panels": [], "title": "API Stats", "type": "row" }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "description": "Request rate, computed over every minute, of an API", "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 3 }, "hiddenSeries": false, "id": 2, "legend": { "alignAsTable": true, "avg": true, "current": true, "max": true, "min": true, "rightSide": false, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(cortex_async_request_count{api_name=\"$api_name\"}[1m])) by (api_name)", "interval": "", "legendFormat": "{{api_name}}", "refId": "Request Rate" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Request Rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "reqps", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "description": "In-flight requests for an API.\n\nNote: In-flight requests are recorded every 10 seconds, which will correspond to the minimum resolution.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 3 }, "hiddenSeries": false, "id": 4, "legend": { "alignAsTable": true, "avg": true, "current": true, "max": true, "min": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(cortex_async_active{api_kind=\"AsyncAPI\",api_name=\"$api_name\"}) by (api_name)", "hide": false, "interval": "", "legendFormat": "active", "refId": "Active" }, { "exemplar": true, "expr": "sum(cortex_async_queued{api_kind=\"AsyncAPI\",api_name=\"$api_name\"}) by (api_name)", "hide": false, "interval": "", "legendFormat": "queued", "refId": "Queued" }, { "exemplar": true, "expr": "sum(cortex_async_in_flight{api_kind=\"AsyncAPI\",api_name=\"$api_name\"}) by (api_name)", "hide": true, "interval": "", "legendFormat": "in flight", "refId": "In Flight" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "In-Flight Requests", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": 0, "format": "short", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "description": "Request rate, computed over every minute, for responses with status code 2XX of an API", "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 12 }, "hiddenSeries": false, "id": 8, "legend": { "alignAsTable": true, "avg": true, "current": true, "max": true, "min": true, "rightSide": false, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(cortex_async_request_count{api_kind=\"AsyncAPI\",api_name=\"$api_name\",status_code=~\"2.+\"}[1m])) by (api_name, status_code)", "interval": "", "legendFormat": "{{api_name}}", "refId": "2XX" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "2XX Responses", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "reqps", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 12 }, "hiddenSeries": false, "id": 7, "legend": { "alignAsTable": true, "avg": true, "current": true, "max": true, "min": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(kube_deployment_status_replicas_available{deployment=\"api-$api_name\"}) by (deployment)", "interval": "", "legendFormat": "{{deployment}}", "refId": "Active Replicas" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Active Replicas", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [ { "id": "renameByRegex", "options": { "regex": "api-(.*)", "renamePattern": "$1" } } ], "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": 0, "format": "short", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "description": "Request rate, computed over every minute, for responses with status code 4XX of an API", "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 21 }, "hiddenSeries": false, "id": 9, "legend": { "alignAsTable": true, "avg": true, "current": true, "max": true, "min": true, "rightSide": false, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(cortex_async_request_count{api_kind=\"AsyncAPI\",api_name=\"$api_name\",status_code=~\"4.+\"}[1m])) by (api_name, status_code)", "interval": "", "legendFormat": "{{api_name}}", "refId": "4XX" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "4XX Responses", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "reqps", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "description": "Request rate, computed over every minute, for responses with status code 5XX of an API", "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 21 }, "hiddenSeries": false, "id": 10, "legend": { "alignAsTable": true, "avg": true, "current": true, "max": true, "min": true, "rightSide": false, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(cortex_async_request_count{api_kind=\"AsyncAPI\",api_name=\"$api_name\",status_code=~\"5.+\"}[1m])) by (api_name, status_code)", "interval": "", "legendFormat": "{{api_name}}", "refId": "5XX" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "5XX Responses", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "reqps", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "description": "99th percentile latency, computed over a minute, for an API", "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 30 }, "hiddenSeries": false, "id": 6, "legend": { "alignAsTable": true, "avg": true, "current": true, "max": true, "min": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum by (api_name, le) (rate(cortex_async_latency_bucket{api_kind=\"AsyncAPI\",api_name=\"$api_name\"}[1m])))", "interval": "", "legendFormat": "{{api_name}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "p99 Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "description": "90th percentile latency, computed over a minute, for an API", "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 30 }, "hiddenSeries": false, "id": 11, "legend": { "alignAsTable": true, "avg": true, "current": true, "max": true, "min": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.90, sum by (api_name, le) (rate(cortex_async_latency_bucket{api_kind=\"AsyncAPI\",api_name=\"$api_name\"}[1m])))", "hide": false, "interval": "", "legendFormat": "{{api_name}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "p90 Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "description": "50th percentile latency, computed over a minute, for an API", "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 39 }, "hiddenSeries": false, "id": 16, "legend": { "alignAsTable": true, "avg": true, "current": true, "max": true, "min": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.50, sum by (api_name, le) (rate(cortex_async_latency_bucket{api_kind=\"AsyncAPI\",api_name=\"$api_name\"}[1m])))", "hide": false, "interval": "", "legendFormat": "{{api_name}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "p50 Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "description": "Average latency, computed over a minute, for an API", "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 39 }, "hiddenSeries": false, "id": 12, "legend": { "alignAsTable": true, "avg": true, "current": true, "max": true, "min": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(cortex_async_latency_sum{api_kind=\"AsyncAPI\",api_name=\"$api_name\"}[1m])) by (api_name) / \nsum(rate(cortex_async_latency_count{api_kind=\"AsyncAPI\",api_name=\"$api_name\"}[1m])) by (api_name)", "hide": false, "interval": "", "legendFormat": "{{api_name}}", "refId": "D" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Average Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "collapsed": false, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 48 }, "id": 20, "panels": [], "title": "Aggregate Usage", "type": "row" }, { "aliasColors": { "Total CPU Request": "semi-dark-orange", "Total CPU Usage": "semi-dark-green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "description": "Total CPU usage across all replicas of the API", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 49 }, "hiddenSeries": false, "id": 24, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "sum(rate(container_cpu_usage_seconds_total{pod=~\"api-$api_name.+\", container!=\"POD\", name!=\"\"}[1m]))", "format": "time_series", "instant": false, "interval": "", "legendFormat": "Total CPU Usage", "refId": "CPU Usage" }, { "exemplar": true, "expr": "sum(kube_pod_container_resource_requests{exported_pod=~\"api-$api_name.+\", resource=\"cpu\"})", "hide": false, "interval": "", "legendFormat": "Total CPU Request", "refId": "CPU Request" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Total CPU Usage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "core", "label": "cpu", "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": "", "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "Total Memory Request": "semi-dark-orange", "Total Memory Usage": "semi-dark-green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "description": "Total memory usage across all replicas of the API", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 49 }, "hiddenSeries": false, "id": 26, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "sum(sum_over_time(container_memory_working_set_bytes{pod=~\"api-$api_name.+\", name!=\"\", container!=\"POD\"}[1m])) /\navg(count_over_time(container_memory_working_set_bytes{pod=~\"api-$api_name.+\", name!=\"\", container=\"api\"}[1m])) / 1024^2", "format": "time_series", "instant": false, "interval": "", "legendFormat": "Total Memory Usage", "refId": "Memory Usage" }, { "exemplar": true, "expr": "sum(kube_pod_container_resource_requests{exported_pod=~\"api-$api_name.+\", resource=\"memory\"}) / 1024^2", "hide": false, "interval": "", "legendFormat": "Total Memory Request", "refId": "Memory Request" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Total Memory Usage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "MiB", "label": "memory", "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": "", "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "Total GPU Capacity": "semi-dark-orange", "Total GPU Usage": "semi-dark-green", "Total GPU Utilization": "light-green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "description": "Total GPU core usage across all replicas of the API", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 57 }, "hiddenSeries": false, "id": 28, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(DCGM_FI_DEV_GPU_UTIL{exported_pod=~\"api-$api_name.+\"}) / 100", "hide": false, "interval": "", "legendFormat": "Total GPU Usage", "refId": "GPU Usage" }, { "expr": "count(DCGM_FI_DEV_GPU_UTIL{exported_pod=~\"api-$api_name.+\"})", "hide": false, "interval": "", "legendFormat": "Total GPU Capacity", "refId": "GPU Capacity" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Total GPU Core Usage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "gpuCore", "label": "gpu", "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": "", "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "Total Capacity GPU Memory": "semi-dark-orange", "Total Used GPU Memory": "semi-dark-green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "description": "Total GPU memory usage across all replicas of the API", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 57 }, "hiddenSeries": false, "id": 29, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(DCGM_FI_DEV_FB_USED{exported_pod=~\"api-$api_name.+\"})", "hide": false, "interval": "", "legendFormat": "Total Used GPU Memory", "refId": "GPU Used Memory" }, { "exemplar": false, "expr": "sum(DCGM_FI_DEV_FB_FREE{exported_pod=~\"api-$api_name.+\"}) + sum(DCGM_FI_DEV_FB_USED{exported_pod=~\"api-$api_name.+\"})", "format": "time_series", "instant": false, "interval": "", "legendFormat": "Total Capacity GPU Memory", "refId": "GPU Capacity Memory" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Total GPU Memory Usage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "MiB", "label": "memory", "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": "", "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "collapsed": false, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 65 }, "id": 18, "panels": [], "title": "Average Replica Usage", "type": "row" }, { "aliasColors": { "Avg CPU Request": "semi-dark-orange", "Avg CPU Usage": "semi-dark-green", "Total CPU Request": "semi-dark-orange", "Total CPU Usage": "semi-dark-green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "description": "Avg CPU usage across all replicas of the API", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 66 }, "hiddenSeries": false, "id": 30, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "sum(rate(container_cpu_usage_seconds_total{pod=~\"api-$api_name.+\", container!=\"POD\", name!=\"\"}[1m]))\n/\nsum(kube_pod_info{exported_pod=~\"api-$api_name.+\"})", "format": "time_series", "instant": false, "interval": "", "legendFormat": "Avg CPU Usage", "refId": "CPU Usage" }, { "exemplar": true, "expr": "sum(kube_pod_container_resource_requests{exported_pod=~\"api-$api_name.+\", resource=\"cpu\"})\n/\nsum(kube_pod_info{exported_pod=~\"api-$api_name.+\"})", "hide": false, "interval": "", "legendFormat": "Avg CPU Request", "refId": "CPU Request" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Avg CPU Usage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "core", "label": "cpu", "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": "", "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "Avg Memory Request": "semi-dark-orange", "Avg Memory Usage": "semi-dark-green", "Total Memory Request": "semi-dark-orange", "Total Memory Usage": "semi-dark-green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "description": "Avg memory usage across all replicas of the API", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 66 }, "hiddenSeries": false, "id": 31, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "sum(sum_over_time(container_memory_working_set_bytes{pod=~\"api-$api_name.+\", name!=\"\", container!=\"POD\"}[1m]))\n/\navg(count_over_time(container_memory_working_set_bytes{pod=~\"api-$api_name.+\", name!=\"\", container=\"api\"}[1m])) / 1024^2\n/\nsum(kube_pod_info{exported_pod=~\"api-$api_name.+\"})", "format": "time_series", "instant": false, "interval": "", "legendFormat": "Avg Memory Usage", "refId": "Memory Usage" }, { "exemplar": true, "expr": "sum(kube_pod_container_resource_requests{exported_pod=~\"api-$api_name.+\", resource=\"memory\"}) / 1024^2\n/\nsum(kube_pod_info{exported_pod=~\"api-$api_name.+\"})", "hide": false, "interval": "", "legendFormat": "Avg Memory Request", "refId": "Memory Request" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Avg Memory Usage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "MiB", "label": "memory", "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": "", "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "Avg GPU Capacity": "semi-dark-orange", "Avg GPU Usage": "semi-dark-green", "Total GPU Capacity": "semi-dark-orange", "Total GPU Utilization": "semi-dark-green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "description": "Avg GPU core usage across all replicas of the API", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 74 }, "hiddenSeries": false, "id": 32, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(DCGM_FI_DEV_GPU_UTIL{exported_pod=~\"api-$api_name.+\"}) / 100\n/\ncount(DCGM_FI_DEV_GPU_UTIL{exported_pod=~\"api-$api_name.+\"})", "hide": false, "interval": "", "legendFormat": "Avg GPU Usage", "refId": "GPU Usage" }, { "expr": "count(DCGM_FI_DEV_GPU_UTIL{exported_pod=~\"api-$api_name.+\"})\n/\ncount(count(DCGM_FI_DEV_GPU_UTIL{exported_pod=~\"api-$api_name.+\"}) by (exported_pod))", "hide": false, "interval": "", "legendFormat": "Avg GPU Capacity", "refId": "GPU Capacity" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Avg GPU Core Usage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "gpuCore", "label": "gpu", "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": "", "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "Avg Capacity GPU Memory": "semi-dark-orange", "Avg Used GPU Memory": "semi-dark-green", "Total Capacity GPU Memory": "semi-dark-orange", "Total Used GPU Memory": "semi-dark-green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "description": "Avg GPU memory usage across all replicas of the API", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 74 }, "hiddenSeries": false, "id": 33, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(DCGM_FI_DEV_FB_USED{exported_pod=~\"api-$api_name.+\"})\n/\ncount(DCGM_FI_DEV_FB_USED{exported_pod=~\"api-$api_name.+\"})", "hide": false, "interval": "", "legendFormat": "Avg Used GPU Memory", "refId": "GPU Used Memory" }, { "exemplar": false, "expr": "(sum(DCGM_FI_DEV_FB_FREE{exported_pod=~\"api-$api_name.+\"}) + sum(DCGM_FI_DEV_FB_USED{exported_pod=~\"api-$api_name.+\"}))\n/\ncount(DCGM_FI_DEV_FB_USED{exported_pod=~\"api-$api_name.+\"})", "format": "time_series", "instant": false, "interval": "", "legendFormat": "Avg Capacity GPU Memory", "refId": "GPU Capacity Memory" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Avg GPU Memory Usage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "MiB", "label": "memory", "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": "", "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } } ], "refresh": "30s", "schemaVersion": 30, "style": "dark", "tags": [], "templating": { "list": [ { "allValue": null, "current": { "selected": false, "text": "None", "value": "None" }, "datasource": null, "definition": "label_values(cortex_async_queue_length{api_kind=\"AsyncAPI\"}, api_name)", "description": null, "error": null, "hide": 0, "includeAll": false, "label": "API Name", "multi": true, "name": "api_name", "options": [], "query": { "query": "label_values(cortex_async_queue_length{api_kind=\"AsyncAPI\"}, api_name)", "refId": "StandardVariableQuery" }, "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 1, "tagValuesQuery": "", "tagsQuery": "", "type": "query", "useTags": false } ] }, "time": { "from": "now-1h", "to": "now" }, "timepicker": {}, "timezone": "", "title": "AsyncAPI", "uid": "asyncapi", "version": 1 } ================================================ FILE: manager/manifests/grafana/grafana-dashboard-batch.yaml ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. apiVersion: v1 kind: ConfigMap metadata: name: grafana-dashboard-batch namespace: default data: batch.json: |- { "annotations": { "list": [ { "builtIn": 1, "datasource": "prometheus", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "type": "dashboard" } ] }, "editable": true, "gnetId": null, "graphTooltip": 0, "iteration": 1625169092506, "links": [], "panels": [ { "datasource": null, "gridPos": { "h": 2, "w": 24, "x": 0, "y": 0 }, "id": 7, "options": { "content": "

BatchAPI

\n", "mode": "markdown" }, "pluginVersion": "8.0.4", "timeFrom": null, "timeShift": null, "transparent": true, "type": "text" }, { "datasource": null, "fieldConfig": { "defaults": {}, "overrides": [] }, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 2 }, "id": 22, "title": "API Stats", "type": "row" }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "description": "Number of succeeded batches for an API", "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 3 }, "hiddenSeries": false, "id": 2, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(cortex_batch_succeeded{api_name=~\"$api_name\"}) by (api_name)", "interval": "", "legendFormat": "{{api_name}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "# Succeeded Batches", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:26", "decimals": 0, "format": "short", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "$$hashKey": "object:27", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "description": "Number of failed batches for an API", "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 3 }, "hiddenSeries": false, "id": 3, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(cortex_batch_failed{api_name=~\"$api_name\"}) by (api_name)", "interval": "", "legendFormat": "{{api_name}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "# Failed Batches", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:262", "decimals": 0, "format": "short", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "$$hashKey": "object:263", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "description": "Average time per batch for an API", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 12 }, "hiddenSeries": false, "id": 5, "legend": { "alignAsTable": true, "avg": true, "current": true, "max": true, "min": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(cortex_time_per_batch_sum{api_name=~\"$api_name\"}) by (api_name) / sum(cortex_time_per_batch_count{api_name=~\"$api_name\"}) by (api_name)", "interval": "", "legendFormat": "{{api_name}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Average Time per Batch", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "ms", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "Active Jobs": "semi-dark-green", "Active Workers": "semi-dark-orange" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "description": "Active jobs/workers", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 12 }, "hiddenSeries": false, "id": 20, "legend": { "alignAsTable": true, "avg": true, "current": true, "max": true, "min": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "count(kube_job_status_active{job_name=~\"$api_name.+\"} != 0)", "interval": "", "legendFormat": "Active Jobs", "refId": "Active Batches" }, { "exemplar": true, "expr": "sum(kube_job_status_active{job_name=~\"$api_name.+\"})", "hide": false, "interval": "", "legendFormat": "Active Workers", "refId": "Active Workers" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "# Active Jobs/Workers", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:128", "decimals": 0, "format": "count", "label": "", "logBase": 1, "max": null, "min": "0", "show": true }, { "$$hashKey": "object:129", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "datasource": null, "fieldConfig": { "defaults": {}, "overrides": [] }, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 20 }, "id": 11, "title": "Aggregate Worker Usage", "type": "row" }, { "aliasColors": { "Total CPU Request": "semi-dark-orange", "Total CPU Usage": "semi-dark-green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "description": "Total CPU usage across all workers of the API", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 21 }, "hiddenSeries": false, "id": 13, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "sum(rate(container_cpu_usage_seconds_total{pod=~\"$api_name.+\", container!=\"POD\", name!=\"\"}[1m]))", "format": "time_series", "instant": false, "interval": "", "legendFormat": "Total CPU Usage", "refId": "CPU Usage" }, { "exemplar": true, "expr": "sum(kube_pod_container_resource_requests{exported_pod=~\"$api_name.+\", resource=\"cpu\"})", "hide": false, "interval": "", "legendFormat": "Total CPU Request", "refId": "CPU Request" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Total CPU Usage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:1404", "format": "core", "label": "cpu", "logBase": 1, "max": null, "min": "0", "show": true }, { "$$hashKey": "object:1405", "format": "short", "label": "", "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "Total Memory Request": "semi-dark-orange", "Total Memory Usage": "semi-dark-green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "description": "Total memory usage across all workers of the API", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 21 }, "hiddenSeries": false, "id": 15, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "sum(sum_over_time(container_memory_working_set_bytes{pod=~\"$api_name.+\", name!=\"\", container!=\"POD\"}[1m]))\n/\navg(count_over_time(container_memory_working_set_bytes{pod=~\"$api_name.+\", name!=\"\", container=\"api\"}[1m])) / 1024^2", "format": "time_series", "instant": false, "interval": "", "legendFormat": "Total Memory Usage", "refId": "Memory Usage" }, { "exemplar": true, "expr": "sum(kube_pod_container_resource_requests{exported_pod=~\"$api_name.+\", resource=\"memory\"}) / 1024^2", "hide": false, "interval": "", "legendFormat": "Total Memory Request", "refId": "Memory Request" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Total Memory Usage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:1404", "format": "MiB", "label": "memory", "logBase": 1, "max": null, "min": "0", "show": true }, { "$$hashKey": "object:1405", "format": "short", "label": "", "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "Total GPU Capacity": "semi-dark-orange", "Total GPU Usage": "semi-dark-green", "Total GPU Utilization": "light-green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "description": "Total GPU core usage across all workers of the API", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 29 }, "hiddenSeries": false, "id": 17, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(DCGM_FI_DEV_GPU_UTIL{exported_pod=~\"$api_name.+\"}) / 100", "hide": false, "interval": "", "legendFormat": "Total GPU Usage", "refId": "GPU Usage" }, { "expr": "count(DCGM_FI_DEV_GPU_UTIL{exported_pod=~\"$api_name.+\"})", "hide": false, "interval": "", "legendFormat": "Total GPU Capacity", "refId": "GPU Capacity" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Total GPU Core Usage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:1404", "format": "gpuCore", "label": "gpu", "logBase": 1, "max": null, "min": "0", "show": true }, { "$$hashKey": "object:1405", "format": "short", "label": "", "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "Total Capacity GPU Memory": "semi-dark-orange", "Total Used GPU Memory": "semi-dark-green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "description": "Total GPU memory usage across all workers of the API", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 29 }, "hiddenSeries": false, "id": 19, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(DCGM_FI_DEV_FB_USED{exported_pod=~\"$api_name.+\"})", "hide": false, "interval": "", "legendFormat": "Total Used GPU Memory", "refId": "GPU Used Memory" }, { "exemplar": false, "expr": "sum(DCGM_FI_DEV_FB_FREE{exported_pod=~\"$api_name.+\"}) + sum(DCGM_FI_DEV_FB_USED{exported_pod=~\"$api_name.+\"})", "format": "time_series", "instant": false, "interval": "", "legendFormat": "Total Capacity GPU Memory", "refId": "GPU Capacity Memory" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Total GPU Memory Usage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:1404", "format": "MiB", "label": "memory", "logBase": 1, "max": null, "min": "0", "show": true }, { "$$hashKey": "object:1405", "format": "short", "label": "", "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "collapsed": false, "datasource": null, "fieldConfig": { "defaults": {}, "overrides": [] }, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 37 }, "id": 9, "panels": [], "title": "Avg Worker Usage", "type": "row" }, { "aliasColors": { "Avg CPU Request": "semi-dark-orange", "Avg CPU Usage": "semi-dark-green", "Total CPU Request": "semi-dark-orange", "Total CPU Usage": "semi-dark-green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "description": "Avg CPU usage across all workers of the API", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 38 }, "hiddenSeries": false, "id": 23, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "sum(rate(container_cpu_usage_seconds_total{pod=~\"$api_name.+\", container!=\"POD\", name!=\"\"}[1m]))\n/\nsum(kube_pod_info{exported_pod=~\"$api_name.+\"})", "format": "time_series", "instant": false, "interval": "", "legendFormat": "Avg CPU Usage", "refId": "CPU Usage" }, { "exemplar": true, "expr": "sum(kube_pod_container_resource_requests{exported_pod=~\"$api_name.+\", resource=\"cpu\"})\n/\nsum(kube_pod_info{exported_pod=~\"$api_name.+\"})", "hide": false, "interval": "", "legendFormat": "Avg CPU Request", "refId": "CPU Request" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Avg CPU Usage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:1404", "format": "core", "label": "cpu", "logBase": 1, "max": null, "min": "0", "show": true }, { "$$hashKey": "object:1405", "format": "short", "label": "", "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "Avg Memory Request": "semi-dark-orange", "Avg Memory Usage": "semi-dark-green", "Total Memory Request": "semi-dark-orange", "Total Memory Usage": "semi-dark-green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "description": "Avg memory usage across all workers of the API", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 38 }, "hiddenSeries": false, "id": 24, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "sum(sum_over_time(container_memory_working_set_bytes{pod=~\"$api_name.+\", name!=\"\", container!=\"POD\"}[1m]))\n/\navg(count_over_time(container_memory_working_set_bytes{pod=~\"$api_name.+\", name!=\"\", container=\"api\"}[1m])) / 1024^2\n/\nsum(kube_pod_info{exported_pod=~\"$api_name.+\"})", "format": "time_series", "instant": false, "interval": "", "legendFormat": "Avg Memory Usage", "refId": "Memory Usage" }, { "exemplar": true, "expr": "sum(kube_pod_container_resource{exported_pod=~\"$api_name.+\", resource=\"memory\"}) / 1024^2\n/\nsum(kube_pod_info{exported_pod=~\"$api_name.+\"})", "hide": false, "interval": "", "legendFormat": "Avg Memory Request", "refId": "Memory Request" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Avg Memory Usage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:1404", "format": "MiB", "label": "memory", "logBase": 1, "max": null, "min": "0", "show": true }, { "$$hashKey": "object:1405", "format": "short", "label": "", "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "Avg GPU Capacity": "semi-dark-orange", "Avg GPU Usage": "semi-dark-green", "Total GPU Capacity": "semi-dark-orange", "Total GPU Usage": "semi-dark-green", "Total GPU Utilization": "light-green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "description": "Avg GPU core usage across all workers of the API", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 46 }, "hiddenSeries": false, "id": 25, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(DCGM_FI_DEV_GPU_UTIL{exported_pod=~\"$api_name.+\"}) / 100\n/\ncount(DCGM_FI_DEV_GPU_UTIL{exported_pod=~\"$api_name.+\"})", "hide": false, "instant": false, "interval": "", "legendFormat": "Avg GPU Usage", "refId": "GPU Usage" }, { "expr": "count(DCGM_FI_DEV_GPU_UTIL{exported_pod=~\"$api_name.+\"})\n/\ncount(count(DCGM_FI_DEV_GPU_UTIL{exported_pod=~\"$api_name.+\"}) by (exported_pod))", "hide": false, "instant": false, "interval": "", "legendFormat": "Avg GPU Capacity", "refId": "GPU Capacity" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Avg GPU Core Usage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:1404", "format": "gpuCore", "label": "gpu", "logBase": 1, "max": null, "min": "0", "show": true }, { "$$hashKey": "object:1405", "format": "short", "label": "", "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "Avg Capacity GPU Memory": "semi-dark-orange", "Avg Used GPU Memory": "semi-dark-green", "Total Capacity GPU Memory": "semi-dark-orange", "Total Used GPU Memory": "semi-dark-green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "description": "Avg GPU memory usage across all workers of the API", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 46 }, "hiddenSeries": false, "id": 26, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(DCGM_FI_DEV_FB_USED{exported_pod=~\"$api_name.+\"})\n/\ncount(DCGM_FI_DEV_FB_USED{exported_pod=~\"$api_name.+\"})", "hide": false, "interval": "", "legendFormat": "Avg Used GPU Memory", "refId": "GPU Used Memory" }, { "exemplar": false, "expr": "(sum(DCGM_FI_DEV_FB_FREE{exported_pod=~\"$api_name.+\"}) + sum(DCGM_FI_DEV_FB_USED{exported_pod=~\"$api_name.+\"}))\n/\ncount(DCGM_FI_DEV_FB_USED{exported_pod=~\"$api_name.+\"})", "format": "time_series", "instant": false, "interval": "", "legendFormat": "Avg Capacity GPU Memory", "refId": "GPU Capacity Memory" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Avg GPU Memory Usage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:1404", "format": "MiB", "label": "memory", "logBase": 1, "max": null, "min": "0", "show": true }, { "$$hashKey": "object:1405", "format": "short", "label": "", "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } } ], "refresh": "30s", "schemaVersion": 30, "style": "dark", "tags": [], "templating": { "list": [ { "allValue": null, "current": { "selected": false, "text": "None", "value": "None" }, "datasource": null, "definition": "label_values({__name__=~\"cortex_batch_.+\"}, api_name)", "description": null, "error": null, "hide": 0, "includeAll": false, "label": "API Name", "multi": true, "name": "api_name", "options": [], "query": { "query": "label_values({__name__=~\"cortex_batch_.+\"}, api_name)", "refId": "StandardVariableQuery" }, "refresh": 2, "regex": "", "skipUrlSync": false, "sort": 0, "tagValuesQuery": "", "tagsQuery": "", "type": "query", "useTags": false } ] }, "time": { "from": "now-1h", "to": "now" }, "timepicker": {}, "timezone": "", "title": "BatchAPI", "uid": "batchapi", "version": 1 } ================================================ FILE: manager/manifests/grafana/grafana-dashboard-cluster.yaml ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. apiVersion: v1 kind: ConfigMap metadata: name: grafana-dashboard-cluster namespace: default data: cluster.json: | { "annotations": { "list": [ { "builtIn": 1, "datasource": "prometheus", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "type": "dashboard" } ] }, "editable": true, "gnetId": null, "graphTooltip": 0, "links": [], "panels": [ { "collapsed": false, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 23, "panels": [], "title": "Size", "type": "row" }, { "datasource": null, "gridPos": { "h": 2, "w": 24, "x": 0, "y": 1 }, "id": 16, "options": { "content": "

Cluster

\n", "mode": "html" }, "pluginVersion": "8.0.4", "timeFrom": null, "timeShift": null, "transparent": true, "type": "text" }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "fill": 10, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 3 }, "hiddenSeries": false, "id": 20, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 0, "links": [], "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "count(instance:node_cpu_utilisation:rate1m)", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "Total Nodes", "legendLink": "/dashboard/file/node-rsrc-use.json", "refId": "A", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Nodes", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:117", "decimals": 0, "format": "none", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "$$hashKey": "object:118", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "fill": 10, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 3 }, "hiddenSeries": false, "id": 21, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 0, "links": [], "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(kube_pod_info{exported_pod!~\"(aws-node|grafana|autoscaler|cluster-autoscaler|coredns|event-exporter|fluent-bit|kube-proxy|k8s-neuron-scheduler|kube-state-metrics|metrics-server|node-exporter|operator|operator-controller-manager|prometheus-operator|prometheus-prometheus|prometheus-statsd-exporter|dcgm-exporter|ingressgateway|istiod|activator|enqueuer|gateway|nvidia-device-plugin-daemonset|neuron-device-plugin-daemonset|async-gateway)-(.+)\"})", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "User Workload Pods", "legendLink": "/dashboard/file/node-rsrc-use.json", "refId": "A", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Pods", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:142", "decimals": 0, "format": "none", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "$$hashKey": "object:143", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "collapsed": false, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 10 }, "id": 18, "panels": [], "title": "Costs", "type": "row" }, { "aliasColors": { "Cortex System Costs": "light-red" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "description": "Total cluster costs include the EKS control plane, the system/user nodes, and the EBS volumes.", "fieldConfig": { "defaults": { "unit": "currencyUSD" }, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 11 }, "hiddenSeries": false, "id": 25, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "cortex_cluster_cost{api=\"false\", kind=\"false\", component=\"cluster-costs\"}", "interval": "", "legendFormat": "Total Cluster Costs", "refId": "Total Cluster Costs" }, { "exemplar": true, "expr": "cortex_cluster_cost{api=\"false\", kind=\"false\", component=\"cortex-system-costs\"}", "hide": false, "instant": false, "interval": "", "legendFormat": "Cortex System Costs", "refId": "Cortex System Costs" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Cluster Costs / hr", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:165", "format": "currencyUSD", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "$$hashKey": "object:166", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "Cortex System Costs": "light-red" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "description": "Compute formula: for each \"API kind\" pod, see how many replicas of a given pod can fit on a specific node and then take the total cost of running the node (which can be on-demand or spot) and divide it by the maximum number of pods that can be run. Then, add the cost of each pod to an accumulator for each \"API kind\". Finally, once the cost for every \"API kind\" is computed, renormalize the sum of all API kinds to the actual cost of the cluster (minus the cortex system costs) so that all API kinds summed will give you the cost of the cluster.", "fieldConfig": { "defaults": { "unit": "currencyUSD" }, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 19 }, "hiddenSeries": false, "id": 28, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum by (component) (cortex_cluster_cost{api=\"false\", kind=\"true\"})", "interval": "", "intervalFactor": 1, "legendFormat": "{{component}}", "refId": "Workload Costs by API Kind" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Workload Costs by API Kind / hr", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:165", "format": "currencyUSD", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "$$hashKey": "object:166", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "Cortex System Costs": "light-red" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "description": "Compute formula: for each \"API name\" pod, see how many replicas of a given pod can fit on a specific node and then take the total cost of running the node (which can be on-demand or spot) and divide it by the maximum number of pods that can be run. Then, add the cost of each pod to an accumulator for each \"API name\". Finally, once the cost for every \"API name\" is computed, renormalize the sum of all API names to the actual cost of the cluster (minus the cortex system costs) so that all API names summed will give you the cost of the cluster.", "fieldConfig": { "defaults": { "unit": "currencyUSD" }, "overrides": [] }, "fill": 2, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 19 }, "hiddenSeries": false, "id": 26, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum by (component) (cortex_cluster_cost{api=\"true\", kind=\"false\"})", "interval": "", "legendFormat": "{{component}}", "refId": "Workload Costs by API Name" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Workload Costs by API Name / hr", "tooltip": { "shared": true, "sort": 2, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:165", "format": "currencyUSD", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "$$hashKey": "object:166", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "collapsed": false, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 27 }, "id": 10, "panels": [], "repeat": null, "title": "CPU", "type": "row" }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "fill": 10, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 28 }, "hiddenSeries": false, "id": 1, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 0, "links": [], "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "cluster:cpu_utilization:ratio{}\n", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "", "legendLink": "/dashboard/file/node-rsrc-use.json", "refId": "A", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "CPU Utilisation", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:156", "format": "percentunit", "label": null, "logBase": 1, "max": 1, "min": 0, "show": true }, { "$$hashKey": "object:157", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "fill": 10, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 28 }, "hiddenSeries": false, "id": 2, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 0, "links": [], "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "cluster:load1:ratio{}\n", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "", "legendLink": "/dashboard/file/node-rsrc-use.json", "refId": "A", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "CPU Saturation (load1 per CPU)", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:181", "format": "percentunit", "label": null, "logBase": 1, "max": 1, "min": 0, "show": true }, { "$$hashKey": "object:182", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "collapsed": false, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 35 }, "id": 11, "panels": [], "repeat": null, "title": "Memory", "type": "row" }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "fill": 10, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 36 }, "hiddenSeries": false, "id": 3, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 0, "links": [], "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "cluster:memory_utilization:ratio{}\n", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "", "legendLink": "/dashboard/file/node-rsrc-use.json", "refId": "A", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Memory Utilisation", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:210", "format": "percentunit", "label": null, "logBase": 1, "max": 1, "min": 0, "show": true }, { "$$hashKey": "object:211", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "fill": 10, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 36 }, "hiddenSeries": false, "id": 4, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 0, "links": [], "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "cluster:vmstat_pgmajfault:rate1m{}", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "", "legendLink": "/dashboard/file/node-rsrc-use.json", "refId": "A", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Memory Saturation (Major Page Faults)", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:398", "format": "rps", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "$$hashKey": "object:399", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "collapsed": false, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 43 }, "id": 12, "panels": [], "repeat": null, "title": "Network", "type": "row" }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "fill": 10, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 44 }, "hiddenSeries": false, "id": 5, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 0, "links": [], "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "/ Receive/", "stack": "A" }, { "alias": "/ Transmit/", "stack": "B", "transform": "negative-Y" } ], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "cluster:network_receive_bytes_excluding_low:rate1m{}", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "{{some-non-existent-label}} Receive", "legendLink": "/dashboard/file/node-rsrc-use.json", "refId": "A", "step": 10 }, { "exemplar": true, "expr": "cluster:network_transmit_bytes_excluding_lo:rate1m{}", "format": "time_series", "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{some-non-existent-label}} Transmit", "legendLink": "/dashboard/file/node-rsrc-use.json", "refId": "B", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Net Utilisation (Bytes Receive/Transmit)", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:235", "format": "Bps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:236", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "fill": 10, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 44 }, "hiddenSeries": false, "id": 6, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 0, "links": [], "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "/ Receive/", "stack": "A" }, { "alias": "/ Transmit/", "stack": "B", "transform": "negative-Y" } ], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "cluster:network_receive_drop_excluding_lo:rate1m{}", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "{{some-non-existent-label}} Receive", "legendLink": "/dashboard/file/node-rsrc-use.json", "refId": "A", "step": 10 }, { "exemplar": true, "expr": "cluster:network_transmit_drop_excluding_lo:rate1m{}", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "{{some-non-existent-label}} Transmit", "legendLink": "/dashboard/file/node-rsrc-use.json", "refId": "B", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Net Saturation (Drops Receive/Transmit)", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:521", "format": "rps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:522", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "collapsed": false, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 51 }, "id": 13, "panels": [], "repeat": null, "title": "Disk IO", "type": "row" }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "fill": 10, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 52 }, "hiddenSeries": false, "id": 7, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 0, "links": [], "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "cluster:disk_io_utilization:ratio{}\n", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "", "legendLink": "/dashboard/file/node-rsrc-use.json", "refId": "A", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Disk IO Utilisation", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:446", "format": "percentunit", "label": null, "logBase": 1, "max": 1, "min": 0, "show": true }, { "$$hashKey": "object:447", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "fill": 10, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 52 }, "hiddenSeries": false, "id": 8, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 0, "links": [], "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "cluster:disk_io_saturation:ratio{}\n", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "", "legendLink": "/dashboard/file/node-rsrc-use.json", "refId": "A", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Disk IO Saturation", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:471", "format": "percentunit", "label": null, "logBase": 1, "max": 1, "min": 0, "show": true }, { "$$hashKey": "object:472", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "collapsed": false, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 59 }, "id": 14, "panels": [], "repeat": null, "title": "Disk Space", "type": "row" }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "fill": 10, "fillGradient": 0, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 60 }, "hiddenSeries": false, "id": 9, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 0, "links": [], "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "cluster:disk_space_utilization:ratio{}\n", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "", "legendLink": "/dashboard/file/node-rsrc-use.json", "refId": "A", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Disk Space Utilisation", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:496", "format": "percentunit", "label": null, "logBase": 1, "max": 1, "min": 0, "show": true }, { "$$hashKey": "object:497", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } } ], "refresh": "1m", "schemaVersion": 30, "style": "dark", "tags": [], "templating": { "list": [] }, "time": { "from": "now-12h", "to": "now" }, "timepicker": { "refresh_intervals": [ "5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d" ], "time_options": [ "5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d" ] }, "timezone": "", "title": "Cluster", "uid": "cluster", "version": 4 } ================================================ FILE: manager/manifests/grafana/grafana-dashboard-control-plane.yaml ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. apiVersion: v1 kind: ConfigMap metadata: name: grafana-dashboard-control-plane namespace: default data: control-plane.json: | { "annotations": { "list": [ { "builtIn": 1, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "type": "dashboard" } ] }, "description": "Metrics for API Server, Controller and Scheduler.", "editable": true, "gnetId": 10907, "graphTooltip": 0, "links": [], "panels": [ { "collapsed": false, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 18, "panels": [], "title": "API Server Summary", "type": "row" }, { "cacheTimeout": null, "datasource": null, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": null, "mappings": [], "max": 100, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 7, "w": 3, "x": 0, "y": 1 }, "id": 6, "links": [], "options": { "orientation": "auto", "reduceOptions": { "calcs": [ "last" ], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true, "text": {} }, "pluginVersion": "8.0.4", "targets": [ { "exemplar": true, "expr": "sum(rate(apiserver_request_total[5m]))", "format": "time_series", "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{verb}}", "refId": "A" } ], "timeFrom": null, "timeShift": null, "title": "API Server RPS", "type": "gauge" }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 5, "x": 3, "y": 1 }, "hiddenSeries": false, "id": 47, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(apiserver_current_inflight_requests) by (requestKind)", "legendFormat": "{{requestKind}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Current Inflight Requests", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 5, "x": 8, "y": 1 }, "hiddenSeries": false, "id": 32, "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": false, "sort": "current", "sortDesc": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket{verb!~\"WATCH|CONNECT\"}[5m])) by (le) )", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Latency - p99", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:45", "decimals": 2, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:46", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "dashLength": 10, "dashes": false, "datasource": null, "decimals": 2, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 5, "x": 13, "y": 1 }, "hiddenSeries": false, "id": 21, "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": true, "sort": "current", "sortDesc": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(rate(apiserver_request_duration_seconds_sum{verb!~\"WATCH|CONNECT\"}[5m])) ", "format": "time_series", "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "seconds", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "API Server Response Duration (Seconds)", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:101", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:102", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "dashLength": 10, "dashes": false, "datasource": null, "decimals": 0, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 5, "x": 18, "y": 1 }, "hiddenSeries": false, "id": 15, "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": true, "sort": null, "sortDesc": null, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(rate(apiserver_request_duration_seconds_bucket{code=~\"^(?:4..)$|^(?:5..)$\"}[5m])) by (code)>0", "format": "time_series", "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{code}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "API Server Errors", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:157", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:158", "decimals": 1, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "collapsed": false, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 8 }, "id": 24, "panels": [], "title": "API Server Details", "type": "row" }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "dashLength": 10, "dashes": false, "datasource": null, "decimals": 0, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 23, "x": 0, "y": 9 }, "hiddenSeries": false, "id": 12, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": false, "min": false, "rightSide": true, "show": true, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(rate(apiserver_request_total[5m])) by (verb)", "format": "time_series", "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{verb}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "API Server Requests (by Verb)", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 23, "x": 0, "y": 16 }, "hiddenSeries": false, "id": 8, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": false, "min": false, "rightSide": true, "show": true, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(rate(apiserver_request_total[5m])) by (resource)", "format": "time_series", "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{resource}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "API Requests (by Resource)", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "decimals": 0, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 23, "x": 0, "y": 23 }, "hiddenSeries": false, "id": 9, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": false, "min": false, "rightSide": true, "show": true, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(rate(apiserver_request_total{client!~\"kubelet.*|kube-scheduler.*|kube-controller.*|kube-apiserver.*|kube-proxy.*\"}[5m])) by (client)>0", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "{{client}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "API Requests", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 23, "x": 0, "y": 30 }, "hiddenSeries": false, "id": 2, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": false, "min": false, "rightSide": true, "show": true, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "histogram_quantile(0.90, sum(rate(apiserver_request_duration_seconds_bucket{verb!~\"WATCH|CONNECT\"}[5m])) by (le, resource) )", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "{{resource}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Latency by resource", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:252", "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:253", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 23, "x": 0, "y": 38 }, "hiddenSeries": false, "id": 49, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": false, "min": false, "rightSide": true, "show": true, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "histogram_quantile(0.90, sum(rate(apiserver_request_duration_seconds_bucket{verb!~\"WATCH|CONNECT\"}[5m])) by (le, verb) )", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "{{resource}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Latency by verb", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:194", "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:195", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "dashLength": 10, "dashes": false, "datasource": null, "decimals": 0, "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 23, "x": 0, "y": 46 }, "hiddenSeries": false, "id": 10, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": false, "min": false, "rightSide": true, "show": true, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "aggregation": "Last", "decimals": 2, "displayAliasType": "Warning / Critical", "displayType": "Regular", "displayValueWithAlias": "Never", "exemplar": true, "expr": "sum(rate(apiserver_request_total[5m])) by (instance)>0", "format": "time_series", "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}} ", "refId": "A", "units": "none", "valueHandler": "Number Threshold" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "API Server Requests by server", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:223", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:224", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } } ], "refresh": "30s", "schemaVersion": 30, "style": "dark", "tags": [], "templating": { "list": [] }, "time": { "from": "now-1h", "to": "now" }, "timepicker": { "refresh_intervals": [ "5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d" ], "time_options": [ "5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d" ] }, "timezone": "", "title": "K8S Control Plane", "uid": "k8scontrolplane", "version": 2 } ================================================ FILE: manager/manifests/grafana/grafana-dashboard-nodes.yaml ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. apiVersion: v1 kind: ConfigMap metadata: name: grafana-dashboard-nodes namespace: default data: nodes.json: | { "annotations": { "list": [ { "builtIn": 1, "datasource": "prometheus", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "type": "dashboard" } ] }, "editable": true, "gnetId": null, "graphTooltip": 0, "iteration": 1625177052451, "links": [], "panels": [ { "datasource": null, "description": "", "gridPos": { "h": 2, "w": 24, "x": 0, "y": 0 }, "id": 11, "options": { "content": "

Nodes

\n", "mode": "html" }, "pluginVersion": "8.0.4", "timeFrom": null, "timeShift": null, "transparent": true, "type": "text" }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 2 }, "hiddenSeries": false, "id": 2, "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "seriesOverrides": [], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { "expr": "(\n (1 - rate(node_cpu_seconds_total{job=\"node-exporter\", mode=\"idle\", instance=\"$instance\"}[$__interval]))\n/ ignoring(cpu) group_left\n count without (cpu)( node_cpu_seconds_total{job=\"node-exporter\", mode=\"idle\", instance=\"$instance\"})\n)\n", "format": "time_series", "interval": "1m", "intervalFactor": 5, "legendFormat": "{{cpu}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "CPU Usage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "percentunit", "label": null, "logBase": 1, "max": 1, "min": 0, "show": true }, { "format": "percentunit", "label": null, "logBase": 1, "max": 1, "min": 0, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "fill": 0, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 2 }, "hiddenSeries": false, "id": 3, "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "node_load1{job=\"node-exporter\", instance=\"$instance\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "1m load average", "refId": "A" }, { "expr": "node_load5{job=\"node-exporter\", instance=\"$instance\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "5m load average", "refId": "B" }, { "expr": "node_load15{job=\"node-exporter\", instance=\"$instance\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "15m load average", "refId": "C" }, { "expr": "count(node_cpu_seconds_total{job=\"node-exporter\", instance=\"$instance\", mode=\"idle\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "logical cores", "refId": "D" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Load Average", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": 0, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 18, "x": 0, "y": 9 }, "hiddenSeries": false, "id": 4, "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "seriesOverrides": [], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { "expr": "(\n node_memory_MemTotal_bytes{job=\"node-exporter\", instance=\"$instance\"}\n-\n node_memory_MemFree_bytes{job=\"node-exporter\", instance=\"$instance\"}\n-\n node_memory_Buffers_bytes{job=\"node-exporter\", instance=\"$instance\"}\n-\n node_memory_Cached_bytes{job=\"node-exporter\", instance=\"$instance\"}\n)\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "memory used", "refId": "A" }, { "expr": "node_memory_Buffers_bytes{job=\"node-exporter\", instance=\"$instance\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "memory buffers", "refId": "B" }, { "expr": "node_memory_Cached_bytes{job=\"node-exporter\", instance=\"$instance\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "memory cached", "refId": "C" }, { "expr": "node_memory_MemFree_bytes{job=\"node-exporter\", instance=\"$instance\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "memory free", "refId": "D" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Memory Usage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "bytes", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "bytes", "label": null, "logBase": 1, "max": null, "min": 0, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "cacheTimeout": null, "datasource": null, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [ { "options": { "match": "null", "result": { "text": "N/A" } }, "type": "special" } ], "max": 100, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "rgba(50, 172, 45, 0.97)", "value": null }, { "color": "rgba(237, 129, 40, 0.89)", "value": 80 }, { "color": "rgba(245, 54, 54, 0.9)", "value": 90 } ] }, "unit": "percent" }, "overrides": [] }, "gridPos": { "h": 7, "w": 6, "x": 18, "y": 9 }, "id": 5, "interval": null, "links": [], "maxDataPoints": 100, "options": { "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true, "text": {} }, "pluginVersion": "8.0.4", "targets": [ { "expr": "100 -\n(\n avg(node_memory_MemAvailable_bytes{job=\"node-exporter\", instance=\"$instance\"})\n/\n avg(node_memory_MemTotal_bytes{job=\"node-exporter\", instance=\"$instance\"})\n* 100\n)\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "", "refId": "A" } ], "title": "Memory Usage", "type": "gauge" }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "fill": 0, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 16 }, "hiddenSeries": false, "id": 6, "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "seriesOverrides": [ { "alias": "/ read| written/", "yaxis": 1 }, { "alias": "/ io time/", "yaxis": 2 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "rate(node_disk_read_bytes_total{job=\"node-exporter\", instance=\"$instance\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\"}[$__interval])", "format": "time_series", "interval": "1m", "intervalFactor": 2, "legendFormat": "{{device}} read", "refId": "A" }, { "expr": "rate(node_disk_written_bytes_total{job=\"node-exporter\", instance=\"$instance\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\"}[$__interval])", "format": "time_series", "interval": "1m", "intervalFactor": 2, "legendFormat": "{{device}} written", "refId": "B" }, { "expr": "rate(node_disk_io_time_seconds_total{job=\"node-exporter\", instance=\"$instance\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\"}[$__interval])", "format": "time_series", "interval": "1m", "intervalFactor": 2, "legendFormat": "{{device}} io time", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Disk I/O", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 16 }, "hiddenSeries": false, "id": 7, "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "seriesOverrides": [ { "alias": "used", "color": "#E0B400" }, { "alias": "available", "color": "#73BF69" } ], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { "expr": "sum(\n max by (device) (\n node_filesystem_size_bytes{job=\"node-exporter\", instance=\"$instance\", fstype!=\"\"}\n -\n node_filesystem_avail_bytes{job=\"node-exporter\", instance=\"$instance\", fstype!=\"\"}\n )\n)\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "used", "refId": "A" }, { "expr": "sum(\n max by (device) (\n node_filesystem_avail_bytes{job=\"node-exporter\", instance=\"$instance\", fstype!=\"\"}\n )\n)\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "available", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Disk Space Usage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "bytes", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "bytes", "label": null, "logBase": 1, "max": null, "min": 0, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "fill": 0, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 23 }, "hiddenSeries": false, "id": 8, "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "rate(node_network_receive_bytes_total{job=\"node-exporter\", instance=\"$instance\", device!=\"lo\"}[$__interval])", "format": "time_series", "interval": "1m", "intervalFactor": 2, "legendFormat": "{{device}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Network Received", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "bytes", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "bytes", "label": null, "logBase": 1, "max": null, "min": 0, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "fill": 0, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 23 }, "hiddenSeries": false, "id": 9, "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "rate(node_network_transmit_bytes_total{job=\"node-exporter\", instance=\"$instance\", device!=\"lo\"}[$__interval])", "format": "time_series", "interval": "1m", "intervalFactor": 2, "legendFormat": "{{device}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Network Transmitted", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "bytes", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "bytes", "label": null, "logBase": 1, "max": null, "min": 0, "show": true } ], "yaxis": { "align": false, "alignLevel": null } } ], "refresh": "1m", "schemaVersion": 30, "style": "dark", "tags": [], "templating": { "list": [ { "allValue": null, "current": {}, "datasource": null, "definition": "", "description": null, "error": null, "hide": 0, "includeAll": false, "label": null, "multi": false, "name": "instance", "options": [], "query": { "query": "label_values(node_exporter_build_info{job=\"node-exporter\"}, instance)", "refId": "prometheus-instance-Variable-Query" }, "refresh": 2, "regex": "", "skipUrlSync": false, "sort": 0, "tagValuesQuery": "", "tagsQuery": "", "type": "query", "useTags": false } ] }, "time": { "from": "now-12h", "to": "now" }, "timepicker": { "refresh_intervals": [ "5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d" ], "time_options": [ "5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d" ] }, "timezone": "", "title": "Nodes", "uid": "nodes", "version": 1 } ================================================ FILE: manager/manifests/grafana/grafana-dashboard-realtime.yaml ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. apiVersion: v1 kind: ConfigMap metadata: name: grafana-dashboard-realtime namespace: default data: realtime.json: |- { "annotations": { "list": [ { "builtIn": 1, "datasource": "prometheus", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "type": "dashboard" } ] }, "editable": true, "gnetId": null, "graphTooltip": 0, "iteration": 1625175811197, "links": [], "panels": [ { "datasource": null, "gridPos": { "h": 2, "w": 24, "x": 0, "y": 0 }, "id": 15, "options": { "content": "

RealtimeAPI

", "mode": "markdown" }, "pluginVersion": "8.0.4", "timeFrom": null, "timeShift": null, "transparent": true, "type": "text" }, { "collapsed": false, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 2 }, "id": 22, "panels": [], "title": "API Stats", "type": "row" }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "description": "Request rate, computed over every minute, of an API", "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 3 }, "hiddenSeries": false, "id": 2, "legend": { "alignAsTable": true, "avg": true, "current": true, "max": true, "min": true, "rightSide": false, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum (rate(istio_requests_total{destination_service=~\"api-$api_name.+\"}[1m])) by (destination_service)", "interval": "", "legendFormat": "{{destination_service}}", "refId": "Request Rate" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Request Rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [ { "id": "renameByRegex", "options": { "regex": "([^\\.]+)\\..+", "renamePattern": "$1" } }, { "id": "renameByRegex", "options": { "regex": "api-(.*)", "renamePattern": "$1" } } ], "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "reqps", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "description": "Active in-flight requests for an API.\n\nNote: In-flight requests are recorded every 10 seconds, which will correspond to the minimum resolution.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 3 }, "hiddenSeries": false, "id": 4, "legend": { "alignAsTable": true, "avg": true, "current": true, "max": true, "min": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(cortex_in_flight_requests{api_name=~\"$api_name\"}) by (api_name)", "interval": "", "legendFormat": "{{api_name}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "In-Flight Requests", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": 0, "format": "short", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "description": "Request rate, computed over every minute, for responses with status code 2XX of an API", "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 12 }, "hiddenSeries": false, "id": 8, "legend": { "alignAsTable": true, "avg": true, "current": true, "max": true, "min": true, "rightSide": false, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(rate(istio_requests_total{destination_service=~\"api-$api_name.+\", response_code=~\"2.+\"}[1m])) by (destination_service, response_code)", "interval": "", "legendFormat": "{{destination_service}}", "refId": "2XX" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "2XX Responses", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [ { "id": "renameByRegex", "options": { "regex": "([^\\.]+)\\..+", "renamePattern": "$1" } }, { "id": "renameByRegex", "options": { "regex": "api-(.*)", "renamePattern": "$1" } } ], "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:1217", "format": "reqps", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "$$hashKey": "object:1218", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 12 }, "hiddenSeries": false, "id": 7, "legend": { "alignAsTable": true, "avg": true, "current": true, "max": true, "min": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "count(cortex_in_flight_requests{api_name=~\"$api_name\", container!=\"activator\"}) by (api_name)", "interval": "", "legendFormat": "{{api_name}}", "refId": "Active Replicas" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Active Replicas", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:236", "decimals": 0, "format": "short", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "$$hashKey": "object:237", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "description": "Request rate, computed over every minute, for responses with status code 4XX of an API", "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 21 }, "hiddenSeries": false, "id": 9, "legend": { "alignAsTable": true, "avg": true, "current": true, "max": true, "min": true, "rightSide": false, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(rate(istio_requests_total{destination_service=~\"api-$api_name.+\", response_code=~\"4.+\"}[1m])) by (destination_service, response_code)", "interval": "", "legendFormat": "{{destination_service}}", "refId": "4XX" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "4XX Responses", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [ { "id": "renameByRegex", "options": { "regex": "([^\\.]+)\\..+", "renamePattern": "$1" } }, { "id": "renameByRegex", "options": { "regex": "api-(.*)", "renamePattern": "$1" } } ], "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "reqps", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "description": "Request rate, computed over every minute, for responses with status code 5XX of an API", "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 21 }, "hiddenSeries": false, "id": 10, "legend": { "alignAsTable": true, "avg": true, "current": true, "max": true, "min": true, "rightSide": false, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(rate(istio_requests_total{destination_service=~\"api-$api_name.+\", response_code=~\"5.+\"}[1m])) by (destination_service, response_code)", "interval": "", "legendFormat": "{{destination_service}}", "refId": "5XX" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "5XX Responses", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [ { "id": "renameByRegex", "options": { "regex": "([^\\.]+)\\..+", "renamePattern": "$1" } }, { "id": "renameByRegex", "options": { "regex": "api-(.*)", "renamePattern": "$1" } } ], "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "reqps", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "description": "99th percentile latency, computed over a minute, for an API", "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 30 }, "hiddenSeries": false, "id": 6, "legend": { "alignAsTable": true, "avg": true, "current": true, "max": true, "min": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "histogram_quantile(0.99, sum by (destination_service, le) (rate(istio_request_duration_milliseconds_bucket{destination_service=~\"api-$api_name.+\"}[1m])))", "interval": "", "legendFormat": "{{destination_service}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "p99 Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [ { "id": "renameByRegex", "options": { "regex": "([^\\.]+)\\..+", "renamePattern": "$1" } }, { "id": "renameByRegex", "options": { "regex": "api-(.*)", "renamePattern": "$1" } } ], "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:1302", "format": "ms", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "$$hashKey": "object:1303", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "description": "90th percentile latency, computed over a minute, for an API", "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 30 }, "hiddenSeries": false, "id": 11, "legend": { "alignAsTable": true, "avg": true, "current": true, "max": true, "min": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "histogram_quantile(0.90, sum by (destination_service, le) (rate(istio_request_duration_milliseconds_bucket{destination_service=~\"api-$api_name.+\"}[1m])))", "hide": false, "interval": "", "legendFormat": "{{destination_service}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "p90 Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [ { "id": "renameByRegex", "options": { "regex": "([^\\.]+)\\..+", "renamePattern": "$1" } }, { "id": "renameByRegex", "options": { "regex": "api-(.*)", "renamePattern": "$1" } } ], "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "ms", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "description": "50th percentile latency, computed over a minute, for an API", "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 39 }, "hiddenSeries": false, "id": 16, "legend": { "alignAsTable": true, "avg": true, "current": true, "max": true, "min": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "histogram_quantile(0.50, sum by (destination_service, le) (rate(istio_request_duration_milliseconds_bucket{destination_service=~\"api-$api_name.+\"}[1m])))", "hide": false, "interval": "", "legendFormat": "{{destination_service}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "p50 Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [ { "id": "renameByRegex", "options": { "regex": "([^\\.]+)\\..+", "renamePattern": "$1" } }, { "id": "renameByRegex", "options": { "regex": "api-(.*)", "renamePattern": "$1" } } ], "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "ms", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "description": "Average latency, computed over a minute, for an API", "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 39 }, "hiddenSeries": false, "id": 12, "legend": { "alignAsTable": true, "avg": true, "current": true, "max": true, "min": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(rate(istio_request_duration_milliseconds_sum{destination_service=~\"api-$api_name.+\"}[1m])) by (destination_service) / sum(rate(istio_request_duration_milliseconds_count{destination_service=~\"api-$api_name.+\"}[1m])) by (destination_service)", "hide": false, "interval": "", "legendFormat": "{{destination_service}}", "refId": "D" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Average Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [ { "id": "renameByRegex", "options": { "regex": "([^\\.]+)\\..+", "renamePattern": "$1" } }, { "id": "renameByRegex", "options": { "regex": "api-(.*)", "renamePattern": "$1" } } ], "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "ms", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "collapsed": false, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 48 }, "id": 20, "panels": [], "title": "Aggregate Usage", "type": "row" }, { "aliasColors": { "Total CPU Request": "semi-dark-orange", "Total CPU Usage": "semi-dark-green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "description": "Total CPU usage across all replicas of the API", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 49 }, "hiddenSeries": false, "id": 24, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "sum(rate(container_cpu_usage_seconds_total{pod=~\"api-$api_name.+\", container!=\"POD\", name!=\"\"}[1m]))", "format": "time_series", "instant": false, "interval": "", "legendFormat": "Total CPU Usage", "refId": "CPU Usage" }, { "exemplar": true, "expr": "sum(kube_pod_container_resource_requests{exported_pod=~\"api-$api_name.+\", resource=\"cpu\"})", "hide": false, "interval": "", "legendFormat": "Total CPU Request", "refId": "CPU Request" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Total CPU Usage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:1404", "format": "core", "label": "cpu", "logBase": 1, "max": null, "min": "0", "show": true }, { "$$hashKey": "object:1405", "format": "short", "label": "", "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "Total Memory Request": "semi-dark-orange", "Total Memory Usage": "semi-dark-green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "description": "Total memory usage across all replicas of the API", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 49 }, "hiddenSeries": false, "id": 26, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "sum(sum_over_time(container_memory_working_set_bytes{pod=~\"api-$api_name.+\", name!=\"\", container!=\"POD\"}[1m])) /\navg(count_over_time(container_memory_working_set_bytes{pod=~\"api-$api_name.+\", name!=\"\", container=\"api\"}[1m])) / 1024^2", "format": "time_series", "instant": false, "interval": "", "legendFormat": "Total Memory Usage", "refId": "Memory Usage" }, { "exemplar": true, "expr": "sum(kube_pod_container_resource_requests{exported_pod=~\"api-$api_name.+\", resource=\"memory\"}) / 1024^2", "hide": false, "interval": "", "legendFormat": "Total Memory Request", "refId": "Memory Request" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Total Memory Usage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:1404", "format": "MiB", "label": "memory", "logBase": 1, "max": null, "min": "0", "show": true }, { "$$hashKey": "object:1405", "format": "short", "label": "", "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "Total GPU Capacity": "semi-dark-orange", "Total GPU Usage": "semi-dark-green", "Total GPU Utilization": "light-green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "description": "Total GPU core usage across all replicas of the API", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 57 }, "hiddenSeries": false, "id": 28, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(DCGM_FI_DEV_GPU_UTIL{exported_pod=~\"api-$api_name.+\"}) / 100", "hide": false, "interval": "", "legendFormat": "Total GPU Usage", "refId": "GPU Usage" }, { "exemplar": true, "expr": "count(DCGM_FI_DEV_GPU_UTIL{exported_pod=~\"api-$api_name.+\"})", "hide": false, "interval": "", "legendFormat": "Total GPU Capacity", "refId": "GPU Capacity" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Total GPU Core Usage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:1404", "format": "gpuCore", "label": "gpu", "logBase": 1, "max": null, "min": "0", "show": true }, { "$$hashKey": "object:1405", "format": "short", "label": "", "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "Total Capacity GPU Memory": "semi-dark-orange", "Total Used GPU Memory": "semi-dark-green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "description": "Total GPU memory usage across all replicas of the API", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 57 }, "hiddenSeries": false, "id": 29, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(DCGM_FI_DEV_FB_USED{exported_pod=~\"api-$api_name.+\"})", "hide": false, "interval": "", "legendFormat": "Total Used GPU Memory", "refId": "GPU Used Memory" }, { "exemplar": false, "expr": "sum(DCGM_FI_DEV_FB_FREE{exported_pod=~\"api-$api_name.+\"}) + sum(DCGM_FI_DEV_FB_USED{exported_pod=~\"api-$api_name.+\"})", "format": "time_series", "instant": false, "interval": "", "legendFormat": "Total Capacity GPU Memory", "refId": "GPU Capacity Memory" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Total GPU Memory Usage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:1404", "format": "MiB", "label": "memory", "logBase": 1, "max": null, "min": "0", "show": true }, { "$$hashKey": "object:1405", "format": "short", "label": "", "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "collapsed": false, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 65 }, "id": 18, "panels": [], "title": "Average Replica Usage", "type": "row" }, { "aliasColors": { "Avg CPU Request": "semi-dark-orange", "Avg CPU Usage": "semi-dark-green", "Total CPU Request": "semi-dark-orange", "Total CPU Usage": "semi-dark-green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "description": "Avg CPU usage across all replicas of the API", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 66 }, "hiddenSeries": false, "id": 30, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "sum(rate(container_cpu_usage_seconds_total{pod=~\"api-$api_name.+\", container!=\"POD\", name!=\"\"}[1m]))\n/\nsum(kube_pod_info{exported_pod=~\"api-$api_name.+\"})", "format": "time_series", "instant": false, "interval": "", "legendFormat": "Avg CPU Usage", "refId": "CPU Usage" }, { "exemplar": true, "expr": "sum(kube_pod_container_resource_requests{exported_pod=~\"api-$api_name.+\", resource=\"cpu\"})\n/\nsum(kube_pod_info{exported_pod=~\"api-$api_name.+\"})", "hide": false, "interval": "", "legendFormat": "Avg CPU Request", "refId": "CPU Request" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Avg CPU Usage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:1404", "format": "core", "label": "cpu", "logBase": 1, "max": null, "min": "0", "show": true }, { "$$hashKey": "object:1405", "format": "short", "label": "", "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "Avg Memory Request": "semi-dark-orange", "Avg Memory Usage": "semi-dark-green", "Total Memory Request": "semi-dark-orange", "Total Memory Usage": "semi-dark-green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "description": "Avg memory usage across all replicas of the API", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 66 }, "hiddenSeries": false, "id": 31, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "sum(sum_over_time(container_memory_working_set_bytes{pod=~\"api-$api_name.+\", name!=\"\", container!=\"POD\"}[1m]))\n/\navg(count_over_time(container_memory_working_set_bytes{pod=~\"api-$api_name.+\", name!=\"\", container=\"api\"}[1m])) / 1024^2\n/\nsum(kube_pod_info{exported_pod=~\"api-$api_name.+\"})", "format": "time_series", "instant": false, "interval": "", "legendFormat": "Avg Memory Usage", "refId": "Memory Usage" }, { "exemplar": true, "expr": "sum(kube_pod_container_resource_requests{exported_pod=~\"api-$api_name.+\", resource=\"memory\"}) / 1024^2\n/\nsum(kube_pod_info{exported_pod=~\"api-$api_name.+\"})", "hide": false, "interval": "", "legendFormat": "Avg Memory Request", "refId": "Memory Request" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Avg Memory Usage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:1404", "format": "MiB", "label": "memory", "logBase": 1, "max": null, "min": "0", "show": true }, { "$$hashKey": "object:1405", "format": "short", "label": "", "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "Avg GPU Capacity": "semi-dark-orange", "Avg GPU Usage": "semi-dark-green", "Total GPU Capacity": "semi-dark-orange", "Total GPU Utilization": "semi-dark-green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "description": "Avg GPU core usage across all replicas of the API", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 74 }, "hiddenSeries": false, "id": 32, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(DCGM_FI_DEV_GPU_UTIL{exported_pod=~\"api-$api_name.+\"}) / 100\n/\ncount(DCGM_FI_DEV_GPU_UTIL{exported_pod=~\"api-$api_name.+\"})", "hide": false, "interval": "", "legendFormat": "Avg GPU Usage", "refId": "GPU Usage" }, { "expr": "count(DCGM_FI_DEV_GPU_UTIL{exported_pod=~\"api-$api_name.+\"})\n/\ncount(count(DCGM_FI_DEV_GPU_UTIL{exported_pod=~\"api-$api_name.+\"}) by (exported_pod))", "hide": false, "interval": "", "legendFormat": "Avg GPU Capacity", "refId": "GPU Capacity" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Avg GPU Core Usage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:1404", "format": "gpuCore", "label": "gpu", "logBase": 1, "max": null, "min": "0", "show": true }, { "$$hashKey": "object:1405", "format": "short", "label": "", "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "Avg Capacity GPU Memory": "semi-dark-orange", "Avg Used GPU Memory": "semi-dark-green", "Total Capacity GPU Memory": "semi-dark-orange", "Total Used GPU Memory": "semi-dark-green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "description": "Avg GPU memory usage across all replicas of the API", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 74 }, "hiddenSeries": false, "id": 33, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(DCGM_FI_DEV_FB_USED{exported_pod=~\"api-$api_name.+\"})\n/\ncount(DCGM_FI_DEV_FB_USED{exported_pod=~\"api-$api_name.+\"})", "hide": false, "interval": "", "legendFormat": "Avg Used GPU Memory", "refId": "GPU Used Memory" }, { "exemplar": false, "expr": "(sum(DCGM_FI_DEV_FB_FREE{exported_pod=~\"api-$api_name.+\"}) + sum(DCGM_FI_DEV_FB_USED{exported_pod=~\"api-$api_name.+\"}))\n/\ncount(DCGM_FI_DEV_FB_USED{exported_pod=~\"api-$api_name.+\"})", "format": "time_series", "instant": false, "interval": "", "legendFormat": "Avg Capacity GPU Memory", "refId": "GPU Capacity Memory" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Avg GPU Memory Usage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:1404", "format": "MiB", "label": "memory", "logBase": 1, "max": null, "min": "0", "show": true }, { "$$hashKey": "object:1405", "format": "short", "label": "", "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } } ], "refresh": "30s", "schemaVersion": 30, "style": "dark", "tags": [], "templating": { "list": [ { "allValue": null, "current": { "selected": false, "text": "None", "value": "None" }, "datasource": null, "definition": "label_values(cortex_in_flight_requests, api_name)", "description": null, "error": null, "hide": 0, "includeAll": false, "label": "API Name", "multi": true, "name": "api_name", "options": [], "query": { "query": "label_values(cortex_in_flight_requests, api_name)", "refId": "StandardVariableQuery" }, "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 0, "tagValuesQuery": "", "tagsQuery": "", "type": "query", "useTags": false } ] }, "time": { "from": "now-1h", "to": "now" }, "timepicker": {}, "timezone": "", "title": "RealtimeAPI", "uid": "realtimeapi", "version": 3 } ================================================ FILE: manager/manifests/grafana/grafana-dashboard-task.yaml ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. apiVersion: v1 kind: ConfigMap metadata: name: grafana-dashboard-task namespace: default data: task.json: |- { "annotations": { "list": [ { "builtIn": 1, "datasource": "prometheus", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "type": "dashboard" } ] }, "editable": true, "gnetId": null, "graphTooltip": 0, "id": 7, "iteration": 1625281006508, "links": [], "panels": [ { "datasource": null, "gridPos": { "h": 2, "w": 24, "x": 0, "y": 0 }, "id": 7, "options": { "content": "

TaskAPI

\n", "mode": "markdown" }, "pluginVersion": "8.0.4", "timeFrom": null, "timeShift": null, "transparent": true, "type": "text" }, { "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 2 }, "id": 22, "title": "API Stats", "type": "row" }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "description": "Number of succeeded tasks for an API", "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 3 }, "hiddenSeries": false, "id": 2, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(cortex_task_succeeded{api_name=~\"$api_name\"}) by (api_name)", "interval": "", "legendFormat": "{{api_name}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "# Succeeded Tasks", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:26", "decimals": 0, "format": "short", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "$$hashKey": "object:27", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "description": "Number of failed tasks for an API", "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 3 }, "hiddenSeries": false, "id": 3, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(cortex_task_failed{api_name=~\"$api_name\"}) by (api_name)", "interval": "", "legendFormat": "{{api_name}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "# Failed Tasks", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:262", "decimals": 0, "format": "short", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "$$hashKey": "object:263", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "description": "Average time per task for an API", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 12 }, "hiddenSeries": false, "id": 5, "legend": { "alignAsTable": true, "avg": true, "current": true, "max": true, "min": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(cortex_time_per_task_sum{api_name=~\"$api_name\"}) by (api_name) / sum(cortex_time_per_task_count{api_name=~\"$api_name\"}) by (api_name)", "interval": "", "legendFormat": "{{api_name}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Average Time per Task", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:161", "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "$$hashKey": "object:162", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "Active Jobs": "semi-dark-green", "Active Workers": "semi-dark-orange" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "description": "Active tasks", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 12 }, "hiddenSeries": false, "id": 20, "legend": { "alignAsTable": true, "avg": true, "current": true, "max": true, "min": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "count(kube_job_status_active{job_name=~\"$api_name.+\"} != 0)", "interval": "", "legendFormat": "Active Tasks", "refId": "Active Tasks" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "# Active Tasks", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:128", "decimals": 0, "format": "count", "label": "", "logBase": 1, "max": null, "min": "0", "show": true }, { "$$hashKey": "object:129", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 20 }, "id": 11, "title": "Aggregate Worker Usage", "type": "row" }, { "aliasColors": { "Total CPU Request": "semi-dark-orange", "Total CPU Usage": "semi-dark-green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "description": "Total CPU usage across all workers of the API", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 21 }, "hiddenSeries": false, "id": 13, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "sum(rate(container_cpu_usage_seconds_total{pod=~\"$api_name.+\", container!=\"POD\", name!=\"\"}[1m]))", "format": "time_series", "instant": false, "interval": "", "legendFormat": "Total CPU Usage", "refId": "CPU Usage" }, { "exemplar": true, "expr": "sum(kube_pod_container_resource_requests{exported_pod=~\"$api_name.+\", resource=\"cpu\"})", "hide": false, "interval": "", "legendFormat": "Total CPU Request", "refId": "CPU Request" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Total CPU Usage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:1404", "format": "core", "label": "cpu", "logBase": 1, "max": null, "min": "0", "show": true }, { "$$hashKey": "object:1405", "format": "short", "label": "", "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "Total Memory Request": "semi-dark-orange", "Total Memory Usage": "semi-dark-green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "description": "Total memory usage across all workers of the API", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 21 }, "hiddenSeries": false, "id": 15, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "sum(sum_over_time(container_memory_working_set_bytes{pod=~\"$api_name.+\", name!=\"\", container!=\"\"}[1m]))\n/\navg(count_over_time(container_memory_working_set_bytes{pod=~\"$api_name.+\", name!=\"\", container!=\"\"}[1m])) / 1024^2", "format": "time_series", "instant": false, "interval": "", "legendFormat": "Total Memory Usage", "refId": "Memory Usage" }, { "exemplar": true, "expr": "sum(kube_pod_container_resource_requests{exported_pod=~\"$api_name.+\", resource=\"memory\"}) / 1024^2", "hide": false, "interval": "", "legendFormat": "Total Memory Request", "refId": "Memory Request" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Total Memory Usage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:1404", "format": "MiB", "label": "memory", "logBase": 1, "max": null, "min": "0", "show": true }, { "$$hashKey": "object:1405", "format": "short", "label": "", "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "Total GPU Capacity": "semi-dark-orange", "Total GPU Usage": "semi-dark-green", "Total GPU Utilization": "light-green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "description": "Total GPU core usage across all workers of the API", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 29 }, "hiddenSeries": false, "id": 17, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(DCGM_FI_DEV_GPU_UTIL{exported_pod=~\"$api_name.+\"}) / 100", "hide": false, "interval": "", "legendFormat": "Total GPU Usage", "refId": "GPU Usage" }, { "expr": "count(DCGM_FI_DEV_GPU_UTIL{exported_pod=~\"$api_name.+\"})", "hide": false, "interval": "", "legendFormat": "Total GPU Capacity", "refId": "GPU Capacity" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Total GPU Core Usage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:1404", "format": "gpuCore", "label": "gpu", "logBase": 1, "max": null, "min": "0", "show": true }, { "$$hashKey": "object:1405", "format": "short", "label": "", "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "Total Capacity GPU Memory": "semi-dark-orange", "Total Used GPU Memory": "semi-dark-green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "description": "Total GPU memory usage across all workers of the API", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 29 }, "hiddenSeries": false, "id": 19, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(DCGM_FI_DEV_FB_USED{exported_pod=~\"$api_name.+\"})", "hide": false, "interval": "", "legendFormat": "Total Used GPU Memory", "refId": "GPU Used Memory" }, { "exemplar": false, "expr": "sum(DCGM_FI_DEV_FB_FREE{exported_pod=~\"$api_name.+\"}) + sum(DCGM_FI_DEV_FB_USED{exported_pod=~\"$api_name.+\"})", "format": "time_series", "instant": false, "interval": "", "legendFormat": "Total Capacity GPU Memory", "refId": "GPU Capacity Memory" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Total GPU Memory Usage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:1404", "format": "MiB", "label": "memory", "logBase": 1, "max": null, "min": "0", "show": true }, { "$$hashKey": "object:1405", "format": "short", "label": "", "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "collapsed": false, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 37 }, "id": 9, "panels": [], "title": "Avg Worker Usage", "type": "row" }, { "aliasColors": { "Avg CPU Request": "semi-dark-orange", "Avg CPU Usage": "semi-dark-green", "Total CPU Request": "semi-dark-orange", "Total CPU Usage": "semi-dark-green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "description": "Avg CPU usage across all workers of the API", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 38 }, "hiddenSeries": false, "id": 23, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "sum(rate(container_cpu_usage_seconds_total{pod=~\"$api_name.+\", container!=\"POD\", name!=\"\"}[1m]))\n/\nsum(kube_pod_info{exported_pod=~\"$api_name.+\"})", "format": "time_series", "instant": false, "interval": "", "legendFormat": "Avg CPU Usage", "refId": "CPU Usage" }, { "exemplar": true, "expr": "sum(kube_pod_container_resource_requests{exported_pod=~\"$api_name.+\", resource=\"cpu\"})\n/\nsum(kube_pod_info{exported_pod=~\"$api_name.+\"})", "hide": false, "interval": "", "legendFormat": "Avg CPU Request", "refId": "CPU Request" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Avg CPU Usage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:1404", "format": "core", "label": "cpu", "logBase": 1, "max": null, "min": "0", "show": true }, { "$$hashKey": "object:1405", "format": "short", "label": "", "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "Avg Memory Request": "semi-dark-orange", "Avg Memory Usage": "semi-dark-green", "Total Memory Request": "semi-dark-orange", "Total Memory Usage": "semi-dark-green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "description": "Avg memory usage across all workers of the API", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 38 }, "hiddenSeries": false, "id": 24, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": false, "expr": "sum(sum_over_time(container_memory_working_set_bytes{pod=~\"$api_name.+\", name!=\"\", container!=\"\"}[1m]))\n/\navg(count_over_time(container_memory_working_set_bytes{pod=~\"$api_name.+\", name!=\"\", container!=\"\"}[1m])) / 1024^2\n/\nsum(kube_pod_info{exported_pod=~\"$api_name.+\"})", "format": "time_series", "instant": false, "interval": "", "legendFormat": "Avg Memory Usage", "refId": "Memory Usage" }, { "exemplar": true, "expr": "sum(kube_pod_container_resource{exported_pod=~\"$api_name.+\", resource=\"memory\"}) / 1024^2\n/\nsum(kube_pod_info{exported_pod=~\"$api_name.+\"})", "hide": false, "interval": "", "legendFormat": "Avg Memory Request", "refId": "Memory Request" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Avg Memory Usage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:1404", "format": "MiB", "label": "memory", "logBase": 1, "max": null, "min": "0", "show": true }, { "$$hashKey": "object:1405", "format": "short", "label": "", "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "Avg GPU Capacity": "semi-dark-orange", "Avg GPU Usage": "semi-dark-green", "Total GPU Capacity": "semi-dark-orange", "Total GPU Usage": "semi-dark-green", "Total GPU Utilization": "light-green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "description": "Avg GPU core usage across all workers of the API", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 46 }, "hiddenSeries": false, "id": 25, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(DCGM_FI_DEV_GPU_UTIL{exported_pod=~\"$api_name.+\"}) / 100\n/\ncount(DCGM_FI_DEV_GPU_UTIL{exported_pod=~\"$api_name.+\"})", "hide": false, "instant": false, "interval": "", "legendFormat": "Avg GPU Usage", "refId": "GPU Usage" }, { "expr": "count(DCGM_FI_DEV_GPU_UTIL{exported_pod=~\"$api_name.+\"})\n/\ncount(count(DCGM_FI_DEV_GPU_UTIL{exported_pod=~\"$api_name.+\"}) by (exported_pod))", "hide": false, "instant": false, "interval": "", "legendFormat": "Avg GPU Capacity", "refId": "GPU Capacity" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Avg GPU Core Usage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:1404", "format": "gpuCore", "label": "gpu", "logBase": 1, "max": null, "min": "0", "show": true }, { "$$hashKey": "object:1405", "format": "short", "label": "", "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "Avg Capacity GPU Memory": "semi-dark-orange", "Avg Used GPU Memory": "semi-dark-green", "Total Capacity GPU Memory": "semi-dark-orange", "Total Used GPU Memory": "semi-dark-green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "description": "Avg GPU memory usage across all workers of the API", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 46 }, "hiddenSeries": false, "id": 26, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(DCGM_FI_DEV_FB_USED{exported_pod=~\"$api_name.+\"})\n/\ncount(DCGM_FI_DEV_FB_USED{exported_pod=~\"$api_name.+\"})", "hide": false, "interval": "", "legendFormat": "Avg Used GPU Memory", "refId": "GPU Used Memory" }, { "exemplar": false, "expr": "(sum(DCGM_FI_DEV_FB_FREE{exported_pod=~\"$api_name.+\"}) + sum(DCGM_FI_DEV_FB_USED{exported_pod=~\"$api_name.+\"}))\n/\ncount(DCGM_FI_DEV_FB_USED{exported_pod=~\"$api_name.+\"})", "format": "time_series", "instant": false, "interval": "", "legendFormat": "Avg Capacity GPU Memory", "refId": "GPU Capacity Memory" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Avg GPU Memory Usage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:1404", "format": "MiB", "label": "memory", "logBase": 1, "max": null, "min": "0", "show": true }, { "$$hashKey": "object:1405", "format": "short", "label": "", "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } } ], "refresh": "30s", "schemaVersion": 30, "style": "dark", "tags": [], "templating": { "list": [ { "allValue": null, "current": { "selected": false, "text": "None", "value": "None" }, "datasource": null, "definition": "label_values({__name__=~\"cortex_task_.+\"}, api_name)", "description": null, "error": null, "hide": 0, "includeAll": false, "label": "API Name", "multi": true, "name": "api_name", "options": [], "query": { "query": "label_values({__name__=~\"cortex_task_.+\"}, api_name)", "refId": "StandardVariableQuery" }, "refresh": 2, "regex": "", "skipUrlSync": false, "sort": 0, "tagValuesQuery": "", "tagsQuery": "", "type": "query", "useTags": false } ] }, "time": { "from": "now-1h", "to": "now" }, "timepicker": {}, "timezone": "", "title": "TaskAPI", "uid": "taskapi", "version": 6 } ================================================ FILE: manager/manifests/grafana/grafana.yaml.j2 ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. apiVersion: v1 kind: ConfigMap metadata: name: grafana-datasources namespace: default data: datasources.yaml: | { "apiVersion": 1, "datasources": [ { "access": "proxy", "editable": false, "name": "prometheus", "orgId": 1, "type": "prometheus", "url": "http://prometheus.prometheus:9090", "version": 1, "isDefault": true } ] } --- apiVersion: v1 kind: ConfigMap metadata: name: grafana-dashboards namespace: default data: dashboards.yaml: |- { "apiVersion": 1, "providers": [ { "folder": "Cortex", "name": "Cortex", "options": { "path": "/grafana-dashboard-definitions/cortex" }, "disableDeletion": true, "orgId": 1, "type": "file" } ] } --- apiVersion: v1 kind: PersistentVolumeClaim metadata: name: grafana-storage namespace: default spec: storageClassName: ssd accessModes: - ReadWriteOnce resources: requests: storage: 2Gi --- apiVersion: apps/v1 kind: StatefulSet metadata: labels: app: grafana name: grafana namespace: default spec: serviceName: grafana replicas: 1 selector: matchLabels: app: grafana template: metadata: labels: app: grafana spec: containers: - image: {{ config['image_grafana'] }} name: grafana ports: - containerPort: 3000 name: http readinessProbe: httpGet: path: /api/health port: http resources: limits: cpu: 200m memory: 200Mi requests: cpu: 100m memory: 100Mi env: - name: GF_SERVER_ROOT_URL value: "%(protocol)s://%(domain)s:%(http_port)s/dashboard" - name: GF_SERVER_SERVE_FROM_SUB_PATH value: "true" - name: GF_USERS_DEFAULT_THEME value: "light" volumeMounts: - mountPath: /var/lib/grafana name: grafana-storage readOnly: false - mountPath: /etc/grafana/provisioning/datasources name: grafana-datasources readOnly: false - mountPath: /etc/grafana/provisioning/dashboards name: grafana-dashboards readOnly: false - mountPath: /grafana-dashboard-definitions/cortex/realtime name: grafana-dashboard-realtime readOnly: false - mountPath: /grafana-dashboard-definitions/cortex/async name: grafana-dashboard-async readOnly: false - mountPath: /grafana-dashboard-definitions/cortex/batch name: grafana-dashboard-batch readOnly: false - mountPath: /grafana-dashboard-definitions/cortex/task name: grafana-dashboard-task readOnly: false - mountPath: /grafana-dashboard-definitions/cortex/cluster name: grafana-dashboard-cluster readOnly: false - mountPath: /grafana-dashboard-definitions/cortex/nodes name: grafana-dashboard-nodes readOnly: false {% if env.get("CORTEX_DEV_ADD_CONTROL_PLANE_DASHBOARD") == "true" %} - mountPath: /grafana-dashboard-definitions/cortex/control-plane name: grafana-dashboard-control-plane readOnly: false {% endif %} securityContext: fsGroup: 65534 runAsNonRoot: true runAsUser: 65534 volumes: - name: grafana-storage persistentVolumeClaim: claimName: grafana-storage - name: grafana-datasources configMap: name: grafana-datasources - name: grafana-dashboards configMap: name: grafana-dashboards - name: grafana-dashboard-realtime configMap: name: grafana-dashboard-realtime - name: grafana-dashboard-async configMap: name: grafana-dashboard-async - name: grafana-dashboard-batch configMap: name: grafana-dashboard-batch - name: grafana-dashboard-task configMap: name: grafana-dashboard-task - name: grafana-dashboard-cluster configMap: name: grafana-dashboard-cluster - name: grafana-dashboard-nodes configMap: name: grafana-dashboard-nodes {% if env.get("CORTEX_DEV_ADD_CONTROL_PLANE_DASHBOARD") == "true" %} - name: grafana-dashboard-control-plane configMap: name: grafana-dashboard-control-plane {% endif %} nodeSelector: prometheus: "true" tolerations: - key: prometheus operator: Exists effect: NoSchedule affinity: podAffinity: preferredDuringSchedulingIgnoredDuringExecution: - podAffinityTerm: labelSelector: matchLabels: prometheus: prometheus topologyKey: kubernetes.io/hostname weight: 100 --- apiVersion: v1 kind: Service metadata: labels: app: grafana name: grafana namespace: default spec: type: ClusterIP ports: - name: http port: 3000 targetPort: http selector: app: grafana --- apiVersion: networking.istio.io/v1beta1 kind: VirtualService metadata: name: grafana namespace: default spec: hosts: - "*" gateways: - operator-gateway http: - name: grafana match: - uri: prefix: "/dashboard" - uri: prefix: "/grafana" rewrite: uri: "/dashboard" route: - destination: host: grafana port: number: 3000 ================================================ FILE: manager/manifests/inferentia.yaml ================================================ # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # # Permission is hereby granted, free of charge, to any person obtaining a copy of this # software and associated documentation files (the "Software"), to deal in the Software # without restriction, including without limitation the rights to use, copy, modify, # merge, publish, distribute, sublicense, and/or sell copies of the Software, and to # permit persons to whom the Software is furnished to do so. # # Modifications Copyright 2022 Cortex Labs, Inc. # Source: https://github.com/aws/aws-neuron-sdk/blob/master/src/k8/k8s-neuron-* apiVersion: v1 data: policy.cfg: | { "kind": "Policy", "apiVersion": "v1", "extenders": [ { "urlPrefix": "http://127.0.0.1:32700", "filterVerb": "filter", "bindVerb": "bind", "enableHttps": false, "nodeCacheCapable": true, "managedResources": [ { "name": "aws.amazon.com/neuron", "ignoredByScheduler": false } ], "ignorable": false } ] } kind: ConfigMap metadata: name: scheduler-policy namespace: kube-system --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: k8s-neuron-scheduler rules: - apiGroups: - "" resources: - nodes verbs: - get - list - watch - apiGroups: - "" resources: - events verbs: - create - patch - apiGroups: - "" resources: - pods verbs: - update - patch - get - list - watch - apiGroups: - "" resources: - bindings - pods/binding verbs: - create --- apiVersion: v1 kind: ServiceAccount metadata: name: k8s-neuron-scheduler namespace: kube-system --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: k8s-neuron-scheduler namespace: kube-system roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: k8s-neuron-scheduler subjects: - kind: ServiceAccount name: k8s-neuron-scheduler namespace: kube-system --- kind: Deployment apiVersion: apps/v1 metadata: name: k8s-neuron-scheduler namespace: kube-system spec: replicas: 1 strategy: type: Recreate selector: matchLabels: app: neuron-scheduler component: k8s-neuron-scheduler template: metadata: labels: app: neuron-scheduler component: k8s-neuron-scheduler spec: hostNetwork: true tolerations: - effect: NoSchedule operator: Exists key: node-role.kubernetes.io/master - effect: NoSchedule operator: Exists key: node.cloudprovider.kubernetes.io/uninitialized serviceAccount: k8s-neuron-scheduler priorityClassName: "system-node-critical" containers: - name: neuron-scheduler image: $CORTEX_IMAGE_NEURON_SCHEDULER env: - name: PORT value: "12345" resources: requests: cpu: 50m memory: 100Mi --- apiVersion: v1 kind: Service metadata: name: k8s-neuron-scheduler namespace: kube-system labels: app: neuron-scheduler component: k8s-neuron-scheduler spec: type: NodePort ports: - port: 12345 name: http targetPort: 12345 nodePort: 32700 selector: # select app=ingress-nginx pods app: neuron-scheduler component: k8s-neuron-scheduler --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: neuron-device-plugin rules: - apiGroups: - "" resources: - nodes verbs: - get - list - watch - apiGroups: - "" resources: - events verbs: - create - patch - apiGroups: - "" resources: - pods verbs: - update - patch - get - list - watch - apiGroups: - "" resources: - nodes/status verbs: - patch - update --- apiVersion: v1 kind: ServiceAccount metadata: name: neuron-device-plugin namespace: kube-system --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: neuron-device-plugin namespace: kube-system roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: neuron-device-plugin subjects: - kind: ServiceAccount name: neuron-device-plugin namespace: kube-system --- apiVersion: apps/v1 kind: DaemonSet metadata: name: neuron-device-plugin-daemonset namespace: kube-system spec: selector: matchLabels: name: neuron-device-plugin-ds updateStrategy: type: RollingUpdate template: metadata: labels: name: neuron-device-plugin-ds spec: serviceAccount: neuron-device-plugin tolerations: - key: CriticalAddonsOnly operator: Exists - key: aws.amazon.com/neuron operator: Exists effect: NoSchedule - key: workload operator: Exists effect: NoSchedule # Mark this pod as a critical add-on; when enabled, the critical add-on # scheduler reserves resources for critical add-on pods so that they can # be rescheduled after a failure. # See https://kubernetes.io/docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods/ priorityClassName: "system-node-critical" affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "node.kubernetes.io/instance-type" operator: In values: - inf1.xlarge - inf1.2xlarge - inf1.6xlarge - inf1.24xlarge containers: #Device Plugin containers are available both in us-east and us-west ecr #repos - image: $CORTEX_IMAGE_NEURON_DEVICE_PLUGIN imagePullPolicy: Always name: neuron-device-plugin env: - name: KUBECONFIG value: /etc/kubernetes/kubelet.conf - name: NODE_NAME valueFrom: fieldRef: fieldPath: spec.nodeName securityContext: allowPrivilegeEscalation: false capabilities: drop: ["ALL"] volumeMounts: - name: device-plugin mountPath: /var/lib/kubelet/device-plugins - name: infa-map mountPath: /run resources: requests: cpu: 100m memory: 100Mi nodeSelector: workload: "true" aws.amazon.com/neuron: "true" volumes: - name: device-plugin hostPath: path: /var/lib/kubelet/device-plugins - name: infa-map hostPath: path: /run ================================================ FILE: manager/manifests/istio.yaml.j2 ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. apiVersion: install.istio.io/v1alpha1 kind: IstioOperator spec: profile: minimal hub: {{ env['CORTEX_IMAGE_ISTIO_PROXY_HUB'] }} # this is only used by proxy, since pilot overrides it (proxy doesn't have dedicated hub config) tag: {{ env['CORTEX_IMAGE_ISTIO_PROXY_TAG'] }} # this is only used by proxy, since pilot overrides it (proxy doesn't have dedicated tag config) meshConfig: discoverySelectors: - matchLabels: istio-discovery: enabled components: pilot: # "pilot" refers to the istiod container hub: {{ env['CORTEX_IMAGE_ISTIO_PILOT_HUB'] }} tag: {{ env['CORTEX_IMAGE_ISTIO_PILOT_TAG'] }} k8s: resources: requests: cpu: 100m # default is 500m memory: 700Mi # default is 2048Mi == 2Gi hpaSpec: minReplicas: 1 maxReplicas: 5 metrics: - type: Resource resource: name: cpu targetAverageUtilization: 90 - type: Resource resource: name: memory targetAverageUtilization: 90 scaleTargetRef: apiVersion: apps/v1 kind: Deployment name: istiod cni: enabled: false ingressGateways: - name: istio-ingressgateway enabled: false - name: ingressgateway-operator enabled: true namespace: istio-system label: app: operator-istio-gateway istio: ingressgateway-operator k8s: serviceAnnotations: service.beta.kubernetes.io/aws-load-balancer-type: "nlb" service.beta.kubernetes.io/aws-load-balancer-cross-zone-load-balancing-enabled: "true" service.beta.kubernetes.io/aws-load-balancer-backend-protocol: "tcp" service.beta.kubernetes.io/aws-load-balancer-additional-resource-tags: "{{ env['CORTEX_OPERATOR_LOAD_BALANCER_TAGS'] }}" {% if config.get('operator_load_balancer_scheme') == 'internal' %} service.beta.kubernetes.io/aws-load-balancer-internal: "true" {% endif %} service: type: LoadBalancer externalTrafficPolicy: Cluster # https://medium.com/pablo-perez/k8s-externaltrafficpolicy-local-or-cluster-40b259a19404, https://www.asykim.com/blog/deep-dive-into-kubernetes-external-traffic-policies loadBalancerSourceRanges: {{ config.get('operator_load_balancer_cidr_white_list', ['0.0.0.0/0']) }} selector: app: operator-istio-gateway istio: ingressgateway-operator ports: - name: http2 port: 80 targetPort: 80 - name: https port: 443 targetPort: 443 resources: requests: cpu: 100m memory: 128Mi limits: cpu: 1000m memory: 1024Mi replicaCount: 1 hpaSpec: minReplicas: 1 maxReplicas: 1 metrics: - type: Resource resource: name: cpu targetAverageUtilization: 80 scaleTargetRef: apiVersion: apps/v1 kind: Deployment name: ingressgateway-operator - name: ingressgateway-apis enabled: true namespace: istio-system label: app: apis-istio-gateway istio: ingressgateway-apis k8s: serviceAnnotations: service.beta.kubernetes.io/aws-load-balancer-type: "{{ env['CORTEX_API_LOAD_BALANCER_TYPE'] }}" service.beta.kubernetes.io/aws-load-balancer-cross-zone-load-balancing-enabled: "true" service.beta.kubernetes.io/aws-load-balancer-additional-resource-tags: "{{ env['CORTEX_API_LOAD_BALANCER_TAGS'] }}" service.beta.kubernetes.io/aws-load-balancer-backend-protocol: "tcp" service.beta.kubernetes.io/aws-load-balancer-ssl-ports: "https" # "https" is the name of the https port below {% if config.get('api_load_balancer_scheme') == 'internal' %} service.beta.kubernetes.io/aws-load-balancer-internal: "true" {% endif %} {% if config.get('ssl_certificate_arn', '') != '' %} service.beta.kubernetes.io/aws-load-balancer-ssl-cert: "{{ config['ssl_certificate_arn'] }}" {% endif %} service: type: LoadBalancer loadBalancerSourceRanges: {{ config.get('api_load_balancer_cidr_white_list', ['0.0.0.0/0']) }} externalTrafficPolicy: Cluster # https://medium.com/pablo-perez/k8s-externaltrafficpolicy-local-or-cluster-40b259a19404, https://www.asykim.com/blog/deep-dive-into-kubernetes-external-traffic-policies selector: app: apis-istio-gateway istio: ingressgateway-apis ports: - name: http2 port: 80 targetPort: 80 - name: https port: 443 targetPort: 443 resources: requests: cpu: 400m memory: 512Mi limits: cpu: 1500m memory: 1024Mi replicaCount: 1 hpaSpec: minReplicas: 1 maxReplicas: 100 metrics: - type: Resource resource: name: cpu targetAverageUtilization: 90 - type: Resource resource: name: mem targetAverageUtilization: 90 scaleTargetRef: apiVersion: apps/v1 kind: Deployment name: ingressgateway-apis values: global: defaultResources: requests: cpu: 10m proxy: autoInject: disabled image: {{ env['CORTEX_IMAGE_ISTIO_PROXY_IMAGE'] }} pilot: image: {{ env['CORTEX_IMAGE_ISTIO_PILOT_IMAGE'] }} gateways: istio-ingressgateway: runAsRoot: true autoscaleEnabled: true secretVolumes: - name: customgateway-certs secretName: istio-customgateway-certs mountPath: /etc/istio/customgateway-certs - name: customgateway-ca-certs secretName: istio-customgateway-ca-certs mountPath: /etc/istio/customgateway-ca-certs ================================================ FILE: manager/manifests/kube-proxy.patch.yaml ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # This is a patch that needs to be applied onto the daemonset that's added by eksctl. apiVersion: apps/v1 kind: DaemonSet metadata: name: kube-proxy namespace: kube-system spec: selector: matchLabels: k8s-app: kube-proxy template: spec: containers: - name: kube-proxy command: - kube-proxy - --v=2 - --proxy-mode=ipvs - --ipvs-scheduler=rr - --config=/var/lib/kube-proxy/config env: - name: KUBE_PROXY_MODE value: ipvs updateStrategy: rollingUpdate: maxUnavailable: 20% type: RollingUpdate ================================================ FILE: manager/manifests/metrics-server.yaml ================================================ # Copyright 2021 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Modifications Copyright 2022 Cortex Labs, Inc. # Source: https://github.com/kubernetes-sigs/metrics-server/releases/download/v0.6.1/components.yaml apiVersion: v1 kind: ServiceAccount metadata: labels: k8s-app: metrics-server name: metrics-server namespace: kube-system --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: k8s-app: metrics-server rbac.authorization.k8s.io/aggregate-to-admin: "true" rbac.authorization.k8s.io/aggregate-to-edit: "true" rbac.authorization.k8s.io/aggregate-to-view: "true" name: system:aggregated-metrics-reader rules: - apiGroups: - metrics.k8s.io resources: - pods - nodes verbs: - get - list - watch --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: k8s-app: metrics-server name: system:metrics-server rules: - apiGroups: - "" resources: - nodes/metrics verbs: - get - apiGroups: - "" resources: - pods - nodes verbs: - get - list - watch --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: labels: k8s-app: metrics-server name: metrics-server-auth-reader namespace: kube-system roleRef: apiGroup: rbac.authorization.k8s.io kind: Role name: extension-apiserver-authentication-reader subjects: - kind: ServiceAccount name: metrics-server namespace: kube-system --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: labels: k8s-app: metrics-server name: metrics-server:system:auth-delegator roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: system:auth-delegator subjects: - kind: ServiceAccount name: metrics-server namespace: kube-system --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: labels: k8s-app: metrics-server name: system:metrics-server roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: system:metrics-server subjects: - kind: ServiceAccount name: metrics-server namespace: kube-system --- apiVersion: v1 kind: Service metadata: labels: k8s-app: metrics-server name: metrics-server namespace: kube-system spec: ports: - name: https port: 443 protocol: TCP targetPort: https selector: k8s-app: metrics-server --- apiVersion: apps/v1 kind: Deployment metadata: labels: k8s-app: metrics-server name: metrics-server namespace: kube-system spec: selector: matchLabels: k8s-app: metrics-server strategy: rollingUpdate: maxUnavailable: 0 template: metadata: labels: k8s-app: metrics-server spec: containers: - args: - --cert-dir=/tmp - --secure-port=4443 - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname - --kubelet-use-node-status-port - --metric-resolution=15s image: $CORTEX_IMAGE_METRICS_SERVER imagePullPolicy: Always livenessProbe: failureThreshold: 3 httpGet: path: /livez port: https scheme: HTTPS periodSeconds: 10 name: metrics-server ports: - containerPort: 4443 name: https protocol: TCP readinessProbe: failureThreshold: 3 httpGet: path: /readyz port: https scheme: HTTPS initialDelaySeconds: 20 periodSeconds: 10 resources: requests: cpu: 50m memory: 100Mi limits: cpu: 200m memory: 500Mi securityContext: allowPrivilegeEscalation: false readOnlyRootFilesystem: true runAsNonRoot: true runAsUser: 1000 volumeMounts: - mountPath: /tmp name: tmp-dir nodeSelector: kubernetes.io/os: linux priorityClassName: system-cluster-critical serviceAccountName: metrics-server volumes: - emptyDir: {} name: tmp-dir --- apiVersion: apiregistration.k8s.io/v1 kind: APIService metadata: labels: k8s-app: metrics-server name: v1beta1.metrics.k8s.io spec: group: metrics.k8s.io groupPriorityMinimum: 100 insecureSkipTLSVerify: true service: name: metrics-server namespace: kube-system version: v1beta1 versionPriority: 100 ================================================ FILE: manager/manifests/namespaces.yaml ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. apiVersion: v1 kind: Namespace metadata: name: istio-system --- apiVersion: v1 kind: Namespace metadata: name: logging --- apiVersion: v1 kind: Namespace metadata: name: prometheus --- ================================================ FILE: manager/manifests/nvidia.yaml ================================================ # Copyright (c) 2019-2021, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Modifications Copyright 2022 Cortex Labs, Inc. # Source: https://github.com/NVIDIA/k8s-device-plugin/blob/v0.11.0/nvidia-device-plugin.yml apiVersion: apps/v1 kind: DaemonSet metadata: name: nvidia-device-plugin-daemonset namespace: kube-system spec: selector: matchLabels: name: nvidia-device-plugin-ds updateStrategy: type: RollingUpdate rollingUpdate: maxUnavailable: 1 template: metadata: labels: name: nvidia-device-plugin-ds spec: tolerations: # This toleration is deprecated. Kept here for backward compatibility # See https://kubernetes.io/docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods/ - key: CriticalAddonsOnly operator: Exists - key: nvidia.com/gpu operator: Exists effect: NoSchedule - key: workload operator: Exists effect: NoSchedule # Mark this pod as a critical add-on; when enabled, the critical add-on # scheduler reserves resources for critical add-on pods so that they can # be rescheduled after a failure. # See https://kubernetes.io/docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods/ priorityClassName: "system-node-critical" containers: - image: $CORTEX_IMAGE_NVIDIA_DEVICE_PLUGIN name: nvidia-device-plugin-ctr args: ["--fail-on-init-error=false"] securityContext: allowPrivilegeEscalation: false capabilities: drop: ["ALL"] volumeMounts: - name: device-plugin mountPath: /var/lib/kubelet/device-plugins resources: # https://github.com/kubernetes/kubernetes/blob/master/cluster/addons/device-plugins/nvidia-gpu/daemonset.yaml#L44 requests: cpu: 100m memory: 100Mi limits: memory: 100Mi nodeSelector: workload: "true" nvidia.com/gpu: "true" volumes: - name: device-plugin hostPath: path: /var/lib/kubelet/device-plugins ================================================ FILE: manager/manifests/operator.yaml.j2 ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. apiVersion: v1 kind: ServiceAccount metadata: name: operator namespace: default --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: operator namespace: default subjects: - kind: ServiceAccount name: operator namespace: default roleRef: kind: ClusterRole name: cluster-admin apiGroup: rbac.authorization.k8s.io --- apiVersion: apps/v1 kind: Deployment metadata: name: operator namespace: default labels: workloadID: operator spec: replicas: 1 selector: matchLabels: workloadID: operator template: metadata: labels: workloadID: operator spec: serviceAccountName: operator containers: - name: operator image: {{ config['image_operator'] }} imagePullPolicy: Always resources: requests: cpu: 100m memory: 128Mi limits: cpu: 1500m memory: 2048Mi ports: - containerPort: 8888 envFrom: - configMapRef: name: env-vars volumeMounts: - name: cluster-config mountPath: /configs/cluster - name: docker-client mountPath: /var/run/docker.sock volumes: - name: cluster-config configMap: name: cluster-config - name: docker-client hostPath: path: /var/run/docker.sock type: Socket --- apiVersion: v1 kind: Service metadata: namespace: default name: operator labels: cortex.dev/name: operator spec: selector: workloadID: operator ports: - port: 8888 name: http --- apiVersion: networking.istio.io/v1beta1 kind: Gateway metadata: name: operator-gateway namespace: default spec: selector: istio: ingressgateway-operator servers: - port: number: 80 name: http protocol: HTTP hosts: - "*" - port: number: 443 name: https protocol: HTTPS hosts: - "*" tls: mode: SIMPLE serverCertificate: /etc/istio/customgateway-certs/tls.crt privateKey: /etc/istio/customgateway-certs/tls.key --- apiVersion: networking.istio.io/v1beta1 kind: VirtualService metadata: name: operator namespace: default spec: hosts: - "*" gateways: - operator-gateway http: - route: - destination: host: operator port: number: 8888 ================================================ FILE: manager/manifests/prometheus-additional-scrape-configs.yaml.j2 ================================================ # Copyright 2015 The Prometheus Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Modifications Copyright 2022 Cortex Labs, Inc. {% if env.get("CORTEX_DEV_ADD_CONTROL_PLANE_DASHBOARD") == "true" %} - job_name: "kubernetes-apiservers" kubernetes_sd_configs: - role: endpoints # Default to scraping over https. If required, just disable this or change to # `http`. scheme: https # This TLS & authorization config is used to connect to the actual scrape # endpoints for cluster components. This is separate to discovery auth # configuration because discovery & scraping are two separate concerns in # Prometheus. The discovery auth config is automatic if Prometheus runs inside # the cluster. Otherwise, more config options have to be provided within the # . tls_config: ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt # If your node certificates are self-signed or use a different CA to the # master CA, then disable certificate verification below. Note that # certificate verification is an integral part of a secure infrastructure # so this should only be disabled in a controlled environment. You can # disable certificate verification by uncommenting the line below. # # insecure_skip_verify: true authorization: credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token # Keep only the default/kubernetes service endpoints for the https port. This # will add targets for each API server which Kubernetes adds an endpoint to # the default/kubernetes service. relabel_configs: - source_labels: [ __meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name, ] action: keep regex: default;kubernetes;https {% endif %} ================================================ FILE: manager/manifests/prometheus-dcgm-exporter.yaml ================================================ # Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Modifications Copyright 2022 Cortex Labs, Inc. apiVersion: v1 kind: ServiceAccount metadata: name: dcgm-exporter namespace: prometheus labels: app.kubernetes.io/name: dcgm-exporter app.kubernetes.io/instance: dcgm-exporter app.kubernetes.io/component: dcgm-exporter --- apiVersion: apps/v1 kind: DaemonSet metadata: name: dcgm-exporter namespace: prometheus labels: app.kubernetes.io/name: dcgm-exporter app.kubernetes.io/instance: dcgm-exporter app.kubernetes.io/component: dcgm-exporter spec: updateStrategy: type: RollingUpdate selector: matchLabels: app.kubernetes.io/name: dcgm-exporter app.kubernetes.io/instance: dcgm-exporter app.kubernetes.io/component: dcgm-exporter template: metadata: labels: app.kubernetes.io/name: dcgm-exporter app.kubernetes.io/instance: dcgm-exporter app.kubernetes.io/component: dcgm-exporter spec: serviceAccountName: dcgm-exporter affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: nvidia.com/gpu operator: Exists containers: - env: - name: DCGM_EXPORTER_LISTEN value: :9400 - name: DCGM_EXPORTER_KUBERNETES value: "true" image: $CORTEX_IMAGE_PROMETHEUS_DCGM_EXPORTER imagePullPolicy: Always name: dcgm-exporter ports: - containerPort: 9400 name: metrics protocol: TCP resources: requests: cpu: 50m memory: 50Mi securityContext: capabilities: add: - SYS_ADMIN runAsNonRoot: false runAsUser: 0 volumeMounts: - mountPath: /var/lib/kubelet/pod-resources name: pod-gpu-resources readOnly: true tolerations: - key: workload effect: NoSchedule operator: Exists - key: nvidia.com/gpu operator: Exists effect: NoSchedule volumes: - hostPath: path: /var/lib/kubelet/pod-resources type: "" name: pod-gpu-resources nodeSelector: workload: "true" nvidia.com/gpu: "$NVIDIA_COM_GPU_VALUE" --- apiVersion: monitoring.coreos.com/v1 kind: PodMonitor metadata: name: dcgm-exporter namespace: prometheus labels: monitoring.cortex.dev: dcgm-exporter app.kubernetes.io/name: dcgm-exporter app.kubernetes.io/instance: dcgm-exporter app.kubernetes.io/component: dcgm-exporter annotations: prometheus.io/scrape: 'true' prometheus.io/port: '9400' spec: jobLabel: "dcgm-exporter" podMetricsEndpoints: - port: metrics path: /metrics scheme: http interval: 15s metricRelabelings: - action: keep sourceLabels: [__name__] regex: "DCGM_FI_DEV_(\ GPU_UTIL|\ FB_USED|\ FB_FREE\ )" - action: labelkeep regex: (__name__|exported_pod) namespaceSelector: any: true selector: matchLabels: app.kubernetes.io/name: dcgm-exporter app.kubernetes.io/instance: dcgm-exporter ================================================ FILE: manager/manifests/prometheus-kube-state-metrics.yaml ================================================ # Copyright 2021 The Kubernetes Authors All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Modifications Copyright 2022 Cortex Labs, Inc. apiVersion: v1 kind: ServiceAccount metadata: labels: app.kubernetes.io/name: kube-state-metrics name: kube-state-metrics namespace: prometheus --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: app.kubernetes.io/name: kube-state-metrics name: kube-state-metrics rules: - apiGroups: ["certificates.k8s.io"] resources: - certificatesigningrequests verbs: ["list", "watch"] - apiGroups: [""] resources: - configmaps verbs: ["list", "watch"] - apiGroups: ["batch"] resources: - cronjobs verbs: ["list", "watch"] - apiGroups: ["extensions", "apps"] resources: - daemonsets verbs: ["list", "watch"] - apiGroups: ["extensions", "apps"] resources: - deployments verbs: ["list", "watch"] - apiGroups: [""] resources: - endpoints verbs: ["list", "watch"] - apiGroups: ["autoscaling"] resources: - horizontalpodautoscalers verbs: ["list", "watch"] - apiGroups: ["extensions", "networking.k8s.io"] resources: - ingresses verbs: ["list", "watch"] - apiGroups: ["batch"] resources: - jobs verbs: ["list", "watch"] - apiGroups: [""] resources: - limitranges verbs: ["list", "watch"] - apiGroups: ["admissionregistration.k8s.io"] resources: - mutatingwebhookconfigurations verbs: ["list", "watch"] - apiGroups: [""] resources: - namespaces verbs: ["list", "watch"] - apiGroups: ["networking.k8s.io"] resources: - networkpolicies verbs: ["list", "watch"] - apiGroups: [""] resources: - nodes verbs: ["list", "watch"] - apiGroups: [""] resources: - persistentvolumeclaims verbs: ["list", "watch"] - apiGroups: [""] resources: - persistentvolumes verbs: ["list", "watch"] - apiGroups: ["policy"] resources: - poddisruptionbudgets verbs: ["list", "watch"] - apiGroups: [""] resources: - pods verbs: ["list", "watch"] - apiGroups: ["extensions", "apps"] resources: - replicasets verbs: ["list", "watch"] - apiGroups: [""] resources: - replicationcontrollers verbs: ["list", "watch"] - apiGroups: [""] resources: - resourcequotas verbs: ["list", "watch"] - apiGroups: [""] resources: - secrets verbs: ["list", "watch"] - apiGroups: [""] resources: - services verbs: ["list", "watch"] - apiGroups: ["apps"] resources: - statefulsets verbs: ["list", "watch"] - apiGroups: ["storage.k8s.io"] resources: - storageclasses verbs: ["list", "watch"] - apiGroups: ["admissionregistration.k8s.io"] resources: - validatingwebhookconfigurations verbs: ["list", "watch"] - apiGroups: ["storage.k8s.io"] resources: - volumeattachments verbs: ["list", "watch"] - apiGroups: ["autoscaling.k8s.io"] resources: - verticalpodautoscalers verbs: ["list", "watch"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: labels: app.kubernetes.io/name: kube-state-metrics name: kube-state-metrics roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: kube-state-metrics subjects: - kind: ServiceAccount name: kube-state-metrics namespace: prometheus --- apiVersion: apps/v1 kind: Deployment metadata: name: kube-state-metrics namespace: prometheus labels: app.kubernetes.io/name: kube-state-metrics app.kubernetes.io/version: "2.1.0" spec: selector: matchLabels: app.kubernetes.io/name: kube-state-metrics replicas: 1 template: metadata: labels: app.kubernetes.io/name: kube-state-metrics spec: hostNetwork: false serviceAccountName: kube-state-metrics securityContext: fsGroup: 65534 runAsGroup: 65534 runAsUser: 65534 containers: - name: kube-state-metrics resources: requests: cpu: 300m memory: 400Mi args: - --port=8080 - --resources=certificatesigningrequests,configmaps,cronjobs,daemonsets,deployments,endpoints,horizontalpodautoscalers,ingresses,jobs,limitranges,mutatingwebhookconfigurations,namespaces,networkpolicies,nodes,persistentvolumeclaims,persistentvolumes,poddisruptionbudgets,pods,replicasets,replicationcontrollers,resourcequotas,secrets,services,statefulsets,storageclasses,validatingwebhookconfigurations,verticalpodautoscalers,volumeattachments - --telemetry-port=8081 imagePullPolicy: Always image: $CORTEX_IMAGE_PROMETHEUS_KUBE_STATE_METRICS ports: - containerPort: 8080 name: metrics protocol: TCP livenessProbe: httpGet: path: /healthz port: 8080 initialDelaySeconds: 5 timeoutSeconds: 5 readinessProbe: httpGet: path: / port: 8080 initialDelaySeconds: 5 timeoutSeconds: 5 nodeSelector: prometheus: "true" tolerations: - key: prometheus operator: Exists effect: NoSchedule --- apiVersion: monitoring.coreos.com/v1 kind: PodMonitor metadata: name: kube-state-metrics namespace: prometheus labels: name: kube-state-metrics monitoring.cortex.dev: kube-state-metrics spec: jobLabel: "kube-state-metrics" podMetricsEndpoints: - port: metrics scheme: http path: /metrics interval: 30s metricRelabelings: - action: keep sourceLabels: [__name__] regex: "kube_(\ pod_container_resource_requests|\ pod_info|\ deployment_status_replicas_available|\ job_status_active\ )" - action: labelkeep regex: (__name__|exported_pod|exported_container|job_name|resource|deployment) namespaceSelector: any: true selector: matchLabels: app.kubernetes.io/name: kube-state-metrics ================================================ FILE: manager/manifests/prometheus-kubelet-exporter.yaml ================================================ # Copyright 2016 The prometheus-operator Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Modifications Copyright 2022 Cortex Labs, Inc. apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor metadata: labels: k8s-app: kubelet monitoring.cortex.dev: kubelet-exporter name: kubelet namespace: prometheus spec: endpoints: - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token honorLabels: true interval: 30s metricRelabelings: - action: drop sourceLabels: [__name__] port: https-metrics relabelings: - sourceLabels: - __metrics_path__ targetLabel: metrics_path scheme: https tlsConfig: insecureSkipVerify: true - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token honorLabels: true honorTimestamps: false interval: 30s metricRelabelings: - action: keep sourceLabels: [__name__] regex: "container_(\ cpu_usage_seconds_total|\ memory_working_set_bytes\ )" - action: labelkeep regex: (__name__|pod|container|name) path: /metrics/cadvisor port: https-metrics relabelings: - sourceLabels: - __metrics_path__ targetLabel: metrics_path scheme: https tlsConfig: insecureSkipVerify: true - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token honorLabels: true interval: 30s path: /metrics/probes port: https-metrics relabelings: - sourceLabels: - __metrics_path__ targetLabel: metrics_path metricRelabelings: - action: drop sourceLabels: [__name__] scheme: https tlsConfig: insecureSkipVerify: true jobLabel: k8s-app namespaceSelector: matchNames: - kube-system selector: matchLabels: k8s-app: kubelet ================================================ FILE: manager/manifests/prometheus-monitoring.yaml ================================================ # Copyright 2016 The prometheus-operator Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Modifications Copyright 2022 Cortex Labs, Inc. apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: name: ssd volumeBindingMode: WaitForFirstConsumer provisioner: kubernetes.io/aws-ebs parameters: type: gp2 --- apiVersion: monitoring.coreos.com/v1 kind: Prometheus metadata: name: prometheus namespace: prometheus spec: image: $CORTEX_IMAGE_PROMETHEUS serviceAccountName: prometheus nodeSelector: prometheus: "true" tolerations: - key: prometheus operator: Exists effect: NoSchedule podMonitorSelector: matchExpressions: - key: "monitoring.cortex.dev" operator: "Exists" serviceMonitorSelector: matchExpressions: - key: "monitoring.cortex.dev" operator: "Exists" ruleSelector: matchLabels: prometheus: k8s resources: requests: memory: 400Mi enableAdminAPI: false storage: volumeClaimTemplate: spec: storageClassName: ssd resources: requests: storage: 40Gi additionalScrapeConfigs: name: additional-scrape-configs key: prometheus-additional-scrape-configs.yaml retention: 2w retentionSize: 35GB securityContext: fsGroup: 2000 runAsNonRoot: true runAsUser: 1000 --- apiVersion: v1 kind: ServiceAccount metadata: name: prometheus namespace: prometheus --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: prometheus rules: - apiGroups: [ "" ] resources: - nodes - nodes/metrics - services - endpoints - pods verbs: [ "get", "list", "watch" ] - apiGroups: [ "" ] resources: - configmaps verbs: [ "get" ] - apiGroups: - networking.k8s.io resources: - ingresses verbs: [ "get", "list", "watch" ] - nonResourceURLs: [ "/metrics" ] verbs: [ "get" ] --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: prometheus roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: prometheus subjects: - kind: ServiceAccount name: prometheus namespace: prometheus --- apiVersion: v1 kind: Service metadata: name: prometheus namespace: prometheus spec: type: ClusterIP ports: - port: 9090 targetPort: 9090 selector: prometheus: prometheus --- apiVersion: monitoring.coreos.com/v1 kind: PodMonitor metadata: name: istio-ingress-stats namespace: prometheus labels: monitoring.cortex.dev: "istio" spec: selector: matchExpressions: - { key: prometheus-ignore, operator: DoesNotExist } - { key: istio, operator: Exists } - { key: release, operator: In, values: [ "istio" ]} namespaceSelector: any: true jobLabel: istio-ingress-stats podMetricsEndpoints: - path: /stats/prometheus interval: 15s relabelings: - action: keep sourceLabels: [ __meta_kubernetes_pod_container_name ] regex: "istio-proxy" - action: keep sourceLabels: [ __meta_kubernetes_pod_annotationpresent_prometheus_io_scrape ] - sourceLabels: [ __address__, __meta_kubernetes_pod_annotation_prometheus_io_port ] action: replace regex: ([^:]+)(?::\d+)?;(\d+) replacement: $1:$2 targetLabel: __address__ - action: labeldrop regex: "__meta_kubernetes_pod_label_(.+)" - sourceLabels: [ __meta_kubernetes_namespace ] action: replace targetLabel: namespace - sourceLabels: [ __meta_kubernetes_pod_name ] action: replace targetLabel: pod_name - sourceLabels: [ __name__, __meta_kubernetes_pod_label_istio, __meta_kubernetes_pod_name ] action: replace regex: (istio_requests_total)?;(ingressgateway-apis)?;(.+) replacement: $3 targetLabel: istioingress_podname metricRelabelings: - action: keep sourceLabels: [__name__] regex: "istio_(\ requests_total|\ request_duration_milliseconds_bucket|\ request_duration_milliseconds_sum|\ request_duration_milliseconds_count\ )" - action: labelkeep regex: (__name__|destination_service|response_code|le|istioingress_podname) --- apiVersion: monitoring.coreos.com/v1 kind: PodMonitor metadata: name: proxy-stats namespace: prometheus labels: monitoring.cortex.dev: "proxy" spec: selector: matchLabels: apiKind: RealtimeAPI matchExpressions: - { key: prometheus-ignore, operator: DoesNotExist } namespaceSelector: any: true jobLabel: proxy-stats podMetricsEndpoints: - path: /metrics scheme: http interval: 10s port: admin relabelings: - action: keep sourceLabels: [ __meta_kubernetes_pod_container_name ] regex: "proxy" - sourceLabels: [ __meta_kubernetes_pod_label_apiName ] action: replace targetLabel: api_name - sourceLabels: [ __meta_kubernetes_pod_label_apiKind ] action: replace targetLabel: api_kind - sourceLabels: [ __meta_kubernetes_pod_label_apiID ] action: replace targetLabel: api_id - sourceLabels: [ __address__, __meta_kubernetes_pod_annotation_prometheus_io_port ] action: replace regex: ([^:]+)(?::\d+)?;(\d+) replacement: $1:$2 targetLabel: __address__ - action: labeldrop regex: "__meta_kubernetes_pod_label_(.+)" - sourceLabels: [ __meta_kubernetes_namespace ] action: replace targetLabel: namespace - sourceLabels: [ __meta_kubernetes_pod_name ] action: replace targetLabel: pod_name metricRelabelings: - action: keep sourceLabels: [__name__] regex: "cortex_(.+)" --- apiVersion: monitoring.coreos.com/v1 kind: PodMonitor metadata: name: async-stats namespace: prometheus labels: monitoring.cortex.dev: "dequeuer-async" spec: selector: matchLabels: apiKind: AsyncAPI matchExpressions: - { key: prometheus-ignore, operator: DoesNotExist } namespaceSelector: any: true jobLabel: async-stats podMetricsEndpoints: - path: /metrics scheme: http interval: 10s port: admin relabelings: - action: keep sourceLabels: [ __meta_kubernetes_pod_container_name ] regex: "dequeuer" - sourceLabels: [ __meta_kubernetes_pod_label_apiName ] action: replace targetLabel: api_name - sourceLabels: [ __meta_kubernetes_pod_label_apiKind ] action: replace targetLabel: api_kind - sourceLabels: [ __meta_kubernetes_pod_label_apiID ] action: replace targetLabel: api_id - sourceLabels: [ __address__, __meta_kubernetes_pod_annotation_prometheus_io_port ] action: replace regex: ([^:]+)(?::\d+)?;(\d+) replacement: $1:$2 targetLabel: __address__ - action: labeldrop regex: "__meta_kubernetes_pod_label_(.+)" - sourceLabels: [ __meta_kubernetes_namespace ] action: replace targetLabel: namespace - sourceLabels: [ __meta_kubernetes_pod_name ] action: replace targetLabel: pod_name metricRelabelings: - action: keep sourceLabels: [__name__] regex: "cortex_(.+)" --- apiVersion: monitoring.coreos.com/v1 kind: PodMonitor metadata: name: prometheus-statsd-exporter namespace: prometheus labels: name: prometheus-statsd-exporter monitoring.cortex.dev: "statsd-exporter" spec: jobLabel: "statsd-exporter" podMetricsEndpoints: - port: metrics scheme: http path: /metrics interval: 20s metricRelabelings: - action: keep sourceLabels: [__name__] regex: "cortex_(.+)" namespaceSelector: any: true selector: matchLabels: name: prometheus-statsd-exporter --- apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor metadata: name: operator namespace: prometheus labels: name: operator monitoring.cortex.dev: "operator" spec: jobLabel: "operator" endpoints: - port: http scheme: http path: /metrics interval: 10s metricRelabelings: - action: keep sourceLabels: [__name__] regex: "cortex_(.+)" - action: labeldrop regex: (container|endpoint|instance|job|namespace|pod|service) namespaceSelector: any: true selector: matchLabels: cortex.dev/name: operator --- apiVersion: monitoring.coreos.com/v1 kind: PodMonitor metadata: name: activator-stats labels: monitoring.cortex.dev: "activator" spec: selector: matchLabels: app: activator matchExpressions: - { key: prometheus-ignore, operator: DoesNotExist } namespaceSelector: any: true jobLabel: activator-stats podMetricsEndpoints: - path: /metrics scheme: http interval: 10s port: admin relabelings: - action: keep sourceLabels: [ __meta_kubernetes_pod_container_name ] regex: "activator" - sourceLabels: [ __address__, __meta_kubernetes_pod_annotation_prometheus_io_port ] action: replace regex: ([^:]+)(?::\d+)?;(\d+) replacement: $1:$2 targetLabel: __address__ - action: labeldrop regex: "__meta_kubernetes_pod_label_(.+)" - sourceLabels: [ __meta_kubernetes_namespace ] action: replace targetLabel: namespace - sourceLabels: [ __meta_kubernetes_pod_name ] action: replace targetLabel: pod_name metricRelabelings: - action: keep sourceLabels: [__name__] regex: "cortex_(.+)" ================================================ FILE: manager/manifests/prometheus-node-exporter.yaml ================================================ # Copyright 2016 The prometheus-operator Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Modifications Copyright 2022 Cortex Labs, Inc. apiVersion: v1 kind: ServiceAccount metadata: labels: app.kubernetes.io/version: v1.3.1 name: node-exporter namespace: prometheus --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: app.kubernetes.io/version: v1.3.1 name: node-exporter rules: - apiGroups: - authentication.k8s.io resources: - tokenreviews verbs: - create - apiGroups: - authorization.k8s.io resources: - subjectaccessreviews verbs: - create --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: labels: app.kubernetes.io/version: v1.3.1 name: node-exporter roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: node-exporter subjects: - kind: ServiceAccount name: node-exporter namespace: prometheus --- apiVersion: v1 kind: Service metadata: labels: app.kubernetes.io/name: node-exporter app.kubernetes.io/version: v1.3.1 name: node-exporter namespace: prometheus spec: clusterIP: None ports: - name: https port: 9100 targetPort: https selector: app.kubernetes.io/name: node-exporter --- apiVersion: apps/v1 kind: DaemonSet metadata: labels: app.kubernetes.io/name: node-exporter app.kubernetes.io/version: v1.3.1 name: node-exporter namespace: prometheus spec: selector: matchLabels: app.kubernetes.io/name: node-exporter template: metadata: labels: app.kubernetes.io/name: node-exporter app.kubernetes.io/version: v1.3.1 spec: containers: - args: - --web.listen-address=127.0.0.1:9100 - --path.sysfs=/host/sys - --path.rootfs=/host/root - --no-collector.wifi - --no-collector.hwmon - --collector.filesystem.ignored-mount-points=^/(dev|proc|sys|var/lib/docker/.+|var/lib/kubelet/pods/.+)($|/) - --collector.netclass.ignored-devices=^(veth.*)$ - --collector.netdev.device-exclude=^(veth.*)$ image: $CORTEX_IMAGE_PROMETHEUS_NODE_EXPORTER name: node-exporter resources: limits: cpu: 250m memory: 180Mi requests: cpu: 40m memory: 180Mi volumeMounts: - mountPath: /host/sys mountPropagation: HostToContainer name: sys readOnly: true - mountPath: /host/root mountPropagation: HostToContainer name: root readOnly: true - args: - --logtostderr - --secure-listen-address=[$(IP)]:9100 - --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - --upstream=http://127.0.0.1:9100/ env: - name: IP valueFrom: fieldRef: fieldPath: status.podIP image: $CORTEX_IMAGE_KUBE_RBAC_PROXY name: kube-rbac-proxy ports: - containerPort: 9100 hostPort: 9100 name: https resources: limits: cpu: 20m memory: 40Mi requests: cpu: 10m memory: 20Mi hostNetwork: true hostPID: true nodeSelector: kubernetes.io/os: linux securityContext: runAsNonRoot: true runAsUser: 65534 serviceAccountName: node-exporter tolerations: - operator: Exists volumes: - hostPath: path: /sys name: sys - hostPath: path: / name: root updateStrategy: rollingUpdate: maxUnavailable: 10% type: RollingUpdate --- apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor metadata: labels: app.kubernetes.io/name: node-exporter app.kubernetes.io/version: v1.3.1 monitoring.cortex.dev: node-exporter name: node-exporter namespace: prometheus spec: endpoints: - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token interval: 15s port: https relabelings: - action: replace regex: (.*) replacement: $1 sourceLabels: - __meta_kubernetes_pod_node_name targetLabel: instance metricRelabelings: - action: keep sourceLabels: [__name__] regex: "node_(\ cpu_seconds_total|\ load1|\ load5|\ load15|\ exporter_build_info|\ memory_MemTotal_bytes|\ memory_MemFree_bytes|\ memory_Buffers_bytes|\ memory_Cached_bytes|\ memory_MemAvailable_bytes|\ disk_read_bytes_total|\ disk_written_bytes_total|\ disk_io_time_seconds_total|\ disk_io_time_weighted_seconds_total|\ filesystem_size_bytes|\ filesystem_avail_bytes|\ network_receive_bytes_total|\ network_transmit_bytes_total|\ network_receive_drop_total|\ network_transmit_drop_total|\ vmstat_pgmajfault\ )" - action: labelkeep regex: (__name__|instance|job|device|fstype|mountpoint|mode) scheme: https tlsConfig: insecureSkipVerify: true jobLabel: app.kubernetes.io/name selector: matchLabels: app.kubernetes.io/name: node-exporter --- apiVersion: monitoring.coreos.com/v1 kind: PrometheusRule metadata: labels: app.kubernetes.io/name: node-exporter app.kubernetes.io/version: 1.1.2 prometheus: k8s name: node-exporter-rules namespace: prometheus spec: groups: - name: node-exporter.rules rules: - expr: | count without (cpu) ( count without (mode) ( node_cpu_seconds_total{job="node-exporter"} ) ) record: instance:node_num_cpu:sum - expr: | 1 - avg without (cpu, mode) ( rate(node_cpu_seconds_total{job="node-exporter", mode="idle"}[1m]) ) record: instance:node_cpu_utilisation:rate1m - expr: | ( node_load1{job="node-exporter"} / instance:node_num_cpu:sum{job="node-exporter"} ) record: instance:node_load1_per_cpu:ratio - expr: | 1 - ( node_memory_MemAvailable_bytes{job="node-exporter"} / node_memory_MemTotal_bytes{job="node-exporter"} ) record: instance:node_memory_utilisation:ratio - expr: | rate(node_vmstat_pgmajfault{job="node-exporter"}[1m]) record: instance:node_vmstat_pgmajfault:rate1m - expr: | rate(node_disk_io_time_seconds_total{job="node-exporter", device=~"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+"}[1m]) record: instance_device:node_disk_io_time_seconds:rate1m - expr: | rate(node_disk_io_time_weighted_seconds_total{job="node-exporter", device=~"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+"}[1m]) record: instance_device:node_disk_io_time_weighted_seconds:rate1m - expr: | sum without (device) ( rate(node_network_receive_bytes_total{job="node-exporter", device!="lo"}[1m]) ) record: instance:node_network_receive_bytes_excluding_lo:rate1m - expr: | sum without (device) ( rate(node_network_transmit_bytes_total{job="node-exporter", device!="lo"}[1m]) ) record: instance:node_network_transmit_bytes_excluding_lo:rate1m - expr: | sum without (device) ( rate(node_network_receive_drop_total{job="node-exporter", device!="lo"}[1m]) ) record: instance:node_network_receive_drop_excluding_lo:rate1m - expr: | sum without (device) ( rate(node_network_transmit_drop_total{job="node-exporter", device!="lo"}[1m]) ) record: instance:node_network_transmit_drop_excluding_lo:rate1m - expr: | sum ( ( instance:node_cpu_utilisation:rate1m{job="node-exporter"} * instance:node_num_cpu:sum{job="node-exporter"} ) / scalar(sum(instance:node_num_cpu:sum{job="node-exporter"})) ) record: cluster:cpu_utilization:ratio - expr: | sum ( instance:node_load1_per_cpu:ratio{job="node-exporter"} / scalar(count(instance:node_load1_per_cpu:ratio{job="node-exporter"})) ) record: cluster:load1:ratio - expr: | sum ( instance:node_memory_utilisation:ratio{job="node-exporter"} / scalar(count(instance:node_memory_utilisation:ratio{job="node-exporter"})) ) record: cluster:memory_utilization:ratio - expr: | sum ( instance:node_vmstat_pgmajfault:rate1m{job="node-exporter"} ) record: cluster:vmstat_pgmajfault:rate1m - expr: | sum (instance:node_network_receive_bytes_excluding_lo:rate1m{job="node-exporter"}) record: cluster:network_receive_bytes_excluding_low:rate1m - expr: | sum (instance:node_network_transmit_bytes_excluding_lo:rate1m{job="node-exporter"}) record: cluster:network_transmit_bytes_excluding_lo:rate1m - expr: | sum (instance:node_network_receive_drop_excluding_lo:rate1m{job="node-exporter"}) record: cluster:network_receive_drop_excluding_lo:rate1m - expr: | sum (instance:node_network_transmit_drop_excluding_lo:rate1m{job="node-exporter"}) record: cluster:network_transmit_drop_excluding_lo:rate1m - expr: | sum ( instance_device:node_disk_io_time_seconds:rate1m{job="node-exporter"} / scalar(count(instance_device:node_disk_io_time_seconds:rate1m{job="node-exporter"})) ) record: cluster:disk_io_utilization:ratio - expr: | sum ( instance_device:node_disk_io_time_weighted_seconds:rate1m{job="node-exporter"} / scalar(count(instance_device:node_disk_io_time_weighted_seconds:rate1m{job="node-exporter"})) ) record: cluster:disk_io_saturation:ratio - expr: | sum ( max without (fstype, mountpoint) ( node_filesystem_size_bytes{job="node-exporter", fstype!=""} - node_filesystem_avail_bytes{job="node-exporter", fstype!=""} ) ) / scalar(sum(max without (fstype, mountpoint) (node_filesystem_size_bytes{job="node-exporter", fstype!=""}))) record: cluster:disk_space_utilization:ratio ================================================ FILE: manager/manifests/prometheus-operator.yaml ================================================ # Copyright 2016 The prometheus-operator Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Modifications Copyright 2022 Cortex Labs, Inc. # Source: https://github.com/prometheus-operator/prometheus-operator/blob/release-0.44/bundle.yaml --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.4.1 creationTimestamp: null name: alertmanagerconfigs.monitoring.coreos.com spec: group: monitoring.coreos.com names: categories: - prometheus-operator kind: AlertmanagerConfig listKind: AlertmanagerConfigList plural: alertmanagerconfigs singular: alertmanagerconfig scope: Namespaced versions: - name: v1alpha1 schema: openAPIV3Schema: description: AlertmanagerConfig defines a namespaced AlertmanagerConfig to be aggregated across multiple namespaces configuring one Alertmanager cluster. properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object spec: description: AlertmanagerConfigSpec is a specification of the desired behavior of the Alertmanager configuration. By definition, the Alertmanager configuration only applies to alerts for which the `namespace` label is equal to the namespace of the AlertmanagerConfig resource. properties: inhibitRules: description: List of inhibition rules. The rules will only apply to alerts matching the resource’s namespace. items: description: InhibitRule defines an inhibition rule that allows to mute alerts when other alerts are already firing. See https://prometheus.io/docs/alerting/latest/configuration/#inhibit_rule properties: equal: description: Labels that must have an equal value in the source and target alert for the inhibition to take effect. items: type: string type: array sourceMatch: description: Matchers for which one or more alerts have to exist for the inhibition to take effect. The operator enforces that the alert matches the resource’s namespace. items: description: Matcher defines how to match on alert's labels. properties: name: description: Label to match. minLength: 1 type: string regex: description: Whether to match on equality (false) or regular-expression (true). type: boolean value: description: Label value to match. type: string required: - name type: object type: array targetMatch: description: Matchers that have to be fulfilled in the alerts to be muted. The operator enforces that the alert matches the resource’s namespace. items: description: Matcher defines how to match on alert's labels. properties: name: description: Label to match. minLength: 1 type: string regex: description: Whether to match on equality (false) or regular-expression (true). type: boolean value: description: Label value to match. type: string required: - name type: object type: array type: object type: array receivers: description: List of receivers. items: description: Receiver defines one or more notification integrations. properties: emailConfigs: description: List of Email configurations. items: description: EmailConfig configures notifications via Email. properties: authIdentity: description: The identity to use for authentication. type: string authPassword: description: The secret's key that contains the password to use for authentication. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object authSecret: description: The secret's key that contains the CRAM-MD5 secret. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object authUsername: description: The username to use for authentication. type: string from: description: The sender address. type: string headers: description: Further headers email header key/value pairs. Overrides any headers previously set by the notification implementation. items: description: KeyValue defines a (key, value) tuple. properties: key: description: Key of the tuple. minLength: 1 type: string value: description: Value of the tuple. type: string required: - key - value type: object type: array hello: description: The hostname to identify to the SMTP server. type: string html: description: The HTML body of the email notification. type: string requireTLS: description: The SMTP TLS requirement. Note that Go does not support unencrypted connections to remote SMTP endpoints. type: boolean sendResolved: description: Whether or not to notify about resolved alerts. type: boolean smarthost: description: The SMTP host through which emails are sent. type: string text: description: The text body of the email notification. type: string tlsConfig: description: TLS configuration properties: ca: description: Struct containing the CA cert to use for the targets. properties: configMap: description: ConfigMap containing data to use for the targets. properties: key: description: The key to select. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the ConfigMap or its key must be defined type: boolean required: - key type: object secret: description: Secret containing data to use for the targets. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object type: object cert: description: Struct containing the client cert file for the targets. properties: configMap: description: ConfigMap containing data to use for the targets. properties: key: description: The key to select. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the ConfigMap or its key must be defined type: boolean required: - key type: object secret: description: Secret containing data to use for the targets. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object type: object insecureSkipVerify: description: Disable target certificate validation. type: boolean keySecret: description: Secret containing the client key file for the targets. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object serverName: description: Used to verify the hostname for the targets. type: string type: object to: description: The email address to send notifications to. type: string type: object type: array name: description: Name of the receiver. Must be unique across all items from the list. minLength: 1 type: string opsgenieConfigs: description: List of OpsGenie configurations. items: description: OpsGenieConfig configures notifications via OpsGenie. See https://prometheus.io/docs/alerting/latest/configuration/#opsgenie_config properties: apiKey: description: The secret's key that contains the OpsGenie API key. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object apiURL: description: The URL to send OpsGenie API requests to. type: string description: description: Description of the incident. type: string details: description: A set of arbitrary key/value pairs that provide further detail about the incident. items: description: KeyValue defines a (key, value) tuple. properties: key: description: Key of the tuple. minLength: 1 type: string value: description: Value of the tuple. type: string required: - key - value type: object type: array httpConfig: description: HTTP client configuration. properties: basicAuth: description: BasicAuth for the client. properties: password: description: The secret in the service monitor namespace that contains the password for authentication. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object username: description: The secret in the service monitor namespace that contains the username for authentication. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object type: object bearerTokenSecret: description: The secret's key that contains the bearer token to be used by the client for authentication. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object proxyURL: description: Optional proxy URL. type: string tlsConfig: description: TLS configuration for the client. properties: ca: description: Struct containing the CA cert to use for the targets. properties: configMap: description: ConfigMap containing data to use for the targets. properties: key: description: The key to select. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the ConfigMap or its key must be defined type: boolean required: - key type: object secret: description: Secret containing data to use for the targets. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object type: object cert: description: Struct containing the client cert file for the targets. properties: configMap: description: ConfigMap containing data to use for the targets. properties: key: description: The key to select. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the ConfigMap or its key must be defined type: boolean required: - key type: object secret: description: Secret containing data to use for the targets. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object type: object insecureSkipVerify: description: Disable target certificate validation. type: boolean keySecret: description: Secret containing the client key file for the targets. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object serverName: description: Used to verify the hostname for the targets. type: string type: object type: object message: description: Alert text limited to 130 characters. type: string note: description: Additional alert note. type: string priority: description: Priority level of alert. Possible values are P1, P2, P3, P4, and P5. type: string responders: description: List of responders responsible for notifications. items: description: OpsGenieConfigResponder defines a responder to an incident. One of `id`, `name` or `username` has to be defined. properties: id: description: ID of the responder. type: string name: description: Name of the responder. type: string type: description: Type of responder. minLength: 1 type: string username: description: Username of the responder. type: string required: - type type: object type: array sendResolved: description: Whether or not to notify about resolved alerts. type: boolean source: description: Backlink to the sender of the notification. type: string tags: description: Comma separated list of tags attached to the notifications. type: string type: object type: array pagerdutyConfigs: description: List of PagerDuty configurations. items: description: PagerDutyConfig configures notifications via PagerDuty. See https://prometheus.io/docs/alerting/latest/configuration/#pagerduty_config properties: class: description: The class/type of the event. type: string client: description: Client identification. type: string clientURL: description: Backlink to the sender of notification. type: string component: description: The part or component of the affected system that is broken. type: string description: description: Description of the incident. type: string details: description: Arbitrary key/value pairs that provide further detail about the incident. items: description: KeyValue defines a (key, value) tuple. properties: key: description: Key of the tuple. minLength: 1 type: string value: description: Value of the tuple. type: string required: - key - value type: object type: array group: description: A cluster or grouping of sources. type: string httpConfig: description: HTTP client configuration. properties: basicAuth: description: BasicAuth for the client. properties: password: description: The secret in the service monitor namespace that contains the password for authentication. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object username: description: The secret in the service monitor namespace that contains the username for authentication. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object type: object bearerTokenSecret: description: The secret's key that contains the bearer token to be used by the client for authentication. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object proxyURL: description: Optional proxy URL. type: string tlsConfig: description: TLS configuration for the client. properties: ca: description: Struct containing the CA cert to use for the targets. properties: configMap: description: ConfigMap containing data to use for the targets. properties: key: description: The key to select. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the ConfigMap or its key must be defined type: boolean required: - key type: object secret: description: Secret containing data to use for the targets. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object type: object cert: description: Struct containing the client cert file for the targets. properties: configMap: description: ConfigMap containing data to use for the targets. properties: key: description: The key to select. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the ConfigMap or its key must be defined type: boolean required: - key type: object secret: description: Secret containing data to use for the targets. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object type: object insecureSkipVerify: description: Disable target certificate validation. type: boolean keySecret: description: Secret containing the client key file for the targets. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object serverName: description: Used to verify the hostname for the targets. type: string type: object type: object routingKey: description: The secret's key that contains the PagerDuty integration key (when using Events API v2). Either this field or `serviceKey` needs to be defined. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object sendResolved: description: Whether or not to notify about resolved alerts. type: boolean serviceKey: description: The secret's key that contains the PagerDuty service key (when using integration type "Prometheus"). Either this field or `routingKey` needs to be defined. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object severity: description: Severity of the incident. type: string url: description: The URL to send requests to. type: string type: object type: array pushoverConfigs: description: List of Pushover configurations. items: description: PushoverConfig configures notifications via Pushover. See https://prometheus.io/docs/alerting/latest/configuration/#pushover_config properties: expire: description: How long your notification will continue to be retried for, unless the user acknowledges the notification. type: string html: description: Whether notification message is HTML or plain text. type: boolean httpConfig: description: HTTP client configuration. properties: basicAuth: description: BasicAuth for the client. properties: password: description: The secret in the service monitor namespace that contains the password for authentication. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object username: description: The secret in the service monitor namespace that contains the username for authentication. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object type: object bearerTokenSecret: description: The secret's key that contains the bearer token to be used by the client for authentication. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object proxyURL: description: Optional proxy URL. type: string tlsConfig: description: TLS configuration for the client. properties: ca: description: Struct containing the CA cert to use for the targets. properties: configMap: description: ConfigMap containing data to use for the targets. properties: key: description: The key to select. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the ConfigMap or its key must be defined type: boolean required: - key type: object secret: description: Secret containing data to use for the targets. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object type: object cert: description: Struct containing the client cert file for the targets. properties: configMap: description: ConfigMap containing data to use for the targets. properties: key: description: The key to select. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the ConfigMap or its key must be defined type: boolean required: - key type: object secret: description: Secret containing data to use for the targets. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object type: object insecureSkipVerify: description: Disable target certificate validation. type: boolean keySecret: description: Secret containing the client key file for the targets. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object serverName: description: Used to verify the hostname for the targets. type: string type: object type: object message: description: Notification message. type: string priority: description: Priority, see https://pushover.net/api#priority type: string retry: description: How often the Pushover servers will send the same notification to the user. Must be at least 30 seconds. type: string sendResolved: description: Whether or not to notify about resolved alerts. type: boolean sound: description: The name of one of the sounds supported by device clients to override the user's default sound choice type: string title: description: Notification title. type: string token: description: The secret's key that contains the registered application’s API token, see https://pushover.net/apps. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object url: description: A supplementary URL shown alongside the message. type: string urlTitle: description: A title for supplementary URL, otherwise just the URL is shown type: string userKey: description: The secret's key that contains the recipient user’s user key. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object type: object type: array slackConfigs: description: List of Slack configurations. items: description: SlackConfig configures notifications via Slack. See https://prometheus.io/docs/alerting/latest/configuration/#slack_config properties: actions: description: A list of Slack actions that are sent with each notification. items: description: SlackAction configures a single Slack action that is sent with each notification. See https://api.slack.com/docs/message-attachments#action_fields and https://api.slack.com/docs/message-buttons for more information. properties: confirm: description: SlackConfirmationField protect users from destructive actions or particularly distinguished decisions by asking them to confirm their button click one more time. See https://api.slack.com/docs/interactive-message-field-guide#confirmation_fields for more information. properties: dismissText: type: string okText: type: string text: minLength: 1 type: string title: type: string required: - text type: object name: type: string style: type: string text: minLength: 1 type: string type: minLength: 1 type: string url: type: string value: type: string required: - text - type type: object type: array apiURL: description: The secret's key that contains the Slack webhook URL. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object callbackId: type: string channel: description: The channel or user to send notifications to. type: string color: type: string fallback: type: string fields: description: A list of Slack fields that are sent with each notification. items: description: SlackField configures a single Slack field that is sent with each notification. Each field must contain a title, value, and optionally, a boolean value to indicate if the field is short enough to be displayed next to other fields designated as short. See https://api.slack.com/docs/message-attachments#fields for more information. properties: short: type: boolean title: minLength: 1 type: string value: minLength: 1 type: string required: - title - value type: object type: array footer: type: string httpConfig: description: HTTP client configuration. properties: basicAuth: description: BasicAuth for the client. properties: password: description: The secret in the service monitor namespace that contains the password for authentication. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object username: description: The secret in the service monitor namespace that contains the username for authentication. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object type: object bearerTokenSecret: description: The secret's key that contains the bearer token to be used by the client for authentication. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object proxyURL: description: Optional proxy URL. type: string tlsConfig: description: TLS configuration for the client. properties: ca: description: Struct containing the CA cert to use for the targets. properties: configMap: description: ConfigMap containing data to use for the targets. properties: key: description: The key to select. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the ConfigMap or its key must be defined type: boolean required: - key type: object secret: description: Secret containing data to use for the targets. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object type: object cert: description: Struct containing the client cert file for the targets. properties: configMap: description: ConfigMap containing data to use for the targets. properties: key: description: The key to select. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the ConfigMap or its key must be defined type: boolean required: - key type: object secret: description: Secret containing data to use for the targets. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object type: object insecureSkipVerify: description: Disable target certificate validation. type: boolean keySecret: description: Secret containing the client key file for the targets. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object serverName: description: Used to verify the hostname for the targets. type: string type: object type: object iconEmoji: type: string iconURL: type: string imageURL: type: string linkNames: type: boolean mrkdwnIn: items: type: string type: array pretext: type: string sendResolved: description: Whether or not to notify about resolved alerts. type: boolean shortFields: type: boolean text: type: string thumbURL: type: string title: type: string titleLink: type: string username: type: string type: object type: array victoropsConfigs: description: List of VictorOps configurations. items: description: VictorOpsConfig configures notifications via VictorOps. See https://prometheus.io/docs/alerting/latest/configuration/#victorops_config properties: apiKey: description: The secret's key that contains the API key to use when talking to the VictorOps API. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object apiUrl: description: The VictorOps API URL. type: string customFields: description: Additional custom fields for notification. items: description: KeyValue defines a (key, value) tuple. properties: key: description: Key of the tuple. minLength: 1 type: string value: description: Value of the tuple. type: string required: - key - value type: object type: array entityDisplayName: description: Contains summary of the alerted problem. type: string httpConfig: description: The HTTP client's configuration. properties: basicAuth: description: BasicAuth for the client. properties: password: description: The secret in the service monitor namespace that contains the password for authentication. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object username: description: The secret in the service monitor namespace that contains the username for authentication. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object type: object bearerTokenSecret: description: The secret's key that contains the bearer token to be used by the client for authentication. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object proxyURL: description: Optional proxy URL. type: string tlsConfig: description: TLS configuration for the client. properties: ca: description: Struct containing the CA cert to use for the targets. properties: configMap: description: ConfigMap containing data to use for the targets. properties: key: description: The key to select. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the ConfigMap or its key must be defined type: boolean required: - key type: object secret: description: Secret containing data to use for the targets. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object type: object cert: description: Struct containing the client cert file for the targets. properties: configMap: description: ConfigMap containing data to use for the targets. properties: key: description: The key to select. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the ConfigMap or its key must be defined type: boolean required: - key type: object secret: description: Secret containing data to use for the targets. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object type: object insecureSkipVerify: description: Disable target certificate validation. type: boolean keySecret: description: Secret containing the client key file for the targets. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object serverName: description: Used to verify the hostname for the targets. type: string type: object type: object messageType: description: Describes the behavior of the alert (CRITICAL, WARNING, INFO). type: string monitoringTool: description: The monitoring tool the state message is from. type: string routingKey: description: A key used to map the alert to a team. type: string sendResolved: description: Whether or not to notify about resolved alerts. type: boolean stateMessage: description: Contains long explanation of the alerted problem. type: string type: object type: array webhookConfigs: description: List of webhook configurations. items: description: WebhookConfig configures notifications via a generic receiver supporting the webhook payload. See https://prometheus.io/docs/alerting/latest/configuration/#webhook_config properties: httpConfig: description: HTTP client configuration. properties: basicAuth: description: BasicAuth for the client. properties: password: description: The secret in the service monitor namespace that contains the password for authentication. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object username: description: The secret in the service monitor namespace that contains the username for authentication. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object type: object bearerTokenSecret: description: The secret's key that contains the bearer token to be used by the client for authentication. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object proxyURL: description: Optional proxy URL. type: string tlsConfig: description: TLS configuration for the client. properties: ca: description: Struct containing the CA cert to use for the targets. properties: configMap: description: ConfigMap containing data to use for the targets. properties: key: description: The key to select. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the ConfigMap or its key must be defined type: boolean required: - key type: object secret: description: Secret containing data to use for the targets. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object type: object cert: description: Struct containing the client cert file for the targets. properties: configMap: description: ConfigMap containing data to use for the targets. properties: key: description: The key to select. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the ConfigMap or its key must be defined type: boolean required: - key type: object secret: description: Secret containing data to use for the targets. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object type: object insecureSkipVerify: description: Disable target certificate validation. type: boolean keySecret: description: Secret containing the client key file for the targets. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object serverName: description: Used to verify the hostname for the targets. type: string type: object type: object maxAlerts: description: Maximum number of alerts to be sent per webhook message. When 0, all alerts are included. format: int32 minimum: 0 type: integer sendResolved: description: Whether or not to notify about resolved alerts. type: boolean url: description: The URL to send HTTP POST requests to. `urlSecret` takes precedence over `url`. One of `urlSecret` and `url` should be defined. type: string urlSecret: description: The secret's key that contains the webhook URL to send HTTP requests to. `urlSecret` takes precedence over `url`. One of `urlSecret` and `url` should be defined. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object type: object type: array wechatConfigs: description: List of WeChat configurations. items: description: WeChatConfig configures notifications via WeChat. See https://prometheus.io/docs/alerting/latest/configuration/#wechat_config properties: agentID: type: string apiSecret: description: The secret's key that contains the WeChat API key. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object apiURL: description: The WeChat API URL. type: string corpID: description: The corp id for authentication. type: string httpConfig: description: HTTP client configuration. properties: basicAuth: description: BasicAuth for the client. properties: password: description: The secret in the service monitor namespace that contains the password for authentication. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object username: description: The secret in the service monitor namespace that contains the username for authentication. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object type: object bearerTokenSecret: description: The secret's key that contains the bearer token to be used by the client for authentication. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object proxyURL: description: Optional proxy URL. type: string tlsConfig: description: TLS configuration for the client. properties: ca: description: Struct containing the CA cert to use for the targets. properties: configMap: description: ConfigMap containing data to use for the targets. properties: key: description: The key to select. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the ConfigMap or its key must be defined type: boolean required: - key type: object secret: description: Secret containing data to use for the targets. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object type: object cert: description: Struct containing the client cert file for the targets. properties: configMap: description: ConfigMap containing data to use for the targets. properties: key: description: The key to select. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the ConfigMap or its key must be defined type: boolean required: - key type: object secret: description: Secret containing data to use for the targets. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object type: object insecureSkipVerify: description: Disable target certificate validation. type: boolean keySecret: description: Secret containing the client key file for the targets. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object serverName: description: Used to verify the hostname for the targets. type: string type: object type: object message: description: API request data as defined by the WeChat API. type: string messageType: type: string sendResolved: description: Whether or not to notify about resolved alerts. type: boolean toParty: type: string toTag: type: string toUser: type: string type: object type: array required: - name type: object type: array route: description: The Alertmanager route definition for alerts matching the resource’s namespace. If present, it will be added to the generated Alertmanager configuration as a first-level route. properties: continue: description: Boolean indicating whether an alert should continue matching subsequent sibling nodes. It will always be overridden to true for the first-level route by the Prometheus operator. type: boolean groupBy: description: List of labels to group by. items: type: string type: array groupInterval: description: How long to wait before sending an updated notification. Must match the regular expression `[0-9]+(ms|s|m|h)` (milliseconds seconds minutes hours). type: string groupWait: description: How long to wait before sending the initial notification. Must match the regular expression `[0-9]+(ms|s|m|h)` (milliseconds seconds minutes hours). type: string matchers: description: 'List of matchers that the alert’s labels should match. For the first level route, the operator removes any existing equality and regexp matcher on the `namespace` label and adds a `namespace: ` matcher.' items: description: Matcher defines how to match on alert's labels. properties: name: description: Label to match. minLength: 1 type: string regex: description: Whether to match on equality (false) or regular-expression (true). type: boolean value: description: Label value to match. type: string required: - name type: object type: array receiver: description: Name of the receiver for this route. If not empty, it should be listed in the `receivers` field. type: string repeatInterval: description: How long to wait before repeating the last notification. Must match the regular expression `[0-9]+(ms|s|m|h)` (milliseconds seconds minutes hours). type: string routes: description: Child routes. items: x-kubernetes-preserve-unknown-fields: true type: array type: object type: object required: - spec type: object served: true storage: true status: acceptedNames: kind: "" plural: "" conditions: [] storedVersions: [] --- --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.4.1 creationTimestamp: null name: alertmanagers.monitoring.coreos.com spec: group: monitoring.coreos.com names: categories: - prometheus-operator kind: Alertmanager listKind: AlertmanagerList plural: alertmanagers singular: alertmanager scope: Namespaced versions: - additionalPrinterColumns: - description: The version of Alertmanager jsonPath: .spec.version name: Version type: string - description: The desired replicas number of Alertmanagers jsonPath: .spec.replicas name: Replicas type: integer - jsonPath: .metadata.creationTimestamp name: Age type: date name: v1 schema: openAPIV3Schema: description: Alertmanager describes an Alertmanager cluster. properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object spec: description: 'Specification of the desired behavior of the Alertmanager cluster. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#spec-and-status' properties: additionalPeers: description: AdditionalPeers allows injecting a set of additional Alertmanagers to peer with to form a highly available cluster. items: type: string type: array affinity: description: If specified, the pod's scheduling constraints. properties: nodeAffinity: description: Describes node affinity scheduling rules for the pod. properties: preferredDuringSchedulingIgnoredDuringExecution: description: The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding "weight" to the sum if the node matches the corresponding matchExpressions; the node(s) with the highest sum are the most preferred. items: description: An empty preferred scheduling term matches all objects with implicit weight 0 (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). properties: preference: description: A node selector term, associated with the corresponding weight. properties: matchExpressions: description: A list of node selector requirements by node's labels. items: description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: The label key that the selector applies to. type: string operator: description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. type: string values: description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. items: type: string type: array required: - key - operator type: object type: array matchFields: description: A list of node selector requirements by node's fields. items: description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: The label key that the selector applies to. type: string operator: description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. type: string values: description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. items: type: string type: array required: - key - operator type: object type: array type: object weight: description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100. format: int32 type: integer required: - preference - weight type: object type: array requiredDuringSchedulingIgnoredDuringExecution: description: If the affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to an update), the system may or may not try to eventually evict the pod from its node. properties: nodeSelectorTerms: description: Required. A list of node selector terms. The terms are ORed. items: description: A null or empty node selector term matches no objects. The requirements of them are ANDed. The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. properties: matchExpressions: description: A list of node selector requirements by node's labels. items: description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: The label key that the selector applies to. type: string operator: description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. type: string values: description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. items: type: string type: array required: - key - operator type: object type: array matchFields: description: A list of node selector requirements by node's fields. items: description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: The label key that the selector applies to. type: string operator: description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. type: string values: description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. items: type: string type: array required: - key - operator type: object type: array type: object type: array required: - nodeSelectorTerms type: object type: object podAffinity: description: Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)). properties: preferredDuringSchedulingIgnoredDuringExecution: description: The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. items: description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) properties: podAffinityTerm: description: Required. A pod affinity term, associated with the corresponding weight. properties: labelSelector: description: A label query over a set of resources, in this case pods. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. items: type: string type: array required: - key - operator type: object type: array matchLabels: additionalProperties: type: string description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object namespaces: description: namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means "this pod's namespace" items: type: string type: array topologyKey: description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. type: string required: - topologyKey type: object weight: description: weight associated with matching the corresponding podAffinityTerm, in the range 1-100. format: int32 type: integer required: - podAffinityTerm - weight type: object type: array requiredDuringSchedulingIgnoredDuringExecution: description: If the affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied. items: description: Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key matches that of any node on which a pod of the set of pods is running properties: labelSelector: description: A label query over a set of resources, in this case pods. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. items: type: string type: array required: - key - operator type: object type: array matchLabels: additionalProperties: type: string description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object namespaces: description: namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means "this pod's namespace" items: type: string type: array topologyKey: description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. type: string required: - topologyKey type: object type: array type: object podAntiAffinity: description: Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)). properties: preferredDuringSchedulingIgnoredDuringExecution: description: The scheduler will prefer to schedule pods to nodes that satisfy the anti-affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling anti-affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. items: description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) properties: podAffinityTerm: description: Required. A pod affinity term, associated with the corresponding weight. properties: labelSelector: description: A label query over a set of resources, in this case pods. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. items: type: string type: array required: - key - operator type: object type: array matchLabels: additionalProperties: type: string description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object namespaces: description: namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means "this pod's namespace" items: type: string type: array topologyKey: description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. type: string required: - topologyKey type: object weight: description: weight associated with matching the corresponding podAffinityTerm, in the range 1-100. format: int32 type: integer required: - podAffinityTerm - weight type: object type: array requiredDuringSchedulingIgnoredDuringExecution: description: If the anti-affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the anti-affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied. items: description: Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key matches that of any node on which a pod of the set of pods is running properties: labelSelector: description: A label query over a set of resources, in this case pods. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. items: type: string type: array required: - key - operator type: object type: array matchLabels: additionalProperties: type: string description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object namespaces: description: namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means "this pod's namespace" items: type: string type: array topologyKey: description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. type: string required: - topologyKey type: object type: array type: object type: object alertmanagerConfigNamespaceSelector: description: Namespaces to be selected for AlertmanagerConfig discovery. If nil, only check own namespace. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. items: type: string type: array required: - key - operator type: object type: array matchLabels: additionalProperties: type: string description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object alertmanagerConfigSelector: description: AlertmanagerConfigs to be selected for to merge and configure Alertmanager with. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. items: type: string type: array required: - key - operator type: object type: array matchLabels: additionalProperties: type: string description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object baseImage: description: 'Base image that is used to deploy pods, without tag. Deprecated: use ''image'' instead' type: string clusterAdvertiseAddress: description: 'ClusterAdvertiseAddress is the explicit address to advertise in cluster. Needs to be provided for non RFC1918 [1] (public) addresses. [1] RFC1918: https://tools.ietf.org/html/rfc1918' type: string clusterGossipInterval: description: Interval between gossip attempts. type: string clusterPeerTimeout: description: Timeout for cluster peering. type: string clusterPushpullInterval: description: Interval between pushpull attempts. type: string configMaps: description: ConfigMaps is a list of ConfigMaps in the same namespace as the Alertmanager object, which shall be mounted into the Alertmanager Pods. The ConfigMaps are mounted into /etc/alertmanager/configmaps/. items: type: string type: array configSecret: description: ConfigSecret is the name of a Kubernetes Secret in the same namespace as the Alertmanager object, which contains configuration for this Alertmanager instance. Defaults to 'alertmanager-' The secret is mounted into /etc/alertmanager/config. type: string containers: description: 'Containers allows injecting additional containers. This is meant to allow adding an authentication proxy to an Alertmanager pod. Containers described here modify an operator generated container if they share the same name and modifications are done via a strategic merge patch. The current container names are: `alertmanager` and `config-reloader`. Overriding containers is entirely outside the scope of what the maintainers will support and by doing so, you accept that this behaviour may break at any time without notice.' items: description: A single application container that you want to run within a pod. properties: args: description: 'Arguments to the entrypoint. The docker image''s CMD is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container''s environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' items: type: string type: array command: description: 'Entrypoint array. Not executed within a shell. The docker image''s ENTRYPOINT is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container''s environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' items: type: string type: array env: description: List of environment variables to set in the container. Cannot be updated. items: description: EnvVar represents an environment variable present in a Container. properties: name: description: Name of the environment variable. Must be a C_IDENTIFIER. type: string value: description: 'Variable references $(VAR_NAME) are expanded using the previous defined environment variables in the container and any service environment variables. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Defaults to "".' type: string valueFrom: description: Source for the environment variable's value. Cannot be used if value is not empty. properties: configMapKeyRef: description: Selects a key of a ConfigMap. properties: key: description: The key to select. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the ConfigMap or its key must be defined type: boolean required: - key type: object fieldRef: description: 'Selects a field of the pod: supports metadata.name, metadata.namespace, metadata.labels, metadata.annotations, spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs.' properties: apiVersion: description: Version of the schema the FieldPath is written in terms of, defaults to "v1". type: string fieldPath: description: Path of the field to select in the specified API version. type: string required: - fieldPath type: object resourceFieldRef: description: 'Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported.' properties: containerName: description: 'Container name: required for volumes, optional for env vars' type: string divisor: anyOf: - type: integer - type: string description: Specifies the output format of the exposed resources, defaults to "1" pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true resource: description: 'Required: resource to select' type: string required: - resource type: object secretKeyRef: description: Selects a key of a secret in the pod's namespace properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object type: object required: - name type: object type: array envFrom: description: List of sources to populate environment variables in the container. The keys defined within a source must be a C_IDENTIFIER. All invalid keys will be reported as an event when the container is starting. When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated. items: description: EnvFromSource represents the source of a set of ConfigMaps properties: configMapRef: description: The ConfigMap to select from properties: name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the ConfigMap must be defined type: boolean type: object prefix: description: An optional identifier to prepend to each key in the ConfigMap. Must be a C_IDENTIFIER. type: string secretRef: description: The Secret to select from properties: name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret must be defined type: boolean type: object type: object type: array image: description: 'Docker image name. More info: https://kubernetes.io/docs/concepts/containers/images This field is optional to allow higher level config management to default or override container images in workload controllers like Deployments and StatefulSets.' type: string imagePullPolicy: description: 'Image pull policy. One of Always, Never, IfNotPresent. Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. Cannot be updated. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images' type: string lifecycle: description: Actions that the management system should take in response to container lifecycle events. Cannot be updated. properties: postStart: description: 'PostStart is called immediately after a container is created. If the handler fails, the container is terminated and restarted according to its restart policy. Other management of the container blocks until the hook completes. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' properties: exec: description: One and only one of the following should be specified. Exec specifies the action to take. properties: command: description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array type: object httpGet: description: HTTPGet specifies the http request to perform. properties: host: description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. type: string httpHeaders: description: Custom headers to set in the request. HTTP allows repeated headers. items: description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: description: The header field name type: string value: description: The header field value type: string required: - name - value type: object type: array path: description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: description: Scheme to use for connecting to the host. Defaults to HTTP. type: string required: - port type: object tcpSocket: description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' properties: host: description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object type: object preStop: description: 'PreStop is called immediately before a container is terminated due to an API request or management event such as liveness/startup probe failure, preemption, resource contention, etc. The handler is not called if the container crashes or exits. The reason for termination is passed to the handler. The Pod''s termination grace period countdown begins before the PreStop hooked is executed. Regardless of the outcome of the handler, the container will eventually terminate within the Pod''s termination grace period. Other management of the container blocks until the hook completes or until the termination grace period is reached. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' properties: exec: description: One and only one of the following should be specified. Exec specifies the action to take. properties: command: description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array type: object httpGet: description: HTTPGet specifies the http request to perform. properties: host: description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. type: string httpHeaders: description: Custom headers to set in the request. HTTP allows repeated headers. items: description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: description: The header field name type: string value: description: The header field value type: string required: - name - value type: object type: array path: description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: description: Scheme to use for connecting to the host. Defaults to HTTP. type: string required: - port type: object tcpSocket: description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' properties: host: description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object type: object type: object livenessProbe: description: 'Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' properties: exec: description: One and only one of the following should be specified. Exec specifies the action to take. properties: command: description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array type: object failureThreshold: description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1. format: int32 type: integer httpGet: description: HTTPGet specifies the http request to perform. properties: host: description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. type: string httpHeaders: description: Custom headers to set in the request. HTTP allows repeated headers. items: description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: description: The header field name type: string value: description: The header field value type: string required: - name - value type: object type: array path: description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: description: Scheme to use for connecting to the host. Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer periodSeconds: description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1. format: int32 type: integer successThreshold: description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. format: int32 type: integer tcpSocket: description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' properties: host: description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object timeoutSeconds: description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer type: object name: description: Name of the container specified as a DNS_LABEL. Each container in a pod must have a unique name (DNS_LABEL). Cannot be updated. type: string ports: description: List of ports to expose from the container. Exposing a port here gives the system additional information about the network connections a container uses, but is primarily informational. Not specifying a port here DOES NOT prevent that port from being exposed. Any port which is listening on the default "0.0.0.0" address inside a container will be accessible from the network. Cannot be updated. items: description: ContainerPort represents a network port in a single container. properties: containerPort: description: Number of port to expose on the pod's IP address. This must be a valid port number, 0 < x < 65536. format: int32 type: integer hostIP: description: What host IP to bind the external port to. type: string hostPort: description: Number of port to expose on the host. If specified, this must be a valid port number, 0 < x < 65536. If HostNetwork is specified, this must match ContainerPort. Most containers do not need this. format: int32 type: integer name: description: If specified, this must be an IANA_SVC_NAME and unique within the pod. Each named port in a pod must have a unique name. Name for the port that can be referred to by services. type: string protocol: default: TCP description: Protocol for port. Must be UDP, TCP, or SCTP. Defaults to "TCP". type: string required: - containerPort type: object type: array x-kubernetes-list-map-keys: - containerPort - protocol x-kubernetes-list-type: map readinessProbe: description: 'Periodic probe of container service readiness. Container will be removed from service endpoints if the probe fails. Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' properties: exec: description: One and only one of the following should be specified. Exec specifies the action to take. properties: command: description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array type: object failureThreshold: description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1. format: int32 type: integer httpGet: description: HTTPGet specifies the http request to perform. properties: host: description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. type: string httpHeaders: description: Custom headers to set in the request. HTTP allows repeated headers. items: description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: description: The header field name type: string value: description: The header field value type: string required: - name - value type: object type: array path: description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: description: Scheme to use for connecting to the host. Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer periodSeconds: description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1. format: int32 type: integer successThreshold: description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. format: int32 type: integer tcpSocket: description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' properties: host: description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object timeoutSeconds: description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer type: object resources: description: 'Compute Resources required by this container. Cannot be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' properties: limits: additionalProperties: anyOf: - type: integer - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' type: object requests: additionalProperties: anyOf: - type: integer - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' type: object type: object securityContext: description: 'Security options the pod should run with. More info: https://kubernetes.io/docs/concepts/policy/security-context/ More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/' properties: allowPrivilegeEscalation: description: 'AllowPrivilegeEscalation controls whether a process can gain more privileges than its parent process. This bool directly controls if the no_new_privs flag will be set on the container process. AllowPrivilegeEscalation is true always when the container is: 1) run as Privileged 2) has CAP_SYS_ADMIN' type: boolean capabilities: description: The capabilities to add/drop when running containers. Defaults to the default set of capabilities granted by the container runtime. properties: add: description: Added capabilities items: description: Capability represent POSIX capabilities type type: string type: array drop: description: Removed capabilities items: description: Capability represent POSIX capabilities type type: string type: array type: object privileged: description: Run container in privileged mode. Processes in privileged containers are essentially equivalent to root on the host. Defaults to false. type: boolean procMount: description: procMount denotes the type of proc mount to use for the containers. The default is DefaultProcMount which uses the container runtime defaults for readonly paths and masked paths. This requires the ProcMountType feature flag to be enabled. type: string readOnlyRootFilesystem: description: Whether this container has a read-only root filesystem. Default is false. type: boolean runAsGroup: description: The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. format: int64 type: integer runAsNonRoot: description: Indicates that the container must run as a non-root user. If true, the Kubelet will validate the image at runtime to ensure that it does not run as UID 0 (root) and fail to start the container if it does. If unset or false, no such validation will be performed. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. type: boolean runAsUser: description: The UID to run the entrypoint of the container process. Defaults to user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. format: int64 type: integer seLinuxOptions: description: The SELinux context to be applied to the container. If unspecified, the container runtime will allocate a random SELinux context for each container. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. properties: level: description: Level is SELinux level label that applies to the container. type: string role: description: Role is a SELinux role label that applies to the container. type: string type: description: Type is a SELinux type label that applies to the container. type: string user: description: User is a SELinux user label that applies to the container. type: string type: object windowsOptions: description: The Windows specific settings applied to all containers. If unspecified, the options from the PodSecurityContext will be used. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. properties: gmsaCredentialSpec: description: GMSACredentialSpec is where the GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the GMSA credential spec named by the GMSACredentialSpecName field. type: string gmsaCredentialSpecName: description: GMSACredentialSpecName is the name of the GMSA credential spec to use. type: string runAsUserName: description: The UserName in Windows to run the entrypoint of the container process. Defaults to the user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. type: string type: object type: object startupProbe: description: 'StartupProbe indicates that the Pod has successfully initialized. If specified, no other probes are executed until this completes successfully. If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. This can be used to provide different probe parameters at the beginning of a Pod''s lifecycle, when it might take a long time to load data or warm a cache, than during steady-state operation. This cannot be updated. This is a beta feature enabled by the StartupProbe feature flag. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' properties: exec: description: One and only one of the following should be specified. Exec specifies the action to take. properties: command: description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array type: object failureThreshold: description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1. format: int32 type: integer httpGet: description: HTTPGet specifies the http request to perform. properties: host: description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. type: string httpHeaders: description: Custom headers to set in the request. HTTP allows repeated headers. items: description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: description: The header field name type: string value: description: The header field value type: string required: - name - value type: object type: array path: description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: description: Scheme to use for connecting to the host. Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer periodSeconds: description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1. format: int32 type: integer successThreshold: description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. format: int32 type: integer tcpSocket: description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' properties: host: description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object timeoutSeconds: description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer type: object stdin: description: Whether this container should allocate a buffer for stdin in the container runtime. If this is not set, reads from stdin in the container will always result in EOF. Default is false. type: boolean stdinOnce: description: Whether the container runtime should close the stdin channel after it has been opened by a single attach. When stdin is true the stdin stream will remain open across multiple attach sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the first client attaches to stdin, and then remains open and accepts data until the client disconnects, at which time stdin is closed and remains closed until the container is restarted. If this flag is false, a container processes that reads from stdin will never receive an EOF. Default is false type: boolean terminationMessagePath: description: 'Optional: Path at which the file to which the container''s termination message will be written is mounted into the container''s filesystem. Message written is intended to be brief final status, such as an assertion failure message. Will be truncated by the node if greater than 4096 bytes. The total message length across all containers will be limited to 12kb. Defaults to /dev/termination-log. Cannot be updated.' type: string terminationMessagePolicy: description: Indicate how the termination message should be populated. File will use the contents of terminationMessagePath to populate the container status message on both success and failure. FallbackToLogsOnError will use the last chunk of container log output if the termination message file is empty and the container exited with an error. The log output is limited to 2048 bytes or 80 lines, whichever is smaller. Defaults to File. Cannot be updated. type: string tty: description: Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. Default is false. type: boolean volumeDevices: description: volumeDevices is the list of block devices to be used by the container. items: description: volumeDevice describes a mapping of a raw block device within a container. properties: devicePath: description: devicePath is the path inside of the container that the device will be mapped to. type: string name: description: name must match the name of a persistentVolumeClaim in the pod type: string required: - devicePath - name type: object type: array volumeMounts: description: Pod volumes to mount into the container's filesystem. Cannot be updated. items: description: VolumeMount describes a mounting of a Volume within a container. properties: mountPath: description: Path within the container at which the volume should be mounted. Must not contain ':'. type: string mountPropagation: description: mountPropagation determines how mounts are propagated from the host to container and the other way around. When not set, MountPropagationNone is used. This field is beta in 1.10. type: string name: description: This must match the Name of a Volume. type: string readOnly: description: Mounted read-only if true, read-write otherwise (false or unspecified). Defaults to false. type: boolean subPath: description: Path within the volume from which the container's volume should be mounted. Defaults to "" (volume's root). type: string subPathExpr: description: Expanded path within the volume from which the container's volume should be mounted. Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. Defaults to "" (volume's root). SubPathExpr and SubPath are mutually exclusive. type: string required: - mountPath - name type: object type: array workingDir: description: Container's working directory. If not specified, the container runtime's default will be used, which might be configured in the container image. Cannot be updated. type: string required: - name type: object type: array externalUrl: description: The external URL the Alertmanager instances will be available under. This is necessary to generate correct URLs. This is necessary if Alertmanager is not served from root of a DNS name. type: string forceEnableClusterMode: description: ForceEnableClusterMode ensures Alertmanager does not deactivate the cluster mode when running with a single replica. Use case is e.g. spanning an Alertmanager cluster across Kubernetes clusters with a single replica in each. type: boolean image: description: Image if specified has precedence over baseImage, tag and sha combinations. Specifying the version is still necessary to ensure the Prometheus Operator knows what version of Alertmanager is being configured. type: string imagePullSecrets: description: An optional list of references to secrets in the same namespace to use for pulling prometheus and alertmanager images from registries see http://kubernetes.io/docs/user-guide/images#specifying-imagepullsecrets-on-a-pod items: description: LocalObjectReference contains enough information to let you locate the referenced object inside the same namespace. properties: name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string type: object type: array initContainers: description: 'InitContainers allows adding initContainers to the pod definition. Those can be used to e.g. fetch secrets for injection into the Alertmanager configuration from external sources. Any errors during the execution of an initContainer will lead to a restart of the Pod. More info: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ Using initContainers for any use case other then secret fetching is entirely outside the scope of what the maintainers will support and by doing so, you accept that this behaviour may break at any time without notice.' items: description: A single application container that you want to run within a pod. properties: args: description: 'Arguments to the entrypoint. The docker image''s CMD is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container''s environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' items: type: string type: array command: description: 'Entrypoint array. Not executed within a shell. The docker image''s ENTRYPOINT is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container''s environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' items: type: string type: array env: description: List of environment variables to set in the container. Cannot be updated. items: description: EnvVar represents an environment variable present in a Container. properties: name: description: Name of the environment variable. Must be a C_IDENTIFIER. type: string value: description: 'Variable references $(VAR_NAME) are expanded using the previous defined environment variables in the container and any service environment variables. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Defaults to "".' type: string valueFrom: description: Source for the environment variable's value. Cannot be used if value is not empty. properties: configMapKeyRef: description: Selects a key of a ConfigMap. properties: key: description: The key to select. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the ConfigMap or its key must be defined type: boolean required: - key type: object fieldRef: description: 'Selects a field of the pod: supports metadata.name, metadata.namespace, metadata.labels, metadata.annotations, spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs.' properties: apiVersion: description: Version of the schema the FieldPath is written in terms of, defaults to "v1". type: string fieldPath: description: Path of the field to select in the specified API version. type: string required: - fieldPath type: object resourceFieldRef: description: 'Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported.' properties: containerName: description: 'Container name: required for volumes, optional for env vars' type: string divisor: anyOf: - type: integer - type: string description: Specifies the output format of the exposed resources, defaults to "1" pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true resource: description: 'Required: resource to select' type: string required: - resource type: object secretKeyRef: description: Selects a key of a secret in the pod's namespace properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object type: object required: - name type: object type: array envFrom: description: List of sources to populate environment variables in the container. The keys defined within a source must be a C_IDENTIFIER. All invalid keys will be reported as an event when the container is starting. When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated. items: description: EnvFromSource represents the source of a set of ConfigMaps properties: configMapRef: description: The ConfigMap to select from properties: name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the ConfigMap must be defined type: boolean type: object prefix: description: An optional identifier to prepend to each key in the ConfigMap. Must be a C_IDENTIFIER. type: string secretRef: description: The Secret to select from properties: name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret must be defined type: boolean type: object type: object type: array image: description: 'Docker image name. More info: https://kubernetes.io/docs/concepts/containers/images This field is optional to allow higher level config management to default or override container images in workload controllers like Deployments and StatefulSets.' type: string imagePullPolicy: description: 'Image pull policy. One of Always, Never, IfNotPresent. Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. Cannot be updated. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images' type: string lifecycle: description: Actions that the management system should take in response to container lifecycle events. Cannot be updated. properties: postStart: description: 'PostStart is called immediately after a container is created. If the handler fails, the container is terminated and restarted according to its restart policy. Other management of the container blocks until the hook completes. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' properties: exec: description: One and only one of the following should be specified. Exec specifies the action to take. properties: command: description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array type: object httpGet: description: HTTPGet specifies the http request to perform. properties: host: description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. type: string httpHeaders: description: Custom headers to set in the request. HTTP allows repeated headers. items: description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: description: The header field name type: string value: description: The header field value type: string required: - name - value type: object type: array path: description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: description: Scheme to use for connecting to the host. Defaults to HTTP. type: string required: - port type: object tcpSocket: description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' properties: host: description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object type: object preStop: description: 'PreStop is called immediately before a container is terminated due to an API request or management event such as liveness/startup probe failure, preemption, resource contention, etc. The handler is not called if the container crashes or exits. The reason for termination is passed to the handler. The Pod''s termination grace period countdown begins before the PreStop hooked is executed. Regardless of the outcome of the handler, the container will eventually terminate within the Pod''s termination grace period. Other management of the container blocks until the hook completes or until the termination grace period is reached. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' properties: exec: description: One and only one of the following should be specified. Exec specifies the action to take. properties: command: description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array type: object httpGet: description: HTTPGet specifies the http request to perform. properties: host: description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. type: string httpHeaders: description: Custom headers to set in the request. HTTP allows repeated headers. items: description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: description: The header field name type: string value: description: The header field value type: string required: - name - value type: object type: array path: description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: description: Scheme to use for connecting to the host. Defaults to HTTP. type: string required: - port type: object tcpSocket: description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' properties: host: description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object type: object type: object livenessProbe: description: 'Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' properties: exec: description: One and only one of the following should be specified. Exec specifies the action to take. properties: command: description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array type: object failureThreshold: description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1. format: int32 type: integer httpGet: description: HTTPGet specifies the http request to perform. properties: host: description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. type: string httpHeaders: description: Custom headers to set in the request. HTTP allows repeated headers. items: description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: description: The header field name type: string value: description: The header field value type: string required: - name - value type: object type: array path: description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: description: Scheme to use for connecting to the host. Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer periodSeconds: description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1. format: int32 type: integer successThreshold: description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. format: int32 type: integer tcpSocket: description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' properties: host: description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object timeoutSeconds: description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer type: object name: description: Name of the container specified as a DNS_LABEL. Each container in a pod must have a unique name (DNS_LABEL). Cannot be updated. type: string ports: description: List of ports to expose from the container. Exposing a port here gives the system additional information about the network connections a container uses, but is primarily informational. Not specifying a port here DOES NOT prevent that port from being exposed. Any port which is listening on the default "0.0.0.0" address inside a container will be accessible from the network. Cannot be updated. items: description: ContainerPort represents a network port in a single container. properties: containerPort: description: Number of port to expose on the pod's IP address. This must be a valid port number, 0 < x < 65536. format: int32 type: integer hostIP: description: What host IP to bind the external port to. type: string hostPort: description: Number of port to expose on the host. If specified, this must be a valid port number, 0 < x < 65536. If HostNetwork is specified, this must match ContainerPort. Most containers do not need this. format: int32 type: integer name: description: If specified, this must be an IANA_SVC_NAME and unique within the pod. Each named port in a pod must have a unique name. Name for the port that can be referred to by services. type: string protocol: default: TCP description: Protocol for port. Must be UDP, TCP, or SCTP. Defaults to "TCP". type: string required: - containerPort type: object type: array x-kubernetes-list-map-keys: - containerPort - protocol x-kubernetes-list-type: map readinessProbe: description: 'Periodic probe of container service readiness. Container will be removed from service endpoints if the probe fails. Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' properties: exec: description: One and only one of the following should be specified. Exec specifies the action to take. properties: command: description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array type: object failureThreshold: description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1. format: int32 type: integer httpGet: description: HTTPGet specifies the http request to perform. properties: host: description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. type: string httpHeaders: description: Custom headers to set in the request. HTTP allows repeated headers. items: description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: description: The header field name type: string value: description: The header field value type: string required: - name - value type: object type: array path: description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: description: Scheme to use for connecting to the host. Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer periodSeconds: description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1. format: int32 type: integer successThreshold: description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. format: int32 type: integer tcpSocket: description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' properties: host: description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object timeoutSeconds: description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer type: object resources: description: 'Compute Resources required by this container. Cannot be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' properties: limits: additionalProperties: anyOf: - type: integer - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' type: object requests: additionalProperties: anyOf: - type: integer - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' type: object type: object securityContext: description: 'Security options the pod should run with. More info: https://kubernetes.io/docs/concepts/policy/security-context/ More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/' properties: allowPrivilegeEscalation: description: 'AllowPrivilegeEscalation controls whether a process can gain more privileges than its parent process. This bool directly controls if the no_new_privs flag will be set on the container process. AllowPrivilegeEscalation is true always when the container is: 1) run as Privileged 2) has CAP_SYS_ADMIN' type: boolean capabilities: description: The capabilities to add/drop when running containers. Defaults to the default set of capabilities granted by the container runtime. properties: add: description: Added capabilities items: description: Capability represent POSIX capabilities type type: string type: array drop: description: Removed capabilities items: description: Capability represent POSIX capabilities type type: string type: array type: object privileged: description: Run container in privileged mode. Processes in privileged containers are essentially equivalent to root on the host. Defaults to false. type: boolean procMount: description: procMount denotes the type of proc mount to use for the containers. The default is DefaultProcMount which uses the container runtime defaults for readonly paths and masked paths. This requires the ProcMountType feature flag to be enabled. type: string readOnlyRootFilesystem: description: Whether this container has a read-only root filesystem. Default is false. type: boolean runAsGroup: description: The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. format: int64 type: integer runAsNonRoot: description: Indicates that the container must run as a non-root user. If true, the Kubelet will validate the image at runtime to ensure that it does not run as UID 0 (root) and fail to start the container if it does. If unset or false, no such validation will be performed. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. type: boolean runAsUser: description: The UID to run the entrypoint of the container process. Defaults to user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. format: int64 type: integer seLinuxOptions: description: The SELinux context to be applied to the container. If unspecified, the container runtime will allocate a random SELinux context for each container. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. properties: level: description: Level is SELinux level label that applies to the container. type: string role: description: Role is a SELinux role label that applies to the container. type: string type: description: Type is a SELinux type label that applies to the container. type: string user: description: User is a SELinux user label that applies to the container. type: string type: object windowsOptions: description: The Windows specific settings applied to all containers. If unspecified, the options from the PodSecurityContext will be used. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. properties: gmsaCredentialSpec: description: GMSACredentialSpec is where the GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the GMSA credential spec named by the GMSACredentialSpecName field. type: string gmsaCredentialSpecName: description: GMSACredentialSpecName is the name of the GMSA credential spec to use. type: string runAsUserName: description: The UserName in Windows to run the entrypoint of the container process. Defaults to the user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. type: string type: object type: object startupProbe: description: 'StartupProbe indicates that the Pod has successfully initialized. If specified, no other probes are executed until this completes successfully. If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. This can be used to provide different probe parameters at the beginning of a Pod''s lifecycle, when it might take a long time to load data or warm a cache, than during steady-state operation. This cannot be updated. This is a beta feature enabled by the StartupProbe feature flag. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' properties: exec: description: One and only one of the following should be specified. Exec specifies the action to take. properties: command: description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array type: object failureThreshold: description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1. format: int32 type: integer httpGet: description: HTTPGet specifies the http request to perform. properties: host: description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. type: string httpHeaders: description: Custom headers to set in the request. HTTP allows repeated headers. items: description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: description: The header field name type: string value: description: The header field value type: string required: - name - value type: object type: array path: description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: description: Scheme to use for connecting to the host. Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer periodSeconds: description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1. format: int32 type: integer successThreshold: description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. format: int32 type: integer tcpSocket: description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' properties: host: description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object timeoutSeconds: description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer type: object stdin: description: Whether this container should allocate a buffer for stdin in the container runtime. If this is not set, reads from stdin in the container will always result in EOF. Default is false. type: boolean stdinOnce: description: Whether the container runtime should close the stdin channel after it has been opened by a single attach. When stdin is true the stdin stream will remain open across multiple attach sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the first client attaches to stdin, and then remains open and accepts data until the client disconnects, at which time stdin is closed and remains closed until the container is restarted. If this flag is false, a container processes that reads from stdin will never receive an EOF. Default is false type: boolean terminationMessagePath: description: 'Optional: Path at which the file to which the container''s termination message will be written is mounted into the container''s filesystem. Message written is intended to be brief final status, such as an assertion failure message. Will be truncated by the node if greater than 4096 bytes. The total message length across all containers will be limited to 12kb. Defaults to /dev/termination-log. Cannot be updated.' type: string terminationMessagePolicy: description: Indicate how the termination message should be populated. File will use the contents of terminationMessagePath to populate the container status message on both success and failure. FallbackToLogsOnError will use the last chunk of container log output if the termination message file is empty and the container exited with an error. The log output is limited to 2048 bytes or 80 lines, whichever is smaller. Defaults to File. Cannot be updated. type: string tty: description: Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. Default is false. type: boolean volumeDevices: description: volumeDevices is the list of block devices to be used by the container. items: description: volumeDevice describes a mapping of a raw block device within a container. properties: devicePath: description: devicePath is the path inside of the container that the device will be mapped to. type: string name: description: name must match the name of a persistentVolumeClaim in the pod type: string required: - devicePath - name type: object type: array volumeMounts: description: Pod volumes to mount into the container's filesystem. Cannot be updated. items: description: VolumeMount describes a mounting of a Volume within a container. properties: mountPath: description: Path within the container at which the volume should be mounted. Must not contain ':'. type: string mountPropagation: description: mountPropagation determines how mounts are propagated from the host to container and the other way around. When not set, MountPropagationNone is used. This field is beta in 1.10. type: string name: description: This must match the Name of a Volume. type: string readOnly: description: Mounted read-only if true, read-write otherwise (false or unspecified). Defaults to false. type: boolean subPath: description: Path within the volume from which the container's volume should be mounted. Defaults to "" (volume's root). type: string subPathExpr: description: Expanded path within the volume from which the container's volume should be mounted. Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. Defaults to "" (volume's root). SubPathExpr and SubPath are mutually exclusive. type: string required: - mountPath - name type: object type: array workingDir: description: Container's working directory. If not specified, the container runtime's default will be used, which might be configured in the container image. Cannot be updated. type: string required: - name type: object type: array listenLocal: description: ListenLocal makes the Alertmanager server listen on loopback, so that it does not bind against the Pod IP. Note this is only for the Alertmanager UI, not the gossip communication. type: boolean logFormat: description: Log format for Alertmanager to be configured with. type: string logLevel: description: Log level for Alertmanager to be configured with. type: string nodeSelector: additionalProperties: type: string description: Define which Nodes the Pods are scheduled on. type: object paused: description: If set to true all actions on the underlying managed objects are not goint to be performed, except for delete actions. type: boolean podMetadata: description: PodMetadata configures Labels and Annotations which are propagated to the alertmanager pods. properties: annotations: additionalProperties: type: string description: 'Annotations is an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should be preserved when modifying objects. More info: http://kubernetes.io/docs/user-guide/annotations' type: object labels: additionalProperties: type: string description: 'Map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. More info: http://kubernetes.io/docs/user-guide/labels' type: object name: description: 'Name must be unique within a namespace. Is required when creating resources, although some resources may allow a client to request the generation of an appropriate name automatically. Name is primarily intended for creation idempotence and configuration definition. Cannot be updated. More info: http://kubernetes.io/docs/user-guide/identifiers#names' type: string type: object portName: description: Port name used for the pods and governing service. This defaults to web type: string priorityClassName: description: Priority class assigned to the Pods type: string replicas: description: Size is the expected size of the alertmanager cluster. The controller will eventually make the size of the running cluster equal to the expected size. format: int32 type: integer resources: description: Define resources requests and limits for single Pods. properties: limits: additionalProperties: anyOf: - type: integer - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' type: object requests: additionalProperties: anyOf: - type: integer - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' type: object type: object retention: description: Time duration Alertmanager shall retain data for. Default is '120h', and must match the regular expression `[0-9]+(ms|s|m|h)` (milliseconds seconds minutes hours). type: string routePrefix: description: The route prefix Alertmanager registers HTTP handlers for. This is useful, if using ExternalURL and a proxy is rewriting HTTP routes of a request, and the actual ExternalURL is still true, but the server serves requests under a different route prefix. For example for use with `kubectl proxy`. type: string secrets: description: Secrets is a list of Secrets in the same namespace as the Alertmanager object, which shall be mounted into the Alertmanager Pods. The Secrets are mounted into /etc/alertmanager/secrets/. items: type: string type: array securityContext: description: SecurityContext holds pod-level security attributes and common container settings. This defaults to the default PodSecurityContext. properties: fsGroup: description: "A special supplemental group that applies to all containers in a pod. Some volume types allow the Kubelet to change the ownership of that volume to be owned by the pod: \n 1. The owning GID will be the FSGroup 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) 3. The permission bits are OR'd with rw-rw---- \n If unset, the Kubelet will not modify the ownership and permissions of any volume." format: int64 type: integer fsGroupChangePolicy: description: 'fsGroupChangePolicy defines behavior of changing ownership and permission of the volume before being exposed inside Pod. This field will only apply to volume types which support fsGroup based ownership(and permissions). It will have no effect on ephemeral volume types such as: secret, configmaps and emptydir. Valid values are "OnRootMismatch" and "Always". If not specified defaults to "Always".' type: string runAsGroup: description: The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence for that container. format: int64 type: integer runAsNonRoot: description: Indicates that the container must run as a non-root user. If true, the Kubelet will validate the image at runtime to ensure that it does not run as UID 0 (root) and fail to start the container if it does. If unset or false, no such validation will be performed. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. type: boolean runAsUser: description: The UID to run the entrypoint of the container process. Defaults to user specified in image metadata if unspecified. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence for that container. format: int64 type: integer seLinuxOptions: description: The SELinux context to be applied to all containers. If unspecified, the container runtime will allocate a random SELinux context for each container. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence for that container. properties: level: description: Level is SELinux level label that applies to the container. type: string role: description: Role is a SELinux role label that applies to the container. type: string type: description: Type is a SELinux type label that applies to the container. type: string user: description: User is a SELinux user label that applies to the container. type: string type: object supplementalGroups: description: A list of groups applied to the first process run in each container, in addition to the container's primary GID. If unspecified, no groups will be added to any container. items: format: int64 type: integer type: array sysctls: description: Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported sysctls (by the container runtime) might fail to launch. items: description: Sysctl defines a kernel parameter to be set properties: name: description: Name of a property to set type: string value: description: Value of a property to set type: string required: - name - value type: object type: array windowsOptions: description: The Windows specific settings applied to all containers. If unspecified, the options within a container's SecurityContext will be used. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. properties: gmsaCredentialSpec: description: GMSACredentialSpec is where the GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the GMSA credential spec named by the GMSACredentialSpecName field. type: string gmsaCredentialSpecName: description: GMSACredentialSpecName is the name of the GMSA credential spec to use. type: string runAsUserName: description: The UserName in Windows to run the entrypoint of the container process. Defaults to the user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. type: string type: object type: object serviceAccountName: description: ServiceAccountName is the name of the ServiceAccount to use to run the Prometheus Pods. type: string sha: description: 'SHA of Alertmanager container image to be deployed. Defaults to the value of `version`. Similar to a tag, but the SHA explicitly deploys an immutable container image. Version and Tag are ignored if SHA is set. Deprecated: use ''image'' instead. The image digest can be specified as part of the image URL.' type: string storage: description: Storage is the definition of how storage will be used by the Alertmanager instances. properties: disableMountSubPath: description: 'Deprecated: subPath usage will be disabled by default in a future release, this option will become unnecessary. DisableMountSubPath allows to remove any subPath usage in volume mounts.' type: boolean emptyDir: description: 'EmptyDirVolumeSource to be used by the Prometheus StatefulSets. If specified, used in place of any volumeClaimTemplate. More info: https://kubernetes.io/docs/concepts/storage/volumes/#emptydir' properties: medium: description: 'What type of storage medium should back this directory. The default is "" which means to use the node''s default medium. Must be an empty string (default) or Memory. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' type: string sizeLimit: anyOf: - type: integer - type: string description: 'Total amount of local storage required for this EmptyDir volume. The size limit is also applicable for memory medium. The maximum usage on memory medium EmptyDir would be the minimum value between the SizeLimit specified here and the sum of memory limits of all containers in a pod. The default is nil which means that the limit is undefined. More info: http://kubernetes.io/docs/user-guide/volumes#emptydir' pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true type: object volumeClaimTemplate: description: A PVC spec to be used by the Prometheus StatefulSets. properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: description: EmbeddedMetadata contains metadata relevant to an EmbeddedResource. properties: annotations: additionalProperties: type: string description: 'Annotations is an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should be preserved when modifying objects. More info: http://kubernetes.io/docs/user-guide/annotations' type: object labels: additionalProperties: type: string description: 'Map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. More info: http://kubernetes.io/docs/user-guide/labels' type: object name: description: 'Name must be unique within a namespace. Is required when creating resources, although some resources may allow a client to request the generation of an appropriate name automatically. Name is primarily intended for creation idempotence and configuration definition. Cannot be updated. More info: http://kubernetes.io/docs/user-guide/identifiers#names' type: string type: object spec: description: 'Spec defines the desired characteristics of a volume requested by a pod author. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' properties: accessModes: description: 'AccessModes contains the desired access modes the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1' items: type: string type: array dataSource: description: 'This field can be used to specify either: * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot - Beta) * An existing PVC (PersistentVolumeClaim) * An existing custom resource/object that implements data population (Alpha) In order to use VolumeSnapshot object types, the appropriate feature gate must be enabled (VolumeSnapshotDataSource or AnyVolumeDataSource) If the provisioner or an external controller can support the specified data source, it will create a new volume based on the contents of the specified data source. If the specified data source is not supported, the volume will not be created and the failure will be reported as an event. In the future, we plan to support more data source types and the behavior of the provisioner may change.' properties: apiGroup: description: APIGroup is the group for the resource being referenced. If APIGroup is not specified, the specified Kind must be in the core API group. For any other third-party types, APIGroup is required. type: string kind: description: Kind is the type of resource being referenced type: string name: description: Name is the name of resource being referenced type: string required: - kind - name type: object resources: description: 'Resources represents the minimum resources the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources' properties: limits: additionalProperties: anyOf: - type: integer - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' type: object requests: additionalProperties: anyOf: - type: integer - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' type: object type: object selector: description: A label query over volumes to consider for binding. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. items: type: string type: array required: - key - operator type: object type: array matchLabels: additionalProperties: type: string description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object storageClassName: description: 'Name of the StorageClass required by the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1' type: string volumeMode: description: volumeMode defines what type of volume is required by the claim. Value of Filesystem is implied when not included in claim spec. type: string volumeName: description: VolumeName is the binding reference to the PersistentVolume backing this claim. type: string type: object status: description: 'Status represents the current information/status of a persistent volume claim. Read-only. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' properties: accessModes: description: 'AccessModes contains the actual access modes the volume backing the PVC has. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1' items: type: string type: array capacity: additionalProperties: anyOf: - type: integer - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true description: Represents the actual resources of the underlying volume. type: object conditions: description: Current Condition of persistent volume claim. If underlying persistent volume is being resized then the Condition will be set to 'ResizeStarted'. items: description: PersistentVolumeClaimCondition contails details about state of pvc properties: lastProbeTime: description: Last time we probed the condition. format: date-time type: string lastTransitionTime: description: Last time the condition transitioned from one status to another. format: date-time type: string message: description: Human-readable message indicating details about last transition. type: string reason: description: Unique, this should be a short, machine understandable string that gives the reason for condition's last transition. If it reports "ResizeStarted" that means the underlying persistent volume is being resized. type: string status: type: string type: description: PersistentVolumeClaimConditionType is a valid value of PersistentVolumeClaimCondition.Type type: string required: - status - type type: object type: array phase: description: Phase represents the current phase of PersistentVolumeClaim. type: string type: object type: object type: object tag: description: 'Tag of Alertmanager container image to be deployed. Defaults to the value of `version`. Version is ignored if Tag is set. Deprecated: use ''image'' instead. The image tag can be specified as part of the image URL.' type: string tolerations: description: If specified, the pod's tolerations. items: description: The pod this Toleration is attached to tolerates any taint that matches the triple using the matching operator . properties: effect: description: Effect indicates the taint effect to match. Empty means match all taint effects. When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. type: string key: description: Key is the taint key that the toleration applies to. Empty means match all taint keys. If the key is empty, operator must be Exists; this combination means to match all values and all keys. type: string operator: description: Operator represents a key's relationship to the value. Valid operators are Exists and Equal. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category. type: string tolerationSeconds: description: TolerationSeconds represents the period of time the toleration (which must be of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, it is not set, which means tolerate the taint forever (do not evict). Zero and negative values will be treated as 0 (evict immediately) by the system. format: int64 type: integer value: description: Value is the taint value the toleration matches to. If the operator is Exists, the value should be empty, otherwise just a regular string. type: string type: object type: array topologySpreadConstraints: description: If specified, the pod's topology spread constraints. items: description: TopologySpreadConstraint specifies how to spread matching pods among the given topology. properties: labelSelector: description: LabelSelector is used to find matching pods. Pods that match this label selector are counted to determine the number of pods in their corresponding topology domain. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. items: type: string type: array required: - key - operator type: object type: array matchLabels: additionalProperties: type: string description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object maxSkew: description: 'MaxSkew describes the degree to which pods may be unevenly distributed. It''s the maximum permitted difference between the number of matching pods in any two topology domains of a given topology type. For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same labelSelector spread as 1/1/0: | zone1 | zone2 | zone3 | | P | P | | - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 1/1/1; scheduling it onto zone1(zone2) would make the ActualSkew(2-0) on zone1(zone2) violate MaxSkew(1). - if MaxSkew is 2, incoming pod can be scheduled onto any zone. It''s a required field. Default value is 1 and 0 is not allowed.' format: int32 type: integer topologyKey: description: TopologyKey is the key of node labels. Nodes that have a label with this key and identical values are considered to be in the same topology. We consider each as a "bucket", and try to put balanced number of pods into each bucket. It's a required field. type: string whenUnsatisfiable: description: 'WhenUnsatisfiable indicates how to deal with a pod if it doesn''t satisfy the spread constraint. - DoNotSchedule (default) tells the scheduler not to schedule it - ScheduleAnyway tells the scheduler to still schedule it It''s considered as "Unsatisfiable" if and only if placing incoming pod on any topology violates "MaxSkew". For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same labelSelector spread as 3/1/1: | zone1 | zone2 | zone3 | | P P P | P | P | If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler won''t make it *more* imbalanced. It''s a required field.' type: string required: - maxSkew - topologyKey - whenUnsatisfiable type: object type: array version: description: Version the cluster should be on. type: string volumeMounts: description: VolumeMounts allows configuration of additional VolumeMounts on the output StatefulSet definition. VolumeMounts specified will be appended to other VolumeMounts in the alertmanager container, that are generated as a result of StorageSpec objects. items: description: VolumeMount describes a mounting of a Volume within a container. properties: mountPath: description: Path within the container at which the volume should be mounted. Must not contain ':'. type: string mountPropagation: description: mountPropagation determines how mounts are propagated from the host to container and the other way around. When not set, MountPropagationNone is used. This field is beta in 1.10. type: string name: description: This must match the Name of a Volume. type: string readOnly: description: Mounted read-only if true, read-write otherwise (false or unspecified). Defaults to false. type: boolean subPath: description: Path within the volume from which the container's volume should be mounted. Defaults to "" (volume's root). type: string subPathExpr: description: Expanded path within the volume from which the container's volume should be mounted. Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. Defaults to "" (volume's root). SubPathExpr and SubPath are mutually exclusive. type: string required: - mountPath - name type: object type: array volumes: description: Volumes allows configuration of additional volumes on the output StatefulSet definition. Volumes specified will be appended to other volumes that are generated as a result of StorageSpec objects. items: description: Volume represents a named volume in a pod that may be accessed by any container in the pod. properties: awsElasticBlockStore: description: 'AWSElasticBlockStore represents an AWS Disk resource that is attached to a kubelet''s host machine and then exposed to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' properties: fsType: description: 'Filesystem type of the volume that you want to mount. Tip: Ensure that the filesystem type is supported by the host operating system. Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore TODO: how do we prevent errors in the filesystem from compromising the machine' type: string partition: description: 'The partition in the volume that you want to mount. If omitted, the default is to mount by volume name. Examples: For volume /dev/sda1, you specify the partition as "1". Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty).' format: int32 type: integer readOnly: description: 'Specify "true" to force and set the ReadOnly property in VolumeMounts to "true". If omitted, the default is "false". More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' type: boolean volumeID: description: 'Unique ID of the persistent disk resource in AWS (Amazon EBS volume). More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' type: string required: - volumeID type: object azureDisk: description: AzureDisk represents an Azure Data Disk mount on the host and bind mount to the pod. properties: cachingMode: description: 'Host Caching mode: None, Read Only, Read Write.' type: string diskName: description: The Name of the data disk in the blob storage type: string diskURI: description: The URI the data disk in the blob storage type: string fsType: description: Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. type: string kind: description: 'Expected values Shared: multiple blob disks per storage account Dedicated: single blob disk per storage account Managed: azure managed data disk (only in managed availability set). defaults to shared' type: string readOnly: description: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts. type: boolean required: - diskName - diskURI type: object azureFile: description: AzureFile represents an Azure File Service mount on the host and bind mount to the pod. properties: readOnly: description: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts. type: boolean secretName: description: the name of secret that contains Azure Storage Account Name and Key type: string shareName: description: Share Name type: string required: - secretName - shareName type: object cephfs: description: CephFS represents a Ceph FS mount on the host that shares a pod's lifetime properties: monitors: description: 'Required: Monitors is a collection of Ceph monitors More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' items: type: string type: array path: description: 'Optional: Used as the mounted root, rather than the full Ceph tree, default is /' type: string readOnly: description: 'Optional: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts. More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' type: boolean secretFile: description: 'Optional: SecretFile is the path to key ring for User, default is /etc/ceph/user.secret More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' type: string secretRef: description: 'Optional: SecretRef is reference to the authentication secret for User, default is empty. More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' properties: name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string type: object user: description: 'Optional: User is the rados user name, default is admin More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' type: string required: - monitors type: object cinder: description: 'Cinder represents a cinder volume attached and mounted on kubelets host machine. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' properties: fsType: description: 'Filesystem type to mount. Must be a filesystem type supported by the host operating system. Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' type: string readOnly: description: 'Optional: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' type: boolean secretRef: description: 'Optional: points to a secret object containing parameters used to connect to OpenStack.' properties: name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string type: object volumeID: description: 'volume id used to identify the volume in cinder. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' type: string required: - volumeID type: object configMap: description: ConfigMap represents a configMap that should populate this volume properties: defaultMode: description: 'Optional: mode bits to use on created files by default. Must be a value between 0 and 0777. Defaults to 0644. Directories within the path are not affected by this setting. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.' format: int32 type: integer items: description: If unspecified, each key-value pair in the Data field of the referenced ConfigMap will be projected into the volume as a file whose name is the key and content is the value. If specified, the listed keys will be projected into the specified paths, and unlisted keys will not be present. If a key is specified which is not present in the ConfigMap, the volume setup will error unless it is marked optional. Paths must be relative and may not contain the '..' path or start with '..'. items: description: Maps a string key to a path within a volume. properties: key: description: The key to project. type: string mode: description: 'Optional: mode bits to use on this file, must be a value between 0 and 0777. If not specified, the volume defaultMode will be used. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.' format: int32 type: integer path: description: The relative path of the file to map the key to. May not be an absolute path. May not contain the path element '..'. May not start with the string '..'. type: string required: - key - path type: object type: array name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the ConfigMap or its keys must be defined type: boolean type: object csi: description: CSI (Container Storage Interface) represents storage that is handled by an external CSI driver (Alpha feature). properties: driver: description: Driver is the name of the CSI driver that handles this volume. Consult with your admin for the correct name as registered in the cluster. type: string fsType: description: Filesystem type to mount. Ex. "ext4", "xfs", "ntfs". If not provided, the empty value is passed to the associated CSI driver which will determine the default filesystem to apply. type: string nodePublishSecretRef: description: NodePublishSecretRef is a reference to the secret object containing sensitive information to pass to the CSI driver to complete the CSI NodePublishVolume and NodeUnpublishVolume calls. This field is optional, and may be empty if no secret is required. If the secret object contains more than one secret, all secret references are passed. properties: name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string type: object readOnly: description: Specifies a read-only configuration for the volume. Defaults to false (read/write). type: boolean volumeAttributes: additionalProperties: type: string description: VolumeAttributes stores driver-specific properties that are passed to the CSI driver. Consult your driver's documentation for supported values. type: object required: - driver type: object downwardAPI: description: DownwardAPI represents downward API about the pod that should populate this volume properties: defaultMode: description: 'Optional: mode bits to use on created files by default. Must be a value between 0 and 0777. Defaults to 0644. Directories within the path are not affected by this setting. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.' format: int32 type: integer items: description: Items is a list of downward API volume file items: description: DownwardAPIVolumeFile represents information to create the file containing the pod field properties: fieldRef: description: 'Required: Selects a field of the pod: only annotations, labels, name and namespace are supported.' properties: apiVersion: description: Version of the schema the FieldPath is written in terms of, defaults to "v1". type: string fieldPath: description: Path of the field to select in the specified API version. type: string required: - fieldPath type: object mode: description: 'Optional: mode bits to use on this file, must be a value between 0 and 0777. If not specified, the volume defaultMode will be used. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.' format: int32 type: integer path: description: 'Required: Path is the relative path name of the file to be created. Must not be absolute or contain the ''..'' path. Must be utf-8 encoded. The first item of the relative path must not start with ''..''' type: string resourceFieldRef: description: 'Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported.' properties: containerName: description: 'Container name: required for volumes, optional for env vars' type: string divisor: anyOf: - type: integer - type: string description: Specifies the output format of the exposed resources, defaults to "1" pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true resource: description: 'Required: resource to select' type: string required: - resource type: object required: - path type: object type: array type: object emptyDir: description: 'EmptyDir represents a temporary directory that shares a pod''s lifetime. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' properties: medium: description: 'What type of storage medium should back this directory. The default is "" which means to use the node''s default medium. Must be an empty string (default) or Memory. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' type: string sizeLimit: anyOf: - type: integer - type: string description: 'Total amount of local storage required for this EmptyDir volume. The size limit is also applicable for memory medium. The maximum usage on memory medium EmptyDir would be the minimum value between the SizeLimit specified here and the sum of memory limits of all containers in a pod. The default is nil which means that the limit is undefined. More info: http://kubernetes.io/docs/user-guide/volumes#emptydir' pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true type: object fc: description: FC represents a Fibre Channel resource that is attached to a kubelet's host machine and then exposed to the pod. properties: fsType: description: 'Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. TODO: how do we prevent errors in the filesystem from compromising the machine' type: string lun: description: 'Optional: FC target lun number' format: int32 type: integer readOnly: description: 'Optional: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts.' type: boolean targetWWNs: description: 'Optional: FC target worldwide names (WWNs)' items: type: string type: array wwids: description: 'Optional: FC volume world wide identifiers (wwids) Either wwids or combination of targetWWNs and lun must be set, but not both simultaneously.' items: type: string type: array type: object flexVolume: description: FlexVolume represents a generic volume resource that is provisioned/attached using an exec based plugin. properties: driver: description: Driver is the name of the driver to use for this volume. type: string fsType: description: Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. "ext4", "xfs", "ntfs". The default filesystem depends on FlexVolume script. type: string options: additionalProperties: type: string description: 'Optional: Extra command options if any.' type: object readOnly: description: 'Optional: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts.' type: boolean secretRef: description: 'Optional: SecretRef is reference to the secret object containing sensitive information to pass to the plugin scripts. This may be empty if no secret object is specified. If the secret object contains more than one secret, all secrets are passed to the plugin scripts.' properties: name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string type: object required: - driver type: object flocker: description: Flocker represents a Flocker volume attached to a kubelet's host machine. This depends on the Flocker control service being running properties: datasetName: description: Name of the dataset stored as metadata -> name on the dataset for Flocker should be considered as deprecated type: string datasetUUID: description: UUID of the dataset. This is unique identifier of a Flocker dataset type: string type: object gcePersistentDisk: description: 'GCEPersistentDisk represents a GCE Disk resource that is attached to a kubelet''s host machine and then exposed to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' properties: fsType: description: 'Filesystem type of the volume that you want to mount. Tip: Ensure that the filesystem type is supported by the host operating system. Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk TODO: how do we prevent errors in the filesystem from compromising the machine' type: string partition: description: 'The partition in the volume that you want to mount. If omitted, the default is to mount by volume name. Examples: For volume /dev/sda1, you specify the partition as "1". Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' format: int32 type: integer pdName: description: 'Unique name of the PD resource in GCE. Used to identify the disk in GCE. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' type: string readOnly: description: 'ReadOnly here will force the ReadOnly setting in VolumeMounts. Defaults to false. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' type: boolean required: - pdName type: object gitRepo: description: 'GitRepo represents a git repository at a particular revision. DEPRECATED: GitRepo is deprecated. To provision a container with a git repo, mount an EmptyDir into an InitContainer that clones the repo using git, then mount the EmptyDir into the Pod''s container.' properties: directory: description: Target directory name. Must not contain or start with '..'. If '.' is supplied, the volume directory will be the git repository. Otherwise, if specified, the volume will contain the git repository in the subdirectory with the given name. type: string repository: description: Repository URL type: string revision: description: Commit hash for the specified revision. type: string required: - repository type: object glusterfs: description: 'Glusterfs represents a Glusterfs mount on the host that shares a pod''s lifetime. More info: https://examples.k8s.io/volumes/glusterfs/README.md' properties: endpoints: description: 'EndpointsName is the endpoint name that details Glusterfs topology. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' type: string path: description: 'Path is the Glusterfs volume path. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' type: string readOnly: description: 'ReadOnly here will force the Glusterfs volume to be mounted with read-only permissions. Defaults to false. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' type: boolean required: - endpoints - path type: object hostPath: description: 'HostPath represents a pre-existing file or directory on the host machine that is directly exposed to the container. This is generally used for system agents or other privileged things that are allowed to see the host machine. Most containers will NOT need this. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath --- TODO(jonesdl) We need to restrict who can use host directory mounts and who can/can not mount host directories as read/write.' properties: path: description: 'Path of the directory on the host. If the path is a symlink, it will follow the link to the real path. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath' type: string type: description: 'Type for HostPath Volume Defaults to "" More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath' type: string required: - path type: object iscsi: description: 'ISCSI represents an ISCSI Disk resource that is attached to a kubelet''s host machine and then exposed to the pod. More info: https://examples.k8s.io/volumes/iscsi/README.md' properties: chapAuthDiscovery: description: whether support iSCSI Discovery CHAP authentication type: boolean chapAuthSession: description: whether support iSCSI Session CHAP authentication type: boolean fsType: description: 'Filesystem type of the volume that you want to mount. Tip: Ensure that the filesystem type is supported by the host operating system. Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi TODO: how do we prevent errors in the filesystem from compromising the machine' type: string initiatorName: description: Custom iSCSI Initiator Name. If initiatorName is specified with iscsiInterface simultaneously, new iSCSI interface : will be created for the connection. type: string iqn: description: Target iSCSI Qualified Name. type: string iscsiInterface: description: iSCSI Interface Name that uses an iSCSI transport. Defaults to 'default' (tcp). type: string lun: description: iSCSI Target Lun number. format: int32 type: integer portals: description: iSCSI Target Portal List. The portal is either an IP or ip_addr:port if the port is other than default (typically TCP ports 860 and 3260). items: type: string type: array readOnly: description: ReadOnly here will force the ReadOnly setting in VolumeMounts. Defaults to false. type: boolean secretRef: description: CHAP Secret for iSCSI target and initiator authentication properties: name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string type: object targetPortal: description: iSCSI Target Portal. The Portal is either an IP or ip_addr:port if the port is other than default (typically TCP ports 860 and 3260). type: string required: - iqn - lun - targetPortal type: object name: description: 'Volume''s name. Must be a DNS_LABEL and unique within the pod. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' type: string nfs: description: 'NFS represents an NFS mount on the host that shares a pod''s lifetime More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' properties: path: description: 'Path that is exported by the NFS server. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' type: string readOnly: description: 'ReadOnly here will force the NFS export to be mounted with read-only permissions. Defaults to false. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' type: boolean server: description: 'Server is the hostname or IP address of the NFS server. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' type: string required: - path - server type: object persistentVolumeClaim: description: 'PersistentVolumeClaimVolumeSource represents a reference to a PersistentVolumeClaim in the same namespace. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' properties: claimName: description: 'ClaimName is the name of a PersistentVolumeClaim in the same namespace as the pod using this volume. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' type: string readOnly: description: Will force the ReadOnly setting in VolumeMounts. Default false. type: boolean required: - claimName type: object photonPersistentDisk: description: PhotonPersistentDisk represents a PhotonController persistent disk attached and mounted on kubelets host machine properties: fsType: description: Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. type: string pdID: description: ID that identifies Photon Controller persistent disk type: string required: - pdID type: object portworxVolume: description: PortworxVolume represents a portworx volume attached and mounted on kubelets host machine properties: fsType: description: FSType represents the filesystem type to mount Must be a filesystem type supported by the host operating system. Ex. "ext4", "xfs". Implicitly inferred to be "ext4" if unspecified. type: string readOnly: description: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts. type: boolean volumeID: description: VolumeID uniquely identifies a Portworx volume type: string required: - volumeID type: object projected: description: Items for all in one resources secrets, configmaps, and downward API properties: defaultMode: description: Mode bits to use on created files by default. Must be a value between 0 and 0777. Directories within the path are not affected by this setting. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set. format: int32 type: integer sources: description: list of volume projections items: description: Projection that may be projected along with other supported volume types properties: configMap: description: information about the configMap data to project properties: items: description: If unspecified, each key-value pair in the Data field of the referenced ConfigMap will be projected into the volume as a file whose name is the key and content is the value. If specified, the listed keys will be projected into the specified paths, and unlisted keys will not be present. If a key is specified which is not present in the ConfigMap, the volume setup will error unless it is marked optional. Paths must be relative and may not contain the '..' path or start with '..'. items: description: Maps a string key to a path within a volume. properties: key: description: The key to project. type: string mode: description: 'Optional: mode bits to use on this file, must be a value between 0 and 0777. If not specified, the volume defaultMode will be used. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.' format: int32 type: integer path: description: The relative path of the file to map the key to. May not be an absolute path. May not contain the path element '..'. May not start with the string '..'. type: string required: - key - path type: object type: array name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the ConfigMap or its keys must be defined type: boolean type: object downwardAPI: description: information about the downwardAPI data to project properties: items: description: Items is a list of DownwardAPIVolume file items: description: DownwardAPIVolumeFile represents information to create the file containing the pod field properties: fieldRef: description: 'Required: Selects a field of the pod: only annotations, labels, name and namespace are supported.' properties: apiVersion: description: Version of the schema the FieldPath is written in terms of, defaults to "v1". type: string fieldPath: description: Path of the field to select in the specified API version. type: string required: - fieldPath type: object mode: description: 'Optional: mode bits to use on this file, must be a value between 0 and 0777. If not specified, the volume defaultMode will be used. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.' format: int32 type: integer path: description: 'Required: Path is the relative path name of the file to be created. Must not be absolute or contain the ''..'' path. Must be utf-8 encoded. The first item of the relative path must not start with ''..''' type: string resourceFieldRef: description: 'Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported.' properties: containerName: description: 'Container name: required for volumes, optional for env vars' type: string divisor: anyOf: - type: integer - type: string description: Specifies the output format of the exposed resources, defaults to "1" pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true resource: description: 'Required: resource to select' type: string required: - resource type: object required: - path type: object type: array type: object secret: description: information about the secret data to project properties: items: description: If unspecified, each key-value pair in the Data field of the referenced Secret will be projected into the volume as a file whose name is the key and content is the value. If specified, the listed keys will be projected into the specified paths, and unlisted keys will not be present. If a key is specified which is not present in the Secret, the volume setup will error unless it is marked optional. Paths must be relative and may not contain the '..' path or start with '..'. items: description: Maps a string key to a path within a volume. properties: key: description: The key to project. type: string mode: description: 'Optional: mode bits to use on this file, must be a value between 0 and 0777. If not specified, the volume defaultMode will be used. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.' format: int32 type: integer path: description: The relative path of the file to map the key to. May not be an absolute path. May not contain the path element '..'. May not start with the string '..'. type: string required: - key - path type: object type: array name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean type: object serviceAccountToken: description: information about the serviceAccountToken data to project properties: audience: description: Audience is the intended audience of the token. A recipient of a token must identify itself with an identifier specified in the audience of the token, and otherwise should reject the token. The audience defaults to the identifier of the apiserver. type: string expirationSeconds: description: ExpirationSeconds is the requested duration of validity of the service account token. As the token approaches expiration, the kubelet volume plugin will proactively rotate the service account token. The kubelet will start trying to rotate the token if the token is older than 80 percent of its time to live or if the token is older than 24 hours.Defaults to 1 hour and must be at least 10 minutes. format: int64 type: integer path: description: Path is the path relative to the mount point of the file to project the token into. type: string required: - path type: object type: object type: array required: - sources type: object quobyte: description: Quobyte represents a Quobyte mount on the host that shares a pod's lifetime properties: group: description: Group to map volume access to Default is no group type: string readOnly: description: ReadOnly here will force the Quobyte volume to be mounted with read-only permissions. Defaults to false. type: boolean registry: description: Registry represents a single or multiple Quobyte Registry services specified as a string as host:port pair (multiple entries are separated with commas) which acts as the central registry for volumes type: string tenant: description: Tenant owning the given Quobyte volume in the Backend Used with dynamically provisioned Quobyte volumes, value is set by the plugin type: string user: description: User to map volume access to Defaults to serivceaccount user type: string volume: description: Volume is a string that references an already created Quobyte volume by name. type: string required: - registry - volume type: object rbd: description: 'RBD represents a Rados Block Device mount on the host that shares a pod''s lifetime. More info: https://examples.k8s.io/volumes/rbd/README.md' properties: fsType: description: 'Filesystem type of the volume that you want to mount. Tip: Ensure that the filesystem type is supported by the host operating system. Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd TODO: how do we prevent errors in the filesystem from compromising the machine' type: string image: description: 'The rados image name. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' type: string keyring: description: 'Keyring is the path to key ring for RBDUser. Default is /etc/ceph/keyring. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' type: string monitors: description: 'A collection of Ceph monitors. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' items: type: string type: array pool: description: 'The rados pool name. Default is rbd. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' type: string readOnly: description: 'ReadOnly here will force the ReadOnly setting in VolumeMounts. Defaults to false. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' type: boolean secretRef: description: 'SecretRef is name of the authentication secret for RBDUser. If provided overrides keyring. Default is nil. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' properties: name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string type: object user: description: 'The rados user name. Default is admin. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' type: string required: - image - monitors type: object scaleIO: description: ScaleIO represents a ScaleIO persistent volume attached and mounted on Kubernetes nodes. properties: fsType: description: Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. "ext4", "xfs", "ntfs". Default is "xfs". type: string gateway: description: The host address of the ScaleIO API Gateway. type: string protectionDomain: description: The name of the ScaleIO Protection Domain for the configured storage. type: string readOnly: description: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts. type: boolean secretRef: description: SecretRef references to the secret for ScaleIO user and other sensitive information. If this is not provided, Login operation will fail. properties: name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string type: object sslEnabled: description: Flag to enable/disable SSL communication with Gateway, default false type: boolean storageMode: description: Indicates whether the storage for a volume should be ThickProvisioned or ThinProvisioned. Default is ThinProvisioned. type: string storagePool: description: The ScaleIO Storage Pool associated with the protection domain. type: string system: description: The name of the storage system as configured in ScaleIO. type: string volumeName: description: The name of a volume already created in the ScaleIO system that is associated with this volume source. type: string required: - gateway - secretRef - system type: object secret: description: 'Secret represents a secret that should populate this volume. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' properties: defaultMode: description: 'Optional: mode bits to use on created files by default. Must be a value between 0 and 0777. Defaults to 0644. Directories within the path are not affected by this setting. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.' format: int32 type: integer items: description: If unspecified, each key-value pair in the Data field of the referenced Secret will be projected into the volume as a file whose name is the key and content is the value. If specified, the listed keys will be projected into the specified paths, and unlisted keys will not be present. If a key is specified which is not present in the Secret, the volume setup will error unless it is marked optional. Paths must be relative and may not contain the '..' path or start with '..'. items: description: Maps a string key to a path within a volume. properties: key: description: The key to project. type: string mode: description: 'Optional: mode bits to use on this file, must be a value between 0 and 0777. If not specified, the volume defaultMode will be used. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.' format: int32 type: integer path: description: The relative path of the file to map the key to. May not be an absolute path. May not contain the path element '..'. May not start with the string '..'. type: string required: - key - path type: object type: array optional: description: Specify whether the Secret or its keys must be defined type: boolean secretName: description: 'Name of the secret in the pod''s namespace to use. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' type: string type: object storageos: description: StorageOS represents a StorageOS volume attached and mounted on Kubernetes nodes. properties: fsType: description: Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. type: string readOnly: description: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts. type: boolean secretRef: description: SecretRef specifies the secret to use for obtaining the StorageOS API credentials. If not specified, default values will be attempted. properties: name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string type: object volumeName: description: VolumeName is the human-readable name of the StorageOS volume. Volume names are only unique within a namespace. type: string volumeNamespace: description: VolumeNamespace specifies the scope of the volume within StorageOS. If no namespace is specified then the Pod's namespace will be used. This allows the Kubernetes name scoping to be mirrored within StorageOS for tighter integration. Set VolumeName to any name to override the default behaviour. Set to "default" if you are not using namespaces within StorageOS. Namespaces that do not pre-exist within StorageOS will be created. type: string type: object vsphereVolume: description: VsphereVolume represents a vSphere volume attached and mounted on kubelets host machine properties: fsType: description: Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. type: string storagePolicyID: description: Storage Policy Based Management (SPBM) profile ID associated with the StoragePolicyName. type: string storagePolicyName: description: Storage Policy Based Management (SPBM) profile name. type: string volumePath: description: Path that identifies vSphere volume vmdk type: string required: - volumePath type: object required: - name type: object type: array type: object status: description: 'Most recent observed status of the Alertmanager cluster. Read-only. Not included when requesting from the apiserver, only from the Prometheus Operator API itself. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#spec-and-status' properties: availableReplicas: description: Total number of available pods (ready for at least minReadySeconds) targeted by this Alertmanager cluster. format: int32 type: integer paused: description: Represents whether any actions on the underlying managed objects are being performed. Only delete actions will be performed. type: boolean replicas: description: Total number of non-terminated pods targeted by this Alertmanager cluster (their labels match the selector). format: int32 type: integer unavailableReplicas: description: Total number of unavailable pods targeted by this Alertmanager cluster. format: int32 type: integer updatedReplicas: description: Total number of non-terminated pods targeted by this Alertmanager cluster that have the desired version spec. format: int32 type: integer required: - availableReplicas - paused - replicas - unavailableReplicas - updatedReplicas type: object required: - spec type: object served: true storage: true subresources: {} status: acceptedNames: kind: "" plural: "" conditions: [] storedVersions: [] --- --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.4.1 creationTimestamp: null name: podmonitors.monitoring.coreos.com spec: group: monitoring.coreos.com names: categories: - prometheus-operator kind: PodMonitor listKind: PodMonitorList plural: podmonitors singular: podmonitor scope: Namespaced versions: - name: v1 schema: openAPIV3Schema: description: PodMonitor defines monitoring for a set of pods. properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object spec: description: Specification of desired Pod selection for target discovery by Prometheus. properties: jobLabel: description: The label to use to retrieve the job name from. type: string namespaceSelector: description: Selector to select which namespaces the Endpoints objects are discovered from. properties: any: description: Boolean describing whether all namespaces are selected in contrast to a list restricting them. type: boolean matchNames: description: List of namespace names. items: type: string type: array type: object podMetricsEndpoints: description: A list of endpoints allowed as part of this PodMonitor. items: description: PodMetricsEndpoint defines a scrapeable endpoint of a Kubernetes Pod serving Prometheus metrics. properties: basicAuth: description: 'BasicAuth allow an endpoint to authenticate over basic authentication. More info: https://prometheus.io/docs/operating/configuration/#endpoint' properties: password: description: The secret in the service monitor namespace that contains the password for authentication. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object username: description: The secret in the service monitor namespace that contains the username for authentication. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object type: object bearerTokenSecret: description: Secret to mount to read bearer token for scraping targets. The secret needs to be in the same namespace as the pod monitor and accessible by the Prometheus Operator. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object honorLabels: description: HonorLabels chooses the metric's labels on collisions with target labels. type: boolean honorTimestamps: description: HonorTimestamps controls whether Prometheus respects the timestamps present in scraped data. type: boolean interval: description: Interval at which metrics should be scraped type: string metricRelabelings: description: MetricRelabelConfigs to apply to samples before ingestion. items: description: 'RelabelConfig allows dynamic rewriting of the label set, being applied to samples before ingestion. It defines ``-section of Prometheus configuration. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#metric_relabel_configs' properties: action: description: Action to perform based on regex matching. Default is 'replace' type: string modulus: description: Modulus to take of the hash of the source label values. format: int64 type: integer regex: description: Regular expression against which the extracted value is matched. Default is '(.*)' type: string replacement: description: Replacement value against which a regex replace is performed if the regular expression matches. Regex capture groups are available. Default is '$1' type: string separator: description: Separator placed between concatenated source label values. default is ';'. type: string sourceLabels: description: The source labels select values from existing labels. Their content is concatenated using the configured separator and matched against the configured regular expression for the replace, keep, and drop actions. items: type: string type: array targetLabel: description: Label to which the resulting value is written in a replace action. It is mandatory for replace actions. Regex capture groups are available. type: string type: object type: array params: additionalProperties: items: type: string type: array description: Optional HTTP URL parameters type: object path: description: HTTP path to scrape for metrics. type: string port: description: Name of the pod port this endpoint refers to. Mutually exclusive with targetPort. type: string proxyUrl: description: ProxyURL eg http://proxyserver:2195 Directs scrapes to proxy through this endpoint. type: string relabelings: description: 'RelabelConfigs to apply to samples before scraping. Prometheus Operator automatically adds relabelings for a few standard Kubernetes fields and replaces original scrape job name with __tmp_prometheus_job_name. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config' items: description: 'RelabelConfig allows dynamic rewriting of the label set, being applied to samples before ingestion. It defines ``-section of Prometheus configuration. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#metric_relabel_configs' properties: action: description: Action to perform based on regex matching. Default is 'replace' type: string modulus: description: Modulus to take of the hash of the source label values. format: int64 type: integer regex: description: Regular expression against which the extracted value is matched. Default is '(.*)' type: string replacement: description: Replacement value against which a regex replace is performed if the regular expression matches. Regex capture groups are available. Default is '$1' type: string separator: description: Separator placed between concatenated source label values. default is ';'. type: string sourceLabels: description: The source labels select values from existing labels. Their content is concatenated using the configured separator and matched against the configured regular expression for the replace, keep, and drop actions. items: type: string type: array targetLabel: description: Label to which the resulting value is written in a replace action. It is mandatory for replace actions. Regex capture groups are available. type: string type: object type: array scheme: description: HTTP scheme to use for scraping. type: string scrapeTimeout: description: Timeout after which the scrape is ended type: string targetPort: anyOf: - type: integer - type: string description: 'Deprecated: Use ''port'' instead.' x-kubernetes-int-or-string: true tlsConfig: description: TLS configuration to use when scraping the endpoint. properties: ca: description: Struct containing the CA cert to use for the targets. properties: configMap: description: ConfigMap containing data to use for the targets. properties: key: description: The key to select. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the ConfigMap or its key must be defined type: boolean required: - key type: object secret: description: Secret containing data to use for the targets. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object type: object cert: description: Struct containing the client cert file for the targets. properties: configMap: description: ConfigMap containing data to use for the targets. properties: key: description: The key to select. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the ConfigMap or its key must be defined type: boolean required: - key type: object secret: description: Secret containing data to use for the targets. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object type: object insecureSkipVerify: description: Disable target certificate validation. type: boolean keySecret: description: Secret containing the client key file for the targets. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object serverName: description: Used to verify the hostname for the targets. type: string type: object type: object type: array podTargetLabels: description: PodTargetLabels transfers labels on the Kubernetes Pod onto the target. items: type: string type: array sampleLimit: description: SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. format: int64 type: integer selector: description: Selector to select Pod objects. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. items: type: string type: array required: - key - operator type: object type: array matchLabels: additionalProperties: type: string description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object targetLimit: description: TargetLimit defines a limit on the number of scraped targets that will be accepted. format: int64 type: integer required: - podMetricsEndpoints - selector type: object required: - spec type: object served: true storage: true status: acceptedNames: kind: "" plural: "" conditions: [] storedVersions: [] --- --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.4.1 creationTimestamp: null name: probes.monitoring.coreos.com spec: group: monitoring.coreos.com names: categories: - prometheus-operator kind: Probe listKind: ProbeList plural: probes singular: probe scope: Namespaced versions: - name: v1 schema: openAPIV3Schema: description: Probe defines monitoring for a set of static targets or ingresses. properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object spec: description: Specification of desired Ingress selection for target discovery by Prometheus. properties: basicAuth: description: 'BasicAuth allow an endpoint to authenticate over basic authentication. More info: https://prometheus.io/docs/operating/configuration/#endpoint' properties: password: description: The secret in the service monitor namespace that contains the password for authentication. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object username: description: The secret in the service monitor namespace that contains the username for authentication. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object type: object bearerTokenSecret: description: Secret to mount to read bearer token for scraping targets. The secret needs to be in the same namespace as the probe and accessible by the Prometheus Operator. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object interval: description: Interval at which targets are probed using the configured prober. If not specified Prometheus' global scrape interval is used. type: string jobName: description: The job name assigned to scraped metrics by default. type: string module: description: 'The module to use for probing specifying how to probe the target. Example module configuring in the blackbox exporter: https://github.com/prometheus/blackbox_exporter/blob/master/example.yml' type: string prober: description: Specification for the prober to use for probing targets. The prober.URL parameter is required. Targets cannot be probed if left empty. properties: path: description: Path to collect metrics from. Defaults to `/probe`. type: string scheme: description: HTTP scheme to use for scraping. Defaults to `http`. type: string url: description: Mandatory URL of the prober. type: string required: - url type: object scrapeTimeout: description: Timeout for scraping metrics from the Prometheus exporter. type: string targets: description: Targets defines a set of static and/or dynamically discovered targets to be probed using the prober. properties: ingress: description: Ingress defines the set of dynamically discovered ingress objects which hosts are considered for probing. properties: namespaceSelector: description: Select Ingress objects by namespace. properties: any: description: Boolean describing whether all namespaces are selected in contrast to a list restricting them. type: boolean matchNames: description: List of namespace names. items: type: string type: array type: object relabelingConfigs: description: 'RelabelConfigs to apply to samples before ingestion. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config' items: description: 'RelabelConfig allows dynamic rewriting of the label set, being applied to samples before ingestion. It defines ``-section of Prometheus configuration. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#metric_relabel_configs' properties: action: description: Action to perform based on regex matching. Default is 'replace' type: string modulus: description: Modulus to take of the hash of the source label values. format: int64 type: integer regex: description: Regular expression against which the extracted value is matched. Default is '(.*)' type: string replacement: description: Replacement value against which a regex replace is performed if the regular expression matches. Regex capture groups are available. Default is '$1' type: string separator: description: Separator placed between concatenated source label values. default is ';'. type: string sourceLabels: description: The source labels select values from existing labels. Their content is concatenated using the configured separator and matched against the configured regular expression for the replace, keep, and drop actions. items: type: string type: array targetLabel: description: Label to which the resulting value is written in a replace action. It is mandatory for replace actions. Regex capture groups are available. type: string type: object type: array selector: description: Select Ingress objects by labels. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. items: type: string type: array required: - key - operator type: object type: array matchLabels: additionalProperties: type: string description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object type: object staticConfig: description: 'StaticConfig defines static targets which are considers for probing. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#static_config.' properties: labels: additionalProperties: type: string description: Labels assigned to all metrics scraped from the targets. type: object relabelingConfigs: description: 'RelabelConfigs to apply to samples before ingestion. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config' items: description: 'RelabelConfig allows dynamic rewriting of the label set, being applied to samples before ingestion. It defines ``-section of Prometheus configuration. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#metric_relabel_configs' properties: action: description: Action to perform based on regex matching. Default is 'replace' type: string modulus: description: Modulus to take of the hash of the source label values. format: int64 type: integer regex: description: Regular expression against which the extracted value is matched. Default is '(.*)' type: string replacement: description: Replacement value against which a regex replace is performed if the regular expression matches. Regex capture groups are available. Default is '$1' type: string separator: description: Separator placed between concatenated source label values. default is ';'. type: string sourceLabels: description: The source labels select values from existing labels. Their content is concatenated using the configured separator and matched against the configured regular expression for the replace, keep, and drop actions. items: type: string type: array targetLabel: description: Label to which the resulting value is written in a replace action. It is mandatory for replace actions. Regex capture groups are available. type: string type: object type: array static: description: Targets is a list of URLs to probe using the configured prober. items: type: string type: array type: object type: object tlsConfig: description: TLS configuration to use when scraping the endpoint. properties: ca: description: Struct containing the CA cert to use for the targets. properties: configMap: description: ConfigMap containing data to use for the targets. properties: key: description: The key to select. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the ConfigMap or its key must be defined type: boolean required: - key type: object secret: description: Secret containing data to use for the targets. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object type: object cert: description: Struct containing the client cert file for the targets. properties: configMap: description: ConfigMap containing data to use for the targets. properties: key: description: The key to select. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the ConfigMap or its key must be defined type: boolean required: - key type: object secret: description: Secret containing data to use for the targets. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object type: object insecureSkipVerify: description: Disable target certificate validation. type: boolean keySecret: description: Secret containing the client key file for the targets. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object serverName: description: Used to verify the hostname for the targets. type: string type: object type: object required: - spec type: object served: true storage: true status: acceptedNames: kind: "" plural: "" conditions: [] storedVersions: [] --- --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.4.1 creationTimestamp: null name: prometheuses.monitoring.coreos.com spec: group: monitoring.coreos.com names: categories: - prometheus-operator kind: Prometheus listKind: PrometheusList plural: prometheuses singular: prometheus scope: Namespaced versions: - additionalPrinterColumns: - description: The version of Prometheus jsonPath: .spec.version name: Version type: string - description: The desired replicas number of Prometheuses jsonPath: .spec.replicas name: Replicas type: integer - jsonPath: .metadata.creationTimestamp name: Age type: date name: v1 schema: openAPIV3Schema: description: Prometheus defines a Prometheus deployment. properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object spec: description: 'Specification of the desired behavior of the Prometheus cluster. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#spec-and-status' properties: additionalAlertManagerConfigs: description: 'AdditionalAlertManagerConfigs allows specifying a key of a Secret containing additional Prometheus AlertManager configurations. AlertManager configurations specified are appended to the configurations generated by the Prometheus Operator. Job configurations specified must have the form as specified in the official Prometheus documentation: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#alertmanager_config. As AlertManager configs are appended, the user is responsible to make sure it is valid. Note that using this feature may expose the possibility to break upgrades of Prometheus. It is advised to review Prometheus release notes to ensure that no incompatible AlertManager configs are going to break Prometheus after the upgrade.' properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object additionalAlertRelabelConfigs: description: 'AdditionalAlertRelabelConfigs allows specifying a key of a Secret containing additional Prometheus alert relabel configurations. Alert relabel configurations specified are appended to the configurations generated by the Prometheus Operator. Alert relabel configurations specified must have the form as specified in the official Prometheus documentation: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#alert_relabel_configs. As alert relabel configs are appended, the user is responsible to make sure it is valid. Note that using this feature may expose the possibility to break upgrades of Prometheus. It is advised to review Prometheus release notes to ensure that no incompatible alert relabel configs are going to break Prometheus after the upgrade.' properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object additionalScrapeConfigs: description: 'AdditionalScrapeConfigs allows specifying a key of a Secret containing additional Prometheus scrape configurations. Scrape configurations specified are appended to the configurations generated by the Prometheus Operator. Job configurations specified must have the form as specified in the official Prometheus documentation: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config. As scrape configs are appended, the user is responsible to make sure it is valid. Note that using this feature may expose the possibility to break upgrades of Prometheus. It is advised to review Prometheus release notes to ensure that no incompatible scrape configs are going to break Prometheus after the upgrade.' properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object affinity: description: If specified, the pod's scheduling constraints. properties: nodeAffinity: description: Describes node affinity scheduling rules for the pod. properties: preferredDuringSchedulingIgnoredDuringExecution: description: The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding "weight" to the sum if the node matches the corresponding matchExpressions; the node(s) with the highest sum are the most preferred. items: description: An empty preferred scheduling term matches all objects with implicit weight 0 (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). properties: preference: description: A node selector term, associated with the corresponding weight. properties: matchExpressions: description: A list of node selector requirements by node's labels. items: description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: The label key that the selector applies to. type: string operator: description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. type: string values: description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. items: type: string type: array required: - key - operator type: object type: array matchFields: description: A list of node selector requirements by node's fields. items: description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: The label key that the selector applies to. type: string operator: description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. type: string values: description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. items: type: string type: array required: - key - operator type: object type: array type: object weight: description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100. format: int32 type: integer required: - preference - weight type: object type: array requiredDuringSchedulingIgnoredDuringExecution: description: If the affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to an update), the system may or may not try to eventually evict the pod from its node. properties: nodeSelectorTerms: description: Required. A list of node selector terms. The terms are ORed. items: description: A null or empty node selector term matches no objects. The requirements of them are ANDed. The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. properties: matchExpressions: description: A list of node selector requirements by node's labels. items: description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: The label key that the selector applies to. type: string operator: description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. type: string values: description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. items: type: string type: array required: - key - operator type: object type: array matchFields: description: A list of node selector requirements by node's fields. items: description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: The label key that the selector applies to. type: string operator: description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. type: string values: description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. items: type: string type: array required: - key - operator type: object type: array type: object type: array required: - nodeSelectorTerms type: object type: object podAffinity: description: Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)). properties: preferredDuringSchedulingIgnoredDuringExecution: description: The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. items: description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) properties: podAffinityTerm: description: Required. A pod affinity term, associated with the corresponding weight. properties: labelSelector: description: A label query over a set of resources, in this case pods. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. items: type: string type: array required: - key - operator type: object type: array matchLabels: additionalProperties: type: string description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object namespaces: description: namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means "this pod's namespace" items: type: string type: array topologyKey: description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. type: string required: - topologyKey type: object weight: description: weight associated with matching the corresponding podAffinityTerm, in the range 1-100. format: int32 type: integer required: - podAffinityTerm - weight type: object type: array requiredDuringSchedulingIgnoredDuringExecution: description: If the affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied. items: description: Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key matches that of any node on which a pod of the set of pods is running properties: labelSelector: description: A label query over a set of resources, in this case pods. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. items: type: string type: array required: - key - operator type: object type: array matchLabels: additionalProperties: type: string description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object namespaces: description: namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means "this pod's namespace" items: type: string type: array topologyKey: description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. type: string required: - topologyKey type: object type: array type: object podAntiAffinity: description: Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)). properties: preferredDuringSchedulingIgnoredDuringExecution: description: The scheduler will prefer to schedule pods to nodes that satisfy the anti-affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling anti-affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. items: description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) properties: podAffinityTerm: description: Required. A pod affinity term, associated with the corresponding weight. properties: labelSelector: description: A label query over a set of resources, in this case pods. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. items: type: string type: array required: - key - operator type: object type: array matchLabels: additionalProperties: type: string description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object namespaces: description: namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means "this pod's namespace" items: type: string type: array topologyKey: description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. type: string required: - topologyKey type: object weight: description: weight associated with matching the corresponding podAffinityTerm, in the range 1-100. format: int32 type: integer required: - podAffinityTerm - weight type: object type: array requiredDuringSchedulingIgnoredDuringExecution: description: If the anti-affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the anti-affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied. items: description: Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key matches that of any node on which a pod of the set of pods is running properties: labelSelector: description: A label query over a set of resources, in this case pods. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. items: type: string type: array required: - key - operator type: object type: array matchLabels: additionalProperties: type: string description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object namespaces: description: namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means "this pod's namespace" items: type: string type: array topologyKey: description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. type: string required: - topologyKey type: object type: array type: object type: object alerting: description: Define details regarding alerting. properties: alertmanagers: description: AlertmanagerEndpoints Prometheus should fire alerts against. items: description: AlertmanagerEndpoints defines a selection of a single Endpoints object containing alertmanager IPs to fire alerts against. properties: apiVersion: description: Version of the Alertmanager API that Prometheus uses to send alerts. It can be "v1" or "v2". type: string bearerTokenFile: description: BearerTokenFile to read from filesystem to use when authenticating to Alertmanager. type: string name: description: Name of Endpoints object in Namespace. type: string namespace: description: Namespace of Endpoints object. type: string pathPrefix: description: Prefix for the HTTP path alerts are pushed to. type: string port: anyOf: - type: integer - type: string description: Port the Alertmanager API is exposed on. x-kubernetes-int-or-string: true scheme: description: Scheme to use when firing alerts. type: string timeout: description: Timeout is a per-target Alertmanager timeout when pushing alerts. type: string tlsConfig: description: TLS Config to use for alertmanager connection. properties: ca: description: Struct containing the CA cert to use for the targets. properties: configMap: description: ConfigMap containing data to use for the targets. properties: key: description: The key to select. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the ConfigMap or its key must be defined type: boolean required: - key type: object secret: description: Secret containing data to use for the targets. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object type: object caFile: description: Path to the CA cert in the Prometheus container to use for the targets. type: string cert: description: Struct containing the client cert file for the targets. properties: configMap: description: ConfigMap containing data to use for the targets. properties: key: description: The key to select. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the ConfigMap or its key must be defined type: boolean required: - key type: object secret: description: Secret containing data to use for the targets. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object type: object certFile: description: Path to the client cert file in the Prometheus container for the targets. type: string insecureSkipVerify: description: Disable target certificate validation. type: boolean keyFile: description: Path to the client key file in the Prometheus container for the targets. type: string keySecret: description: Secret containing the client key file for the targets. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object serverName: description: Used to verify the hostname for the targets. type: string type: object required: - name - namespace - port type: object type: array required: - alertmanagers type: object allowOverlappingBlocks: description: AllowOverlappingBlocks enables vertical compaction and vertical query merge in Prometheus. This is still experimental in Prometheus so it may change in any upcoming release. type: boolean apiserverConfig: description: APIServerConfig allows specifying a host and auth methods to access apiserver. If left empty, Prometheus is assumed to run inside of the cluster and will discover API servers automatically and use the pod's CA certificate and bearer token file at /var/run/secrets/kubernetes.io/serviceaccount/. properties: basicAuth: description: BasicAuth allow an endpoint to authenticate over basic authentication properties: password: description: The secret in the service monitor namespace that contains the password for authentication. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object username: description: The secret in the service monitor namespace that contains the username for authentication. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object type: object bearerToken: description: Bearer token for accessing apiserver. type: string bearerTokenFile: description: File to read bearer token for accessing apiserver. type: string host: description: Host of apiserver. A valid string consisting of a hostname or IP followed by an optional port number type: string tlsConfig: description: TLS Config to use for accessing apiserver. properties: ca: description: Struct containing the CA cert to use for the targets. properties: configMap: description: ConfigMap containing data to use for the targets. properties: key: description: The key to select. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the ConfigMap or its key must be defined type: boolean required: - key type: object secret: description: Secret containing data to use for the targets. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object type: object caFile: description: Path to the CA cert in the Prometheus container to use for the targets. type: string cert: description: Struct containing the client cert file for the targets. properties: configMap: description: ConfigMap containing data to use for the targets. properties: key: description: The key to select. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the ConfigMap or its key must be defined type: boolean required: - key type: object secret: description: Secret containing data to use for the targets. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object type: object certFile: description: Path to the client cert file in the Prometheus container for the targets. type: string insecureSkipVerify: description: Disable target certificate validation. type: boolean keyFile: description: Path to the client key file in the Prometheus container for the targets. type: string keySecret: description: Secret containing the client key file for the targets. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object serverName: description: Used to verify the hostname for the targets. type: string type: object required: - host type: object arbitraryFSAccessThroughSMs: description: ArbitraryFSAccessThroughSMs configures whether configuration based on a service monitor can access arbitrary files on the file system of the Prometheus container e.g. bearer token files. properties: deny: type: boolean type: object baseImage: description: 'Base image to use for a Prometheus deployment. Deprecated: use ''image'' instead' type: string configMaps: description: ConfigMaps is a list of ConfigMaps in the same namespace as the Prometheus object, which shall be mounted into the Prometheus Pods. The ConfigMaps are mounted into /etc/prometheus/configmaps/. items: type: string type: array containers: description: 'Containers allows injecting additional containers or modifying operator generated containers. This can be used to allow adding an authentication proxy to a Prometheus pod or to change the behavior of an operator generated container. Containers described here modify an operator generated container if they share the same name and modifications are done via a strategic merge patch. The current container names are: `prometheus`, `config-reloader`, and `thanos-sidecar`. Overriding containers is entirely outside the scope of what the maintainers will support and by doing so, you accept that this behaviour may break at any time without notice.' items: description: A single application container that you want to run within a pod. properties: args: description: 'Arguments to the entrypoint. The docker image''s CMD is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container''s environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' items: type: string type: array command: description: 'Entrypoint array. Not executed within a shell. The docker image''s ENTRYPOINT is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container''s environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' items: type: string type: array env: description: List of environment variables to set in the container. Cannot be updated. items: description: EnvVar represents an environment variable present in a Container. properties: name: description: Name of the environment variable. Must be a C_IDENTIFIER. type: string value: description: 'Variable references $(VAR_NAME) are expanded using the previous defined environment variables in the container and any service environment variables. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Defaults to "".' type: string valueFrom: description: Source for the environment variable's value. Cannot be used if value is not empty. properties: configMapKeyRef: description: Selects a key of a ConfigMap. properties: key: description: The key to select. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the ConfigMap or its key must be defined type: boolean required: - key type: object fieldRef: description: 'Selects a field of the pod: supports metadata.name, metadata.namespace, metadata.labels, metadata.annotations, spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs.' properties: apiVersion: description: Version of the schema the FieldPath is written in terms of, defaults to "v1". type: string fieldPath: description: Path of the field to select in the specified API version. type: string required: - fieldPath type: object resourceFieldRef: description: 'Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported.' properties: containerName: description: 'Container name: required for volumes, optional for env vars' type: string divisor: anyOf: - type: integer - type: string description: Specifies the output format of the exposed resources, defaults to "1" pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true resource: description: 'Required: resource to select' type: string required: - resource type: object secretKeyRef: description: Selects a key of a secret in the pod's namespace properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object type: object required: - name type: object type: array envFrom: description: List of sources to populate environment variables in the container. The keys defined within a source must be a C_IDENTIFIER. All invalid keys will be reported as an event when the container is starting. When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated. items: description: EnvFromSource represents the source of a set of ConfigMaps properties: configMapRef: description: The ConfigMap to select from properties: name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the ConfigMap must be defined type: boolean type: object prefix: description: An optional identifier to prepend to each key in the ConfigMap. Must be a C_IDENTIFIER. type: string secretRef: description: The Secret to select from properties: name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret must be defined type: boolean type: object type: object type: array image: description: 'Docker image name. More info: https://kubernetes.io/docs/concepts/containers/images This field is optional to allow higher level config management to default or override container images in workload controllers like Deployments and StatefulSets.' type: string imagePullPolicy: description: 'Image pull policy. One of Always, Never, IfNotPresent. Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. Cannot be updated. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images' type: string lifecycle: description: Actions that the management system should take in response to container lifecycle events. Cannot be updated. properties: postStart: description: 'PostStart is called immediately after a container is created. If the handler fails, the container is terminated and restarted according to its restart policy. Other management of the container blocks until the hook completes. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' properties: exec: description: One and only one of the following should be specified. Exec specifies the action to take. properties: command: description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array type: object httpGet: description: HTTPGet specifies the http request to perform. properties: host: description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. type: string httpHeaders: description: Custom headers to set in the request. HTTP allows repeated headers. items: description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: description: The header field name type: string value: description: The header field value type: string required: - name - value type: object type: array path: description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: description: Scheme to use for connecting to the host. Defaults to HTTP. type: string required: - port type: object tcpSocket: description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' properties: host: description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object type: object preStop: description: 'PreStop is called immediately before a container is terminated due to an API request or management event such as liveness/startup probe failure, preemption, resource contention, etc. The handler is not called if the container crashes or exits. The reason for termination is passed to the handler. The Pod''s termination grace period countdown begins before the PreStop hooked is executed. Regardless of the outcome of the handler, the container will eventually terminate within the Pod''s termination grace period. Other management of the container blocks until the hook completes or until the termination grace period is reached. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' properties: exec: description: One and only one of the following should be specified. Exec specifies the action to take. properties: command: description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array type: object httpGet: description: HTTPGet specifies the http request to perform. properties: host: description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. type: string httpHeaders: description: Custom headers to set in the request. HTTP allows repeated headers. items: description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: description: The header field name type: string value: description: The header field value type: string required: - name - value type: object type: array path: description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: description: Scheme to use for connecting to the host. Defaults to HTTP. type: string required: - port type: object tcpSocket: description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' properties: host: description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object type: object type: object livenessProbe: description: 'Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' properties: exec: description: One and only one of the following should be specified. Exec specifies the action to take. properties: command: description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array type: object failureThreshold: description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1. format: int32 type: integer httpGet: description: HTTPGet specifies the http request to perform. properties: host: description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. type: string httpHeaders: description: Custom headers to set in the request. HTTP allows repeated headers. items: description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: description: The header field name type: string value: description: The header field value type: string required: - name - value type: object type: array path: description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: description: Scheme to use for connecting to the host. Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer periodSeconds: description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1. format: int32 type: integer successThreshold: description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. format: int32 type: integer tcpSocket: description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' properties: host: description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object timeoutSeconds: description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer type: object name: description: Name of the container specified as a DNS_LABEL. Each container in a pod must have a unique name (DNS_LABEL). Cannot be updated. type: string ports: description: List of ports to expose from the container. Exposing a port here gives the system additional information about the network connections a container uses, but is primarily informational. Not specifying a port here DOES NOT prevent that port from being exposed. Any port which is listening on the default "0.0.0.0" address inside a container will be accessible from the network. Cannot be updated. items: description: ContainerPort represents a network port in a single container. properties: containerPort: description: Number of port to expose on the pod's IP address. This must be a valid port number, 0 < x < 65536. format: int32 type: integer hostIP: description: What host IP to bind the external port to. type: string hostPort: description: Number of port to expose on the host. If specified, this must be a valid port number, 0 < x < 65536. If HostNetwork is specified, this must match ContainerPort. Most containers do not need this. format: int32 type: integer name: description: If specified, this must be an IANA_SVC_NAME and unique within the pod. Each named port in a pod must have a unique name. Name for the port that can be referred to by services. type: string protocol: default: TCP description: Protocol for port. Must be UDP, TCP, or SCTP. Defaults to "TCP". type: string required: - containerPort type: object type: array x-kubernetes-list-map-keys: - containerPort - protocol x-kubernetes-list-type: map readinessProbe: description: 'Periodic probe of container service readiness. Container will be removed from service endpoints if the probe fails. Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' properties: exec: description: One and only one of the following should be specified. Exec specifies the action to take. properties: command: description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array type: object failureThreshold: description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1. format: int32 type: integer httpGet: description: HTTPGet specifies the http request to perform. properties: host: description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. type: string httpHeaders: description: Custom headers to set in the request. HTTP allows repeated headers. items: description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: description: The header field name type: string value: description: The header field value type: string required: - name - value type: object type: array path: description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: description: Scheme to use for connecting to the host. Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer periodSeconds: description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1. format: int32 type: integer successThreshold: description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. format: int32 type: integer tcpSocket: description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' properties: host: description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object timeoutSeconds: description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer type: object resources: description: 'Compute Resources required by this container. Cannot be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' properties: limits: additionalProperties: anyOf: - type: integer - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' type: object requests: additionalProperties: anyOf: - type: integer - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' type: object type: object securityContext: description: 'Security options the pod should run with. More info: https://kubernetes.io/docs/concepts/policy/security-context/ More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/' properties: allowPrivilegeEscalation: description: 'AllowPrivilegeEscalation controls whether a process can gain more privileges than its parent process. This bool directly controls if the no_new_privs flag will be set on the container process. AllowPrivilegeEscalation is true always when the container is: 1) run as Privileged 2) has CAP_SYS_ADMIN' type: boolean capabilities: description: The capabilities to add/drop when running containers. Defaults to the default set of capabilities granted by the container runtime. properties: add: description: Added capabilities items: description: Capability represent POSIX capabilities type type: string type: array drop: description: Removed capabilities items: description: Capability represent POSIX capabilities type type: string type: array type: object privileged: description: Run container in privileged mode. Processes in privileged containers are essentially equivalent to root on the host. Defaults to false. type: boolean procMount: description: procMount denotes the type of proc mount to use for the containers. The default is DefaultProcMount which uses the container runtime defaults for readonly paths and masked paths. This requires the ProcMountType feature flag to be enabled. type: string readOnlyRootFilesystem: description: Whether this container has a read-only root filesystem. Default is false. type: boolean runAsGroup: description: The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. format: int64 type: integer runAsNonRoot: description: Indicates that the container must run as a non-root user. If true, the Kubelet will validate the image at runtime to ensure that it does not run as UID 0 (root) and fail to start the container if it does. If unset or false, no such validation will be performed. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. type: boolean runAsUser: description: The UID to run the entrypoint of the container process. Defaults to user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. format: int64 type: integer seLinuxOptions: description: The SELinux context to be applied to the container. If unspecified, the container runtime will allocate a random SELinux context for each container. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. properties: level: description: Level is SELinux level label that applies to the container. type: string role: description: Role is a SELinux role label that applies to the container. type: string type: description: Type is a SELinux type label that applies to the container. type: string user: description: User is a SELinux user label that applies to the container. type: string type: object windowsOptions: description: The Windows specific settings applied to all containers. If unspecified, the options from the PodSecurityContext will be used. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. properties: gmsaCredentialSpec: description: GMSACredentialSpec is where the GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the GMSA credential spec named by the GMSACredentialSpecName field. type: string gmsaCredentialSpecName: description: GMSACredentialSpecName is the name of the GMSA credential spec to use. type: string runAsUserName: description: The UserName in Windows to run the entrypoint of the container process. Defaults to the user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. type: string type: object type: object startupProbe: description: 'StartupProbe indicates that the Pod has successfully initialized. If specified, no other probes are executed until this completes successfully. If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. This can be used to provide different probe parameters at the beginning of a Pod''s lifecycle, when it might take a long time to load data or warm a cache, than during steady-state operation. This cannot be updated. This is a beta feature enabled by the StartupProbe feature flag. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' properties: exec: description: One and only one of the following should be specified. Exec specifies the action to take. properties: command: description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array type: object failureThreshold: description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1. format: int32 type: integer httpGet: description: HTTPGet specifies the http request to perform. properties: host: description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. type: string httpHeaders: description: Custom headers to set in the request. HTTP allows repeated headers. items: description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: description: The header field name type: string value: description: The header field value type: string required: - name - value type: object type: array path: description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: description: Scheme to use for connecting to the host. Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer periodSeconds: description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1. format: int32 type: integer successThreshold: description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. format: int32 type: integer tcpSocket: description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' properties: host: description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object timeoutSeconds: description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer type: object stdin: description: Whether this container should allocate a buffer for stdin in the container runtime. If this is not set, reads from stdin in the container will always result in EOF. Default is false. type: boolean stdinOnce: description: Whether the container runtime should close the stdin channel after it has been opened by a single attach. When stdin is true the stdin stream will remain open across multiple attach sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the first client attaches to stdin, and then remains open and accepts data until the client disconnects, at which time stdin is closed and remains closed until the container is restarted. If this flag is false, a container processes that reads from stdin will never receive an EOF. Default is false type: boolean terminationMessagePath: description: 'Optional: Path at which the file to which the container''s termination message will be written is mounted into the container''s filesystem. Message written is intended to be brief final status, such as an assertion failure message. Will be truncated by the node if greater than 4096 bytes. The total message length across all containers will be limited to 12kb. Defaults to /dev/termination-log. Cannot be updated.' type: string terminationMessagePolicy: description: Indicate how the termination message should be populated. File will use the contents of terminationMessagePath to populate the container status message on both success and failure. FallbackToLogsOnError will use the last chunk of container log output if the termination message file is empty and the container exited with an error. The log output is limited to 2048 bytes or 80 lines, whichever is smaller. Defaults to File. Cannot be updated. type: string tty: description: Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. Default is false. type: boolean volumeDevices: description: volumeDevices is the list of block devices to be used by the container. items: description: volumeDevice describes a mapping of a raw block device within a container. properties: devicePath: description: devicePath is the path inside of the container that the device will be mapped to. type: string name: description: name must match the name of a persistentVolumeClaim in the pod type: string required: - devicePath - name type: object type: array volumeMounts: description: Pod volumes to mount into the container's filesystem. Cannot be updated. items: description: VolumeMount describes a mounting of a Volume within a container. properties: mountPath: description: Path within the container at which the volume should be mounted. Must not contain ':'. type: string mountPropagation: description: mountPropagation determines how mounts are propagated from the host to container and the other way around. When not set, MountPropagationNone is used. This field is beta in 1.10. type: string name: description: This must match the Name of a Volume. type: string readOnly: description: Mounted read-only if true, read-write otherwise (false or unspecified). Defaults to false. type: boolean subPath: description: Path within the volume from which the container's volume should be mounted. Defaults to "" (volume's root). type: string subPathExpr: description: Expanded path within the volume from which the container's volume should be mounted. Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. Defaults to "" (volume's root). SubPathExpr and SubPath are mutually exclusive. type: string required: - mountPath - name type: object type: array workingDir: description: Container's working directory. If not specified, the container runtime's default will be used, which might be configured in the container image. Cannot be updated. type: string required: - name type: object type: array disableCompaction: description: Disable prometheus compaction. type: boolean enableAdminAPI: description: 'Enable access to prometheus web admin API. Defaults to the value of `false`. WARNING: Enabling the admin APIs enables mutating endpoints, to delete data, shutdown Prometheus, and more. Enabling this should be done with care and the user is advised to add additional authentication authorization via a proxy to ensure only clients authorized to perform these actions can do so. For more information see https://prometheus.io/docs/prometheus/latest/querying/api/#tsdb-admin-apis' type: boolean enableFeatures: description: Enable access to Prometheus disabled features. By default, no features are enabled. Enabling disabled features is entirely outside the scope of what the maintainers will support and by doing so, you accept that this behaviour may break at any time without notice. For more information see https://prometheus.io/docs/prometheus/latest/disabled_features/ items: type: string type: array enforcedNamespaceLabel: description: EnforcedNamespaceLabel enforces adding a namespace label of origin for each alert and metric that is user created. The label value will always be the namespace of the object that is being created. type: string enforcedSampleLimit: description: EnforcedSampleLimit defines global limit on number of scraped samples that will be accepted. This overrides any SampleLimit set per ServiceMonitor or/and PodMonitor. It is meant to be used by admins to enforce the SampleLimit to keep overall number of samples/series under the desired limit. Note that if SampleLimit is lower that value will be taken instead. format: int64 type: integer enforcedTargetLimit: description: EnforcedTargetLimit defines a global limit on the number of scraped targets. This overrides any TargetLimit set per ServiceMonitor or/and PodMonitor. It is meant to be used by admins to enforce the TargetLimit to keep overall number of targets under the desired limit. Note that if TargetLimit is higher that value will be taken instead. format: int64 type: integer evaluationInterval: description: Interval between consecutive evaluations. type: string externalLabels: additionalProperties: type: string description: The labels to add to any time series or alerts when communicating with external systems (federation, remote storage, Alertmanager). type: object externalUrl: description: The external URL the Prometheus instances will be available under. This is necessary to generate correct URLs. This is necessary if Prometheus is not served from root of a DNS name. type: string ignoreNamespaceSelectors: description: IgnoreNamespaceSelectors if set to true will ignore NamespaceSelector settings from the podmonitor and servicemonitor configs, and they will only discover endpoints within their current namespace. Defaults to false. type: boolean image: description: Image if specified has precedence over baseImage, tag and sha combinations. Specifying the version is still necessary to ensure the Prometheus Operator knows what version of Prometheus is being configured. type: string imagePullSecrets: description: An optional list of references to secrets in the same namespace to use for pulling prometheus and alertmanager images from registries see http://kubernetes.io/docs/user-guide/images#specifying-imagepullsecrets-on-a-pod items: description: LocalObjectReference contains enough information to let you locate the referenced object inside the same namespace. properties: name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string type: object type: array initContainers: description: 'InitContainers allows adding initContainers to the pod definition. Those can be used to e.g. fetch secrets for injection into the Prometheus configuration from external sources. Any errors during the execution of an initContainer will lead to a restart of the Pod. More info: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ Using initContainers for any use case other then secret fetching is entirely outside the scope of what the maintainers will support and by doing so, you accept that this behaviour may break at any time without notice.' items: description: A single application container that you want to run within a pod. properties: args: description: 'Arguments to the entrypoint. The docker image''s CMD is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container''s environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' items: type: string type: array command: description: 'Entrypoint array. Not executed within a shell. The docker image''s ENTRYPOINT is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container''s environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' items: type: string type: array env: description: List of environment variables to set in the container. Cannot be updated. items: description: EnvVar represents an environment variable present in a Container. properties: name: description: Name of the environment variable. Must be a C_IDENTIFIER. type: string value: description: 'Variable references $(VAR_NAME) are expanded using the previous defined environment variables in the container and any service environment variables. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Defaults to "".' type: string valueFrom: description: Source for the environment variable's value. Cannot be used if value is not empty. properties: configMapKeyRef: description: Selects a key of a ConfigMap. properties: key: description: The key to select. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the ConfigMap or its key must be defined type: boolean required: - key type: object fieldRef: description: 'Selects a field of the pod: supports metadata.name, metadata.namespace, metadata.labels, metadata.annotations, spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs.' properties: apiVersion: description: Version of the schema the FieldPath is written in terms of, defaults to "v1". type: string fieldPath: description: Path of the field to select in the specified API version. type: string required: - fieldPath type: object resourceFieldRef: description: 'Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported.' properties: containerName: description: 'Container name: required for volumes, optional for env vars' type: string divisor: anyOf: - type: integer - type: string description: Specifies the output format of the exposed resources, defaults to "1" pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true resource: description: 'Required: resource to select' type: string required: - resource type: object secretKeyRef: description: Selects a key of a secret in the pod's namespace properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object type: object required: - name type: object type: array envFrom: description: List of sources to populate environment variables in the container. The keys defined within a source must be a C_IDENTIFIER. All invalid keys will be reported as an event when the container is starting. When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated. items: description: EnvFromSource represents the source of a set of ConfigMaps properties: configMapRef: description: The ConfigMap to select from properties: name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the ConfigMap must be defined type: boolean type: object prefix: description: An optional identifier to prepend to each key in the ConfigMap. Must be a C_IDENTIFIER. type: string secretRef: description: The Secret to select from properties: name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret must be defined type: boolean type: object type: object type: array image: description: 'Docker image name. More info: https://kubernetes.io/docs/concepts/containers/images This field is optional to allow higher level config management to default or override container images in workload controllers like Deployments and StatefulSets.' type: string imagePullPolicy: description: 'Image pull policy. One of Always, Never, IfNotPresent. Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. Cannot be updated. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images' type: string lifecycle: description: Actions that the management system should take in response to container lifecycle events. Cannot be updated. properties: postStart: description: 'PostStart is called immediately after a container is created. If the handler fails, the container is terminated and restarted according to its restart policy. Other management of the container blocks until the hook completes. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' properties: exec: description: One and only one of the following should be specified. Exec specifies the action to take. properties: command: description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array type: object httpGet: description: HTTPGet specifies the http request to perform. properties: host: description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. type: string httpHeaders: description: Custom headers to set in the request. HTTP allows repeated headers. items: description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: description: The header field name type: string value: description: The header field value type: string required: - name - value type: object type: array path: description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: description: Scheme to use for connecting to the host. Defaults to HTTP. type: string required: - port type: object tcpSocket: description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' properties: host: description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object type: object preStop: description: 'PreStop is called immediately before a container is terminated due to an API request or management event such as liveness/startup probe failure, preemption, resource contention, etc. The handler is not called if the container crashes or exits. The reason for termination is passed to the handler. The Pod''s termination grace period countdown begins before the PreStop hooked is executed. Regardless of the outcome of the handler, the container will eventually terminate within the Pod''s termination grace period. Other management of the container blocks until the hook completes or until the termination grace period is reached. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' properties: exec: description: One and only one of the following should be specified. Exec specifies the action to take. properties: command: description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array type: object httpGet: description: HTTPGet specifies the http request to perform. properties: host: description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. type: string httpHeaders: description: Custom headers to set in the request. HTTP allows repeated headers. items: description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: description: The header field name type: string value: description: The header field value type: string required: - name - value type: object type: array path: description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: description: Scheme to use for connecting to the host. Defaults to HTTP. type: string required: - port type: object tcpSocket: description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' properties: host: description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object type: object type: object livenessProbe: description: 'Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' properties: exec: description: One and only one of the following should be specified. Exec specifies the action to take. properties: command: description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array type: object failureThreshold: description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1. format: int32 type: integer httpGet: description: HTTPGet specifies the http request to perform. properties: host: description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. type: string httpHeaders: description: Custom headers to set in the request. HTTP allows repeated headers. items: description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: description: The header field name type: string value: description: The header field value type: string required: - name - value type: object type: array path: description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: description: Scheme to use for connecting to the host. Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer periodSeconds: description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1. format: int32 type: integer successThreshold: description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. format: int32 type: integer tcpSocket: description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' properties: host: description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object timeoutSeconds: description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer type: object name: description: Name of the container specified as a DNS_LABEL. Each container in a pod must have a unique name (DNS_LABEL). Cannot be updated. type: string ports: description: List of ports to expose from the container. Exposing a port here gives the system additional information about the network connections a container uses, but is primarily informational. Not specifying a port here DOES NOT prevent that port from being exposed. Any port which is listening on the default "0.0.0.0" address inside a container will be accessible from the network. Cannot be updated. items: description: ContainerPort represents a network port in a single container. properties: containerPort: description: Number of port to expose on the pod's IP address. This must be a valid port number, 0 < x < 65536. format: int32 type: integer hostIP: description: What host IP to bind the external port to. type: string hostPort: description: Number of port to expose on the host. If specified, this must be a valid port number, 0 < x < 65536. If HostNetwork is specified, this must match ContainerPort. Most containers do not need this. format: int32 type: integer name: description: If specified, this must be an IANA_SVC_NAME and unique within the pod. Each named port in a pod must have a unique name. Name for the port that can be referred to by services. type: string protocol: default: TCP description: Protocol for port. Must be UDP, TCP, or SCTP. Defaults to "TCP". type: string required: - containerPort type: object type: array x-kubernetes-list-map-keys: - containerPort - protocol x-kubernetes-list-type: map readinessProbe: description: 'Periodic probe of container service readiness. Container will be removed from service endpoints if the probe fails. Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' properties: exec: description: One and only one of the following should be specified. Exec specifies the action to take. properties: command: description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array type: object failureThreshold: description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1. format: int32 type: integer httpGet: description: HTTPGet specifies the http request to perform. properties: host: description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. type: string httpHeaders: description: Custom headers to set in the request. HTTP allows repeated headers. items: description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: description: The header field name type: string value: description: The header field value type: string required: - name - value type: object type: array path: description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: description: Scheme to use for connecting to the host. Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer periodSeconds: description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1. format: int32 type: integer successThreshold: description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. format: int32 type: integer tcpSocket: description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' properties: host: description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object timeoutSeconds: description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer type: object resources: description: 'Compute Resources required by this container. Cannot be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' properties: limits: additionalProperties: anyOf: - type: integer - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' type: object requests: additionalProperties: anyOf: - type: integer - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' type: object type: object securityContext: description: 'Security options the pod should run with. More info: https://kubernetes.io/docs/concepts/policy/security-context/ More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/' properties: allowPrivilegeEscalation: description: 'AllowPrivilegeEscalation controls whether a process can gain more privileges than its parent process. This bool directly controls if the no_new_privs flag will be set on the container process. AllowPrivilegeEscalation is true always when the container is: 1) run as Privileged 2) has CAP_SYS_ADMIN' type: boolean capabilities: description: The capabilities to add/drop when running containers. Defaults to the default set of capabilities granted by the container runtime. properties: add: description: Added capabilities items: description: Capability represent POSIX capabilities type type: string type: array drop: description: Removed capabilities items: description: Capability represent POSIX capabilities type type: string type: array type: object privileged: description: Run container in privileged mode. Processes in privileged containers are essentially equivalent to root on the host. Defaults to false. type: boolean procMount: description: procMount denotes the type of proc mount to use for the containers. The default is DefaultProcMount which uses the container runtime defaults for readonly paths and masked paths. This requires the ProcMountType feature flag to be enabled. type: string readOnlyRootFilesystem: description: Whether this container has a read-only root filesystem. Default is false. type: boolean runAsGroup: description: The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. format: int64 type: integer runAsNonRoot: description: Indicates that the container must run as a non-root user. If true, the Kubelet will validate the image at runtime to ensure that it does not run as UID 0 (root) and fail to start the container if it does. If unset or false, no such validation will be performed. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. type: boolean runAsUser: description: The UID to run the entrypoint of the container process. Defaults to user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. format: int64 type: integer seLinuxOptions: description: The SELinux context to be applied to the container. If unspecified, the container runtime will allocate a random SELinux context for each container. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. properties: level: description: Level is SELinux level label that applies to the container. type: string role: description: Role is a SELinux role label that applies to the container. type: string type: description: Type is a SELinux type label that applies to the container. type: string user: description: User is a SELinux user label that applies to the container. type: string type: object windowsOptions: description: The Windows specific settings applied to all containers. If unspecified, the options from the PodSecurityContext will be used. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. properties: gmsaCredentialSpec: description: GMSACredentialSpec is where the GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the GMSA credential spec named by the GMSACredentialSpecName field. type: string gmsaCredentialSpecName: description: GMSACredentialSpecName is the name of the GMSA credential spec to use. type: string runAsUserName: description: The UserName in Windows to run the entrypoint of the container process. Defaults to the user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. type: string type: object type: object startupProbe: description: 'StartupProbe indicates that the Pod has successfully initialized. If specified, no other probes are executed until this completes successfully. If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. This can be used to provide different probe parameters at the beginning of a Pod''s lifecycle, when it might take a long time to load data or warm a cache, than during steady-state operation. This cannot be updated. This is a beta feature enabled by the StartupProbe feature flag. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' properties: exec: description: One and only one of the following should be specified. Exec specifies the action to take. properties: command: description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array type: object failureThreshold: description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1. format: int32 type: integer httpGet: description: HTTPGet specifies the http request to perform. properties: host: description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. type: string httpHeaders: description: Custom headers to set in the request. HTTP allows repeated headers. items: description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: description: The header field name type: string value: description: The header field value type: string required: - name - value type: object type: array path: description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: description: Scheme to use for connecting to the host. Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer periodSeconds: description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1. format: int32 type: integer successThreshold: description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. format: int32 type: integer tcpSocket: description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' properties: host: description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object timeoutSeconds: description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer type: object stdin: description: Whether this container should allocate a buffer for stdin in the container runtime. If this is not set, reads from stdin in the container will always result in EOF. Default is false. type: boolean stdinOnce: description: Whether the container runtime should close the stdin channel after it has been opened by a single attach. When stdin is true the stdin stream will remain open across multiple attach sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the first client attaches to stdin, and then remains open and accepts data until the client disconnects, at which time stdin is closed and remains closed until the container is restarted. If this flag is false, a container processes that reads from stdin will never receive an EOF. Default is false type: boolean terminationMessagePath: description: 'Optional: Path at which the file to which the container''s termination message will be written is mounted into the container''s filesystem. Message written is intended to be brief final status, such as an assertion failure message. Will be truncated by the node if greater than 4096 bytes. The total message length across all containers will be limited to 12kb. Defaults to /dev/termination-log. Cannot be updated.' type: string terminationMessagePolicy: description: Indicate how the termination message should be populated. File will use the contents of terminationMessagePath to populate the container status message on both success and failure. FallbackToLogsOnError will use the last chunk of container log output if the termination message file is empty and the container exited with an error. The log output is limited to 2048 bytes or 80 lines, whichever is smaller. Defaults to File. Cannot be updated. type: string tty: description: Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. Default is false. type: boolean volumeDevices: description: volumeDevices is the list of block devices to be used by the container. items: description: volumeDevice describes a mapping of a raw block device within a container. properties: devicePath: description: devicePath is the path inside of the container that the device will be mapped to. type: string name: description: name must match the name of a persistentVolumeClaim in the pod type: string required: - devicePath - name type: object type: array volumeMounts: description: Pod volumes to mount into the container's filesystem. Cannot be updated. items: description: VolumeMount describes a mounting of a Volume within a container. properties: mountPath: description: Path within the container at which the volume should be mounted. Must not contain ':'. type: string mountPropagation: description: mountPropagation determines how mounts are propagated from the host to container and the other way around. When not set, MountPropagationNone is used. This field is beta in 1.10. type: string name: description: This must match the Name of a Volume. type: string readOnly: description: Mounted read-only if true, read-write otherwise (false or unspecified). Defaults to false. type: boolean subPath: description: Path within the volume from which the container's volume should be mounted. Defaults to "" (volume's root). type: string subPathExpr: description: Expanded path within the volume from which the container's volume should be mounted. Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. Defaults to "" (volume's root). SubPathExpr and SubPath are mutually exclusive. type: string required: - mountPath - name type: object type: array workingDir: description: Container's working directory. If not specified, the container runtime's default will be used, which might be configured in the container image. Cannot be updated. type: string required: - name type: object type: array listenLocal: description: ListenLocal makes the Prometheus server listen on loopback, so that it does not bind against the Pod IP. type: boolean logFormat: description: Log format for Prometheus to be configured with. type: string logLevel: description: Log level for Prometheus to be configured with. type: string nodeSelector: additionalProperties: type: string description: Define which Nodes the Pods are scheduled on. type: object overrideHonorLabels: description: OverrideHonorLabels if set to true overrides all user configured honor_labels. If HonorLabels is set in ServiceMonitor or PodMonitor to true, this overrides honor_labels to false. type: boolean overrideHonorTimestamps: description: OverrideHonorTimestamps allows to globally enforce honoring timestamps in all scrape configs. type: boolean paused: description: When a Prometheus deployment is paused, no actions except for deletion will be performed on the underlying objects. type: boolean podMetadata: description: PodMetadata configures Labels and Annotations which are propagated to the prometheus pods. properties: annotations: additionalProperties: type: string description: 'Annotations is an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should be preserved when modifying objects. More info: http://kubernetes.io/docs/user-guide/annotations' type: object labels: additionalProperties: type: string description: 'Map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. More info: http://kubernetes.io/docs/user-guide/labels' type: object name: description: 'Name must be unique within a namespace. Is required when creating resources, although some resources may allow a client to request the generation of an appropriate name automatically. Name is primarily intended for creation idempotence and configuration definition. Cannot be updated. More info: http://kubernetes.io/docs/user-guide/identifiers#names' type: string type: object podMonitorNamespaceSelector: description: Namespace's labels to match for PodMonitor discovery. If nil, only check own namespace. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. items: type: string type: array required: - key - operator type: object type: array matchLabels: additionalProperties: type: string description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object podMonitorSelector: description: '*Experimental* PodMonitors to be selected for target discovery. *Deprecated:* if neither this nor serviceMonitorSelector are specified, configuration is unmanaged.' properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. items: type: string type: array required: - key - operator type: object type: array matchLabels: additionalProperties: type: string description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object portName: description: Port name used for the pods and governing service. This defaults to web type: string priorityClassName: description: Priority class assigned to the Pods type: string probeNamespaceSelector: description: '*Experimental* Namespaces to be selected for Probe discovery. If nil, only check own namespace.' properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. items: type: string type: array required: - key - operator type: object type: array matchLabels: additionalProperties: type: string description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object probeSelector: description: '*Experimental* Probes to be selected for target discovery.' properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. items: type: string type: array required: - key - operator type: object type: array matchLabels: additionalProperties: type: string description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object prometheusExternalLabelName: description: Name of Prometheus external label used to denote Prometheus instance name. Defaults to the value of `prometheus`. External label will _not_ be added when value is set to empty string (`""`). type: string prometheusRulesExcludedFromEnforce: description: PrometheusRulesExcludedFromEnforce - list of prometheus rules to be excluded from enforcing of adding namespace labels. Works only if enforcedNamespaceLabel set to true. Make sure both ruleNamespace and ruleName are set for each pair items: description: PrometheusRuleExcludeConfig enables users to configure excluded PrometheusRule names and their namespaces to be ignored while enforcing namespace label for alerts and metrics. properties: ruleName: description: RuleNamespace - name of excluded rule type: string ruleNamespace: description: RuleNamespace - namespace of excluded rule type: string required: - ruleName - ruleNamespace type: object type: array query: description: QuerySpec defines the query command line flags when starting Prometheus. properties: lookbackDelta: description: The delta difference allowed for retrieving metrics during expression evaluations. type: string maxConcurrency: description: Number of concurrent queries that can be run at once. format: int32 type: integer maxSamples: description: Maximum number of samples a single query can load into memory. Note that queries will fail if they would load more samples than this into memory, so this also limits the number of samples a query can return. format: int32 type: integer timeout: description: Maximum time a query may take before being aborted. type: string type: object queryLogFile: description: QueryLogFile specifies the file to which PromQL queries are logged. Note that this location must be writable, and can be persisted using an attached volume. Alternatively, the location can be set to a stdout location such as `/dev/stdout` to log querie information to the default Prometheus log stream. This is only available in versions of Prometheus >= 2.16.0. For more details, see the Prometheus docs (https://prometheus.io/docs/guides/query-log/) type: string remoteRead: description: If specified, the remote_read spec. This is an experimental feature, it may change in any upcoming release in a breaking way. items: description: RemoteReadSpec defines the remote_read configuration for prometheus. properties: basicAuth: description: BasicAuth for the URL. properties: password: description: The secret in the service monitor namespace that contains the password for authentication. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object username: description: The secret in the service monitor namespace that contains the username for authentication. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object type: object bearerToken: description: Bearer token for remote read. type: string bearerTokenFile: description: File to read bearer token for remote read. type: string name: description: The name of the remote read queue, must be unique if specified. The name is used in metrics and logging in order to differentiate read configurations. Only valid in Prometheus versions 2.15.0 and newer. type: string proxyUrl: description: Optional ProxyURL type: string readRecent: description: Whether reads should be made for queries for time ranges that the local storage should have complete data for. type: boolean remoteTimeout: description: Timeout for requests to the remote read endpoint. type: string requiredMatchers: additionalProperties: type: string description: An optional list of equality matchers which have to be present in a selector to query the remote read endpoint. type: object tlsConfig: description: TLS Config to use for remote read. properties: ca: description: Struct containing the CA cert to use for the targets. properties: configMap: description: ConfigMap containing data to use for the targets. properties: key: description: The key to select. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the ConfigMap or its key must be defined type: boolean required: - key type: object secret: description: Secret containing data to use for the targets. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object type: object caFile: description: Path to the CA cert in the Prometheus container to use for the targets. type: string cert: description: Struct containing the client cert file for the targets. properties: configMap: description: ConfigMap containing data to use for the targets. properties: key: description: The key to select. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the ConfigMap or its key must be defined type: boolean required: - key type: object secret: description: Secret containing data to use for the targets. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object type: object certFile: description: Path to the client cert file in the Prometheus container for the targets. type: string insecureSkipVerify: description: Disable target certificate validation. type: boolean keyFile: description: Path to the client key file in the Prometheus container for the targets. type: string keySecret: description: Secret containing the client key file for the targets. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object serverName: description: Used to verify the hostname for the targets. type: string type: object url: description: The URL of the endpoint to send samples to. type: string required: - url type: object type: array remoteWrite: description: If specified, the remote_write spec. This is an experimental feature, it may change in any upcoming release in a breaking way. items: description: RemoteWriteSpec defines the remote_write configuration for prometheus. properties: basicAuth: description: BasicAuth for the URL. properties: password: description: The secret in the service monitor namespace that contains the password for authentication. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object username: description: The secret in the service monitor namespace that contains the username for authentication. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object type: object bearerToken: description: Bearer token for remote write. type: string bearerTokenFile: description: File to read bearer token for remote write. type: string headers: additionalProperties: type: string description: Custom HTTP headers to be sent along with each remote write request. Be aware that headers that are set by Prometheus itself can't be overwritten. Only valid in Prometheus versions 2.25.0 and newer. type: object metadataConfig: description: MetadataConfig configures the sending of series metadata to remote storage. properties: send: description: Whether metric metadata is sent to remote storage or not. type: boolean sendInterval: description: How frequently metric metadata is sent to remote storage. type: string type: object name: description: The name of the remote write queue, must be unique if specified. The name is used in metrics and logging in order to differentiate queues. Only valid in Prometheus versions 2.15.0 and newer. type: string proxyUrl: description: Optional ProxyURL type: string queueConfig: description: QueueConfig allows tuning of the remote write queue parameters. properties: batchSendDeadline: description: BatchSendDeadline is the maximum time a sample will wait in buffer. type: string capacity: description: Capacity is the number of samples to buffer per shard before we start dropping them. type: integer maxBackoff: description: MaxBackoff is the maximum retry delay. type: string maxRetries: description: MaxRetries is the maximum number of times to retry a batch on recoverable errors. type: integer maxSamplesPerSend: description: MaxSamplesPerSend is the maximum number of samples per send. type: integer maxShards: description: MaxShards is the maximum number of shards, i.e. amount of concurrency. type: integer minBackoff: description: MinBackoff is the initial retry delay. Gets doubled for every retry. type: string minShards: description: MinShards is the minimum number of shards, i.e. amount of concurrency. type: integer type: object remoteTimeout: description: Timeout for requests to the remote write endpoint. type: string tlsConfig: description: TLS Config to use for remote write. properties: ca: description: Struct containing the CA cert to use for the targets. properties: configMap: description: ConfigMap containing data to use for the targets. properties: key: description: The key to select. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the ConfigMap or its key must be defined type: boolean required: - key type: object secret: description: Secret containing data to use for the targets. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object type: object caFile: description: Path to the CA cert in the Prometheus container to use for the targets. type: string cert: description: Struct containing the client cert file for the targets. properties: configMap: description: ConfigMap containing data to use for the targets. properties: key: description: The key to select. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the ConfigMap or its key must be defined type: boolean required: - key type: object secret: description: Secret containing data to use for the targets. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object type: object certFile: description: Path to the client cert file in the Prometheus container for the targets. type: string insecureSkipVerify: description: Disable target certificate validation. type: boolean keyFile: description: Path to the client key file in the Prometheus container for the targets. type: string keySecret: description: Secret containing the client key file for the targets. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object serverName: description: Used to verify the hostname for the targets. type: string type: object url: description: The URL of the endpoint to send samples to. type: string writeRelabelConfigs: description: The list of remote write relabel configurations. items: description: 'RelabelConfig allows dynamic rewriting of the label set, being applied to samples before ingestion. It defines ``-section of Prometheus configuration. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#metric_relabel_configs' properties: action: description: Action to perform based on regex matching. Default is 'replace' type: string modulus: description: Modulus to take of the hash of the source label values. format: int64 type: integer regex: description: Regular expression against which the extracted value is matched. Default is '(.*)' type: string replacement: description: Replacement value against which a regex replace is performed if the regular expression matches. Regex capture groups are available. Default is '$1' type: string separator: description: Separator placed between concatenated source label values. default is ';'. type: string sourceLabels: description: The source labels select values from existing labels. Their content is concatenated using the configured separator and matched against the configured regular expression for the replace, keep, and drop actions. items: type: string type: array targetLabel: description: Label to which the resulting value is written in a replace action. It is mandatory for replace actions. Regex capture groups are available. type: string type: object type: array required: - url type: object type: array replicaExternalLabelName: description: Name of Prometheus external label used to denote replica name. Defaults to the value of `prometheus_replica`. External label will _not_ be added when value is set to empty string (`""`). type: string replicas: description: Number of replicas of each shard to deploy for a Prometheus deployment. Number of replicas multiplied by shards is the total number of Pods created. format: int32 type: integer resources: description: Define resources requests and limits for single Pods. properties: limits: additionalProperties: anyOf: - type: integer - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' type: object requests: additionalProperties: anyOf: - type: integer - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' type: object type: object retention: description: Time duration Prometheus shall retain data for. Default is '24h', and must match the regular expression `[0-9]+(ms|s|m|h|d|w|y)` (milliseconds seconds minutes hours days weeks years). type: string retentionSize: description: 'Maximum amount of disk space used by blocks. Supported units: B, KB, MB, GB, TB, PB, EB. Ex: `512MB`.' type: string routePrefix: description: The route prefix Prometheus registers HTTP handlers for. This is useful, if using ExternalURL and a proxy is rewriting HTTP routes of a request, and the actual ExternalURL is still true, but the server serves requests under a different route prefix. For example for use with `kubectl proxy`. type: string ruleNamespaceSelector: description: Namespaces to be selected for PrometheusRules discovery. If unspecified, only the same namespace as the Prometheus object is in is used. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. items: type: string type: array required: - key - operator type: object type: array matchLabels: additionalProperties: type: string description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object ruleSelector: description: A selector to select which PrometheusRules to mount for loading alerting/recording rules from. Until (excluding) Prometheus Operator v0.24.0 Prometheus Operator will migrate any legacy rule ConfigMaps to PrometheusRule custom resources selected by RuleSelector. Make sure it does not match any config maps that you do not want to be migrated. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. items: type: string type: array required: - key - operator type: object type: array matchLabels: additionalProperties: type: string description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object rules: description: /--rules.*/ command-line arguments. properties: alert: description: /--rules.alert.*/ command-line arguments properties: forGracePeriod: description: Minimum duration between alert and restored 'for' state. This is maintained only for alerts with configured 'for' time greater than grace period. type: string forOutageTolerance: description: Max time to tolerate prometheus outage for restoring 'for' state of alert. type: string resendDelay: description: Minimum amount of time to wait before resending an alert to Alertmanager. type: string type: object type: object scrapeInterval: description: Interval between consecutive scrapes. type: string scrapeTimeout: description: Number of seconds to wait for target to respond before erroring. type: string secrets: description: Secrets is a list of Secrets in the same namespace as the Prometheus object, which shall be mounted into the Prometheus Pods. The Secrets are mounted into /etc/prometheus/secrets/. items: type: string type: array securityContext: description: SecurityContext holds pod-level security attributes and common container settings. This defaults to the default PodSecurityContext. properties: fsGroup: description: "A special supplemental group that applies to all containers in a pod. Some volume types allow the Kubelet to change the ownership of that volume to be owned by the pod: \n 1. The owning GID will be the FSGroup 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) 3. The permission bits are OR'd with rw-rw---- \n If unset, the Kubelet will not modify the ownership and permissions of any volume." format: int64 type: integer fsGroupChangePolicy: description: 'fsGroupChangePolicy defines behavior of changing ownership and permission of the volume before being exposed inside Pod. This field will only apply to volume types which support fsGroup based ownership(and permissions). It will have no effect on ephemeral volume types such as: secret, configmaps and emptydir. Valid values are "OnRootMismatch" and "Always". If not specified defaults to "Always".' type: string runAsGroup: description: The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence for that container. format: int64 type: integer runAsNonRoot: description: Indicates that the container must run as a non-root user. If true, the Kubelet will validate the image at runtime to ensure that it does not run as UID 0 (root) and fail to start the container if it does. If unset or false, no such validation will be performed. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. type: boolean runAsUser: description: The UID to run the entrypoint of the container process. Defaults to user specified in image metadata if unspecified. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence for that container. format: int64 type: integer seLinuxOptions: description: The SELinux context to be applied to all containers. If unspecified, the container runtime will allocate a random SELinux context for each container. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence for that container. properties: level: description: Level is SELinux level label that applies to the container. type: string role: description: Role is a SELinux role label that applies to the container. type: string type: description: Type is a SELinux type label that applies to the container. type: string user: description: User is a SELinux user label that applies to the container. type: string type: object supplementalGroups: description: A list of groups applied to the first process run in each container, in addition to the container's primary GID. If unspecified, no groups will be added to any container. items: format: int64 type: integer type: array sysctls: description: Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported sysctls (by the container runtime) might fail to launch. items: description: Sysctl defines a kernel parameter to be set properties: name: description: Name of a property to set type: string value: description: Value of a property to set type: string required: - name - value type: object type: array windowsOptions: description: The Windows specific settings applied to all containers. If unspecified, the options within a container's SecurityContext will be used. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. properties: gmsaCredentialSpec: description: GMSACredentialSpec is where the GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the GMSA credential spec named by the GMSACredentialSpecName field. type: string gmsaCredentialSpecName: description: GMSACredentialSpecName is the name of the GMSA credential spec to use. type: string runAsUserName: description: The UserName in Windows to run the entrypoint of the container process. Defaults to the user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. type: string type: object type: object serviceAccountName: description: ServiceAccountName is the name of the ServiceAccount to use to run the Prometheus Pods. type: string serviceMonitorNamespaceSelector: description: Namespace's labels to match for ServiceMonitor discovery. If nil, only check own namespace. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. items: type: string type: array required: - key - operator type: object type: array matchLabels: additionalProperties: type: string description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object serviceMonitorSelector: description: ServiceMonitors to be selected for target discovery. *Deprecated:* if neither this nor podMonitorSelector are specified, configuration is unmanaged. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. items: type: string type: array required: - key - operator type: object type: array matchLabels: additionalProperties: type: string description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object sha: description: 'SHA of Prometheus container image to be deployed. Defaults to the value of `version`. Similar to a tag, but the SHA explicitly deploys an immutable container image. Version and Tag are ignored if SHA is set. Deprecated: use ''image'' instead. The image digest can be specified as part of the image URL.' type: string shards: description: 'EXPERIMENTAL: Number of shards to distribute targets onto. Number of replicas multiplied by shards is the total number of Pods created. Note that scaling down shards will not reshard data onto remaining instances, it must be manually moved. Increasing shards will not reshard data either but it will continue to be available from the same instances. To query globally use Thanos sidecar and Thanos querier or remote write data to a central location. Sharding is done on the content of the `__address__` target meta-label.' format: int32 type: integer storage: description: Storage spec to specify how storage shall be used. properties: disableMountSubPath: description: 'Deprecated: subPath usage will be disabled by default in a future release, this option will become unnecessary. DisableMountSubPath allows to remove any subPath usage in volume mounts.' type: boolean emptyDir: description: 'EmptyDirVolumeSource to be used by the Prometheus StatefulSets. If specified, used in place of any volumeClaimTemplate. More info: https://kubernetes.io/docs/concepts/storage/volumes/#emptydir' properties: medium: description: 'What type of storage medium should back this directory. The default is "" which means to use the node''s default medium. Must be an empty string (default) or Memory. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' type: string sizeLimit: anyOf: - type: integer - type: string description: 'Total amount of local storage required for this EmptyDir volume. The size limit is also applicable for memory medium. The maximum usage on memory medium EmptyDir would be the minimum value between the SizeLimit specified here and the sum of memory limits of all containers in a pod. The default is nil which means that the limit is undefined. More info: http://kubernetes.io/docs/user-guide/volumes#emptydir' pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true type: object volumeClaimTemplate: description: A PVC spec to be used by the Prometheus StatefulSets. properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: description: EmbeddedMetadata contains metadata relevant to an EmbeddedResource. properties: annotations: additionalProperties: type: string description: 'Annotations is an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should be preserved when modifying objects. More info: http://kubernetes.io/docs/user-guide/annotations' type: object labels: additionalProperties: type: string description: 'Map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. More info: http://kubernetes.io/docs/user-guide/labels' type: object name: description: 'Name must be unique within a namespace. Is required when creating resources, although some resources may allow a client to request the generation of an appropriate name automatically. Name is primarily intended for creation idempotence and configuration definition. Cannot be updated. More info: http://kubernetes.io/docs/user-guide/identifiers#names' type: string type: object spec: description: 'Spec defines the desired characteristics of a volume requested by a pod author. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' properties: accessModes: description: 'AccessModes contains the desired access modes the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1' items: type: string type: array dataSource: description: 'This field can be used to specify either: * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot - Beta) * An existing PVC (PersistentVolumeClaim) * An existing custom resource/object that implements data population (Alpha) In order to use VolumeSnapshot object types, the appropriate feature gate must be enabled (VolumeSnapshotDataSource or AnyVolumeDataSource) If the provisioner or an external controller can support the specified data source, it will create a new volume based on the contents of the specified data source. If the specified data source is not supported, the volume will not be created and the failure will be reported as an event. In the future, we plan to support more data source types and the behavior of the provisioner may change.' properties: apiGroup: description: APIGroup is the group for the resource being referenced. If APIGroup is not specified, the specified Kind must be in the core API group. For any other third-party types, APIGroup is required. type: string kind: description: Kind is the type of resource being referenced type: string name: description: Name is the name of resource being referenced type: string required: - kind - name type: object resources: description: 'Resources represents the minimum resources the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources' properties: limits: additionalProperties: anyOf: - type: integer - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' type: object requests: additionalProperties: anyOf: - type: integer - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' type: object type: object selector: description: A label query over volumes to consider for binding. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. items: type: string type: array required: - key - operator type: object type: array matchLabels: additionalProperties: type: string description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object storageClassName: description: 'Name of the StorageClass required by the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1' type: string volumeMode: description: volumeMode defines what type of volume is required by the claim. Value of Filesystem is implied when not included in claim spec. type: string volumeName: description: VolumeName is the binding reference to the PersistentVolume backing this claim. type: string type: object status: description: 'Status represents the current information/status of a persistent volume claim. Read-only. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' properties: accessModes: description: 'AccessModes contains the actual access modes the volume backing the PVC has. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1' items: type: string type: array capacity: additionalProperties: anyOf: - type: integer - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true description: Represents the actual resources of the underlying volume. type: object conditions: description: Current Condition of persistent volume claim. If underlying persistent volume is being resized then the Condition will be set to 'ResizeStarted'. items: description: PersistentVolumeClaimCondition contails details about state of pvc properties: lastProbeTime: description: Last time we probed the condition. format: date-time type: string lastTransitionTime: description: Last time the condition transitioned from one status to another. format: date-time type: string message: description: Human-readable message indicating details about last transition. type: string reason: description: Unique, this should be a short, machine understandable string that gives the reason for condition's last transition. If it reports "ResizeStarted" that means the underlying persistent volume is being resized. type: string status: type: string type: description: PersistentVolumeClaimConditionType is a valid value of PersistentVolumeClaimCondition.Type type: string required: - status - type type: object type: array phase: description: Phase represents the current phase of PersistentVolumeClaim. type: string type: object type: object type: object tag: description: 'Tag of Prometheus container image to be deployed. Defaults to the value of `version`. Version is ignored if Tag is set. Deprecated: use ''image'' instead. The image tag can be specified as part of the image URL.' type: string thanos: description: "Thanos configuration allows configuring various aspects of a Prometheus server in a Thanos environment. \n This section is experimental, it may change significantly without deprecation notice in any release. \n This is experimental and may change significantly without backward compatibility in any release." properties: baseImage: description: 'Thanos base image if other than default. Deprecated: use ''image'' instead' type: string grpcServerTlsConfig: description: 'GRPCServerTLSConfig configures the gRPC server from which Thanos Querier reads recorded rule data. Note: Currently only the CAFile, CertFile, and KeyFile fields are supported. Maps to the ''--grpc-server-tls-*'' CLI args.' properties: ca: description: Struct containing the CA cert to use for the targets. properties: configMap: description: ConfigMap containing data to use for the targets. properties: key: description: The key to select. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the ConfigMap or its key must be defined type: boolean required: - key type: object secret: description: Secret containing data to use for the targets. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object type: object caFile: description: Path to the CA cert in the Prometheus container to use for the targets. type: string cert: description: Struct containing the client cert file for the targets. properties: configMap: description: ConfigMap containing data to use for the targets. properties: key: description: The key to select. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the ConfigMap or its key must be defined type: boolean required: - key type: object secret: description: Secret containing data to use for the targets. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object type: object certFile: description: Path to the client cert file in the Prometheus container for the targets. type: string insecureSkipVerify: description: Disable target certificate validation. type: boolean keyFile: description: Path to the client key file in the Prometheus container for the targets. type: string keySecret: description: Secret containing the client key file for the targets. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object serverName: description: Used to verify the hostname for the targets. type: string type: object image: description: Image if specified has precedence over baseImage, tag and sha combinations. Specifying the version is still necessary to ensure the Prometheus Operator knows what version of Thanos is being configured. type: string listenLocal: description: ListenLocal makes the Thanos sidecar listen on loopback, so that it does not bind against the Pod IP. type: boolean logFormat: description: LogFormat for Thanos sidecar to be configured with. type: string logLevel: description: LogLevel for Thanos sidecar to be configured with. type: string minTime: description: MinTime for Thanos sidecar to be configured with. Option can be a constant time in RFC3339 format or time duration relative to current time, such as -1d or 2h45m. Valid duration units are ms, s, m, h, d, w, y. type: string objectStorageConfig: description: ObjectStorageConfig configures object storage in Thanos. Alternative to ObjectStorageConfigFile, and lower order priority. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object objectStorageConfigFile: description: ObjectStorageConfigFile specifies the path of the object storage configuration file. When used alongside with ObjectStorageConfig, ObjectStorageConfigFile takes precedence. type: string resources: description: Resources defines the resource requirements for the Thanos sidecar. If not provided, no requests/limits will be set properties: limits: additionalProperties: anyOf: - type: integer - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' type: object requests: additionalProperties: anyOf: - type: integer - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' type: object type: object sha: description: 'SHA of Thanos container image to be deployed. Defaults to the value of `version`. Similar to a tag, but the SHA explicitly deploys an immutable container image. Version and Tag are ignored if SHA is set. Deprecated: use ''image'' instead. The image digest can be specified as part of the image URL.' type: string tag: description: 'Tag of Thanos sidecar container image to be deployed. Defaults to the value of `version`. Version is ignored if Tag is set. Deprecated: use ''image'' instead. The image tag can be specified as part of the image URL.' type: string tracingConfig: description: TracingConfig configures tracing in Thanos. This is an experimental feature, it may change in any upcoming release in a breaking way. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object tracingConfigFile: description: TracingConfig specifies the path of the tracing configuration file. When used alongside with TracingConfig, TracingConfigFile takes precedence. type: string version: description: Version describes the version of Thanos to use. type: string type: object tolerations: description: If specified, the pod's tolerations. items: description: The pod this Toleration is attached to tolerates any taint that matches the triple using the matching operator . properties: effect: description: Effect indicates the taint effect to match. Empty means match all taint effects. When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. type: string key: description: Key is the taint key that the toleration applies to. Empty means match all taint keys. If the key is empty, operator must be Exists; this combination means to match all values and all keys. type: string operator: description: Operator represents a key's relationship to the value. Valid operators are Exists and Equal. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category. type: string tolerationSeconds: description: TolerationSeconds represents the period of time the toleration (which must be of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, it is not set, which means tolerate the taint forever (do not evict). Zero and negative values will be treated as 0 (evict immediately) by the system. format: int64 type: integer value: description: Value is the taint value the toleration matches to. If the operator is Exists, the value should be empty, otherwise just a regular string. type: string type: object type: array topologySpreadConstraints: description: If specified, the pod's topology spread constraints. items: description: TopologySpreadConstraint specifies how to spread matching pods among the given topology. properties: labelSelector: description: LabelSelector is used to find matching pods. Pods that match this label selector are counted to determine the number of pods in their corresponding topology domain. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. items: type: string type: array required: - key - operator type: object type: array matchLabels: additionalProperties: type: string description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object maxSkew: description: 'MaxSkew describes the degree to which pods may be unevenly distributed. It''s the maximum permitted difference between the number of matching pods in any two topology domains of a given topology type. For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same labelSelector spread as 1/1/0: | zone1 | zone2 | zone3 | | P | P | | - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 1/1/1; scheduling it onto zone1(zone2) would make the ActualSkew(2-0) on zone1(zone2) violate MaxSkew(1). - if MaxSkew is 2, incoming pod can be scheduled onto any zone. It''s a required field. Default value is 1 and 0 is not allowed.' format: int32 type: integer topologyKey: description: TopologyKey is the key of node labels. Nodes that have a label with this key and identical values are considered to be in the same topology. We consider each as a "bucket", and try to put balanced number of pods into each bucket. It's a required field. type: string whenUnsatisfiable: description: 'WhenUnsatisfiable indicates how to deal with a pod if it doesn''t satisfy the spread constraint. - DoNotSchedule (default) tells the scheduler not to schedule it - ScheduleAnyway tells the scheduler to still schedule it It''s considered as "Unsatisfiable" if and only if placing incoming pod on any topology violates "MaxSkew". For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same labelSelector spread as 3/1/1: | zone1 | zone2 | zone3 | | P P P | P | P | If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler won''t make it *more* imbalanced. It''s a required field.' type: string required: - maxSkew - topologyKey - whenUnsatisfiable type: object type: array version: description: Version of Prometheus to be deployed. type: string volumeMounts: description: VolumeMounts allows configuration of additional VolumeMounts on the output StatefulSet definition. VolumeMounts specified will be appended to other VolumeMounts in the prometheus container, that are generated as a result of StorageSpec objects. items: description: VolumeMount describes a mounting of a Volume within a container. properties: mountPath: description: Path within the container at which the volume should be mounted. Must not contain ':'. type: string mountPropagation: description: mountPropagation determines how mounts are propagated from the host to container and the other way around. When not set, MountPropagationNone is used. This field is beta in 1.10. type: string name: description: This must match the Name of a Volume. type: string readOnly: description: Mounted read-only if true, read-write otherwise (false or unspecified). Defaults to false. type: boolean subPath: description: Path within the volume from which the container's volume should be mounted. Defaults to "" (volume's root). type: string subPathExpr: description: Expanded path within the volume from which the container's volume should be mounted. Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. Defaults to "" (volume's root). SubPathExpr and SubPath are mutually exclusive. type: string required: - mountPath - name type: object type: array volumes: description: Volumes allows configuration of additional volumes on the output StatefulSet definition. Volumes specified will be appended to other volumes that are generated as a result of StorageSpec objects. items: description: Volume represents a named volume in a pod that may be accessed by any container in the pod. properties: awsElasticBlockStore: description: 'AWSElasticBlockStore represents an AWS Disk resource that is attached to a kubelet''s host machine and then exposed to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' properties: fsType: description: 'Filesystem type of the volume that you want to mount. Tip: Ensure that the filesystem type is supported by the host operating system. Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore TODO: how do we prevent errors in the filesystem from compromising the machine' type: string partition: description: 'The partition in the volume that you want to mount. If omitted, the default is to mount by volume name. Examples: For volume /dev/sda1, you specify the partition as "1". Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty).' format: int32 type: integer readOnly: description: 'Specify "true" to force and set the ReadOnly property in VolumeMounts to "true". If omitted, the default is "false". More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' type: boolean volumeID: description: 'Unique ID of the persistent disk resource in AWS (Amazon EBS volume). More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' type: string required: - volumeID type: object azureDisk: description: AzureDisk represents an Azure Data Disk mount on the host and bind mount to the pod. properties: cachingMode: description: 'Host Caching mode: None, Read Only, Read Write.' type: string diskName: description: The Name of the data disk in the blob storage type: string diskURI: description: The URI the data disk in the blob storage type: string fsType: description: Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. type: string kind: description: 'Expected values Shared: multiple blob disks per storage account Dedicated: single blob disk per storage account Managed: azure managed data disk (only in managed availability set). defaults to shared' type: string readOnly: description: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts. type: boolean required: - diskName - diskURI type: object azureFile: description: AzureFile represents an Azure File Service mount on the host and bind mount to the pod. properties: readOnly: description: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts. type: boolean secretName: description: the name of secret that contains Azure Storage Account Name and Key type: string shareName: description: Share Name type: string required: - secretName - shareName type: object cephfs: description: CephFS represents a Ceph FS mount on the host that shares a pod's lifetime properties: monitors: description: 'Required: Monitors is a collection of Ceph monitors More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' items: type: string type: array path: description: 'Optional: Used as the mounted root, rather than the full Ceph tree, default is /' type: string readOnly: description: 'Optional: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts. More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' type: boolean secretFile: description: 'Optional: SecretFile is the path to key ring for User, default is /etc/ceph/user.secret More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' type: string secretRef: description: 'Optional: SecretRef is reference to the authentication secret for User, default is empty. More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' properties: name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string type: object user: description: 'Optional: User is the rados user name, default is admin More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' type: string required: - monitors type: object cinder: description: 'Cinder represents a cinder volume attached and mounted on kubelets host machine. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' properties: fsType: description: 'Filesystem type to mount. Must be a filesystem type supported by the host operating system. Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' type: string readOnly: description: 'Optional: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' type: boolean secretRef: description: 'Optional: points to a secret object containing parameters used to connect to OpenStack.' properties: name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string type: object volumeID: description: 'volume id used to identify the volume in cinder. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' type: string required: - volumeID type: object configMap: description: ConfigMap represents a configMap that should populate this volume properties: defaultMode: description: 'Optional: mode bits to use on created files by default. Must be a value between 0 and 0777. Defaults to 0644. Directories within the path are not affected by this setting. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.' format: int32 type: integer items: description: If unspecified, each key-value pair in the Data field of the referenced ConfigMap will be projected into the volume as a file whose name is the key and content is the value. If specified, the listed keys will be projected into the specified paths, and unlisted keys will not be present. If a key is specified which is not present in the ConfigMap, the volume setup will error unless it is marked optional. Paths must be relative and may not contain the '..' path or start with '..'. items: description: Maps a string key to a path within a volume. properties: key: description: The key to project. type: string mode: description: 'Optional: mode bits to use on this file, must be a value between 0 and 0777. If not specified, the volume defaultMode will be used. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.' format: int32 type: integer path: description: The relative path of the file to map the key to. May not be an absolute path. May not contain the path element '..'. May not start with the string '..'. type: string required: - key - path type: object type: array name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the ConfigMap or its keys must be defined type: boolean type: object csi: description: CSI (Container Storage Interface) represents storage that is handled by an external CSI driver (Alpha feature). properties: driver: description: Driver is the name of the CSI driver that handles this volume. Consult with your admin for the correct name as registered in the cluster. type: string fsType: description: Filesystem type to mount. Ex. "ext4", "xfs", "ntfs". If not provided, the empty value is passed to the associated CSI driver which will determine the default filesystem to apply. type: string nodePublishSecretRef: description: NodePublishSecretRef is a reference to the secret object containing sensitive information to pass to the CSI driver to complete the CSI NodePublishVolume and NodeUnpublishVolume calls. This field is optional, and may be empty if no secret is required. If the secret object contains more than one secret, all secret references are passed. properties: name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string type: object readOnly: description: Specifies a read-only configuration for the volume. Defaults to false (read/write). type: boolean volumeAttributes: additionalProperties: type: string description: VolumeAttributes stores driver-specific properties that are passed to the CSI driver. Consult your driver's documentation for supported values. type: object required: - driver type: object downwardAPI: description: DownwardAPI represents downward API about the pod that should populate this volume properties: defaultMode: description: 'Optional: mode bits to use on created files by default. Must be a value between 0 and 0777. Defaults to 0644. Directories within the path are not affected by this setting. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.' format: int32 type: integer items: description: Items is a list of downward API volume file items: description: DownwardAPIVolumeFile represents information to create the file containing the pod field properties: fieldRef: description: 'Required: Selects a field of the pod: only annotations, labels, name and namespace are supported.' properties: apiVersion: description: Version of the schema the FieldPath is written in terms of, defaults to "v1". type: string fieldPath: description: Path of the field to select in the specified API version. type: string required: - fieldPath type: object mode: description: 'Optional: mode bits to use on this file, must be a value between 0 and 0777. If not specified, the volume defaultMode will be used. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.' format: int32 type: integer path: description: 'Required: Path is the relative path name of the file to be created. Must not be absolute or contain the ''..'' path. Must be utf-8 encoded. The first item of the relative path must not start with ''..''' type: string resourceFieldRef: description: 'Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported.' properties: containerName: description: 'Container name: required for volumes, optional for env vars' type: string divisor: anyOf: - type: integer - type: string description: Specifies the output format of the exposed resources, defaults to "1" pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true resource: description: 'Required: resource to select' type: string required: - resource type: object required: - path type: object type: array type: object emptyDir: description: 'EmptyDir represents a temporary directory that shares a pod''s lifetime. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' properties: medium: description: 'What type of storage medium should back this directory. The default is "" which means to use the node''s default medium. Must be an empty string (default) or Memory. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' type: string sizeLimit: anyOf: - type: integer - type: string description: 'Total amount of local storage required for this EmptyDir volume. The size limit is also applicable for memory medium. The maximum usage on memory medium EmptyDir would be the minimum value between the SizeLimit specified here and the sum of memory limits of all containers in a pod. The default is nil which means that the limit is undefined. More info: http://kubernetes.io/docs/user-guide/volumes#emptydir' pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true type: object fc: description: FC represents a Fibre Channel resource that is attached to a kubelet's host machine and then exposed to the pod. properties: fsType: description: 'Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. TODO: how do we prevent errors in the filesystem from compromising the machine' type: string lun: description: 'Optional: FC target lun number' format: int32 type: integer readOnly: description: 'Optional: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts.' type: boolean targetWWNs: description: 'Optional: FC target worldwide names (WWNs)' items: type: string type: array wwids: description: 'Optional: FC volume world wide identifiers (wwids) Either wwids or combination of targetWWNs and lun must be set, but not both simultaneously.' items: type: string type: array type: object flexVolume: description: FlexVolume represents a generic volume resource that is provisioned/attached using an exec based plugin. properties: driver: description: Driver is the name of the driver to use for this volume. type: string fsType: description: Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. "ext4", "xfs", "ntfs". The default filesystem depends on FlexVolume script. type: string options: additionalProperties: type: string description: 'Optional: Extra command options if any.' type: object readOnly: description: 'Optional: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts.' type: boolean secretRef: description: 'Optional: SecretRef is reference to the secret object containing sensitive information to pass to the plugin scripts. This may be empty if no secret object is specified. If the secret object contains more than one secret, all secrets are passed to the plugin scripts.' properties: name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string type: object required: - driver type: object flocker: description: Flocker represents a Flocker volume attached to a kubelet's host machine. This depends on the Flocker control service being running properties: datasetName: description: Name of the dataset stored as metadata -> name on the dataset for Flocker should be considered as deprecated type: string datasetUUID: description: UUID of the dataset. This is unique identifier of a Flocker dataset type: string type: object gcePersistentDisk: description: 'GCEPersistentDisk represents a GCE Disk resource that is attached to a kubelet''s host machine and then exposed to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' properties: fsType: description: 'Filesystem type of the volume that you want to mount. Tip: Ensure that the filesystem type is supported by the host operating system. Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk TODO: how do we prevent errors in the filesystem from compromising the machine' type: string partition: description: 'The partition in the volume that you want to mount. If omitted, the default is to mount by volume name. Examples: For volume /dev/sda1, you specify the partition as "1". Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' format: int32 type: integer pdName: description: 'Unique name of the PD resource in GCE. Used to identify the disk in GCE. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' type: string readOnly: description: 'ReadOnly here will force the ReadOnly setting in VolumeMounts. Defaults to false. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' type: boolean required: - pdName type: object gitRepo: description: 'GitRepo represents a git repository at a particular revision. DEPRECATED: GitRepo is deprecated. To provision a container with a git repo, mount an EmptyDir into an InitContainer that clones the repo using git, then mount the EmptyDir into the Pod''s container.' properties: directory: description: Target directory name. Must not contain or start with '..'. If '.' is supplied, the volume directory will be the git repository. Otherwise, if specified, the volume will contain the git repository in the subdirectory with the given name. type: string repository: description: Repository URL type: string revision: description: Commit hash for the specified revision. type: string required: - repository type: object glusterfs: description: 'Glusterfs represents a Glusterfs mount on the host that shares a pod''s lifetime. More info: https://examples.k8s.io/volumes/glusterfs/README.md' properties: endpoints: description: 'EndpointsName is the endpoint name that details Glusterfs topology. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' type: string path: description: 'Path is the Glusterfs volume path. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' type: string readOnly: description: 'ReadOnly here will force the Glusterfs volume to be mounted with read-only permissions. Defaults to false. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' type: boolean required: - endpoints - path type: object hostPath: description: 'HostPath represents a pre-existing file or directory on the host machine that is directly exposed to the container. This is generally used for system agents or other privileged things that are allowed to see the host machine. Most containers will NOT need this. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath --- TODO(jonesdl) We need to restrict who can use host directory mounts and who can/can not mount host directories as read/write.' properties: path: description: 'Path of the directory on the host. If the path is a symlink, it will follow the link to the real path. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath' type: string type: description: 'Type for HostPath Volume Defaults to "" More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath' type: string required: - path type: object iscsi: description: 'ISCSI represents an ISCSI Disk resource that is attached to a kubelet''s host machine and then exposed to the pod. More info: https://examples.k8s.io/volumes/iscsi/README.md' properties: chapAuthDiscovery: description: whether support iSCSI Discovery CHAP authentication type: boolean chapAuthSession: description: whether support iSCSI Session CHAP authentication type: boolean fsType: description: 'Filesystem type of the volume that you want to mount. Tip: Ensure that the filesystem type is supported by the host operating system. Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi TODO: how do we prevent errors in the filesystem from compromising the machine' type: string initiatorName: description: Custom iSCSI Initiator Name. If initiatorName is specified with iscsiInterface simultaneously, new iSCSI interface : will be created for the connection. type: string iqn: description: Target iSCSI Qualified Name. type: string iscsiInterface: description: iSCSI Interface Name that uses an iSCSI transport. Defaults to 'default' (tcp). type: string lun: description: iSCSI Target Lun number. format: int32 type: integer portals: description: iSCSI Target Portal List. The portal is either an IP or ip_addr:port if the port is other than default (typically TCP ports 860 and 3260). items: type: string type: array readOnly: description: ReadOnly here will force the ReadOnly setting in VolumeMounts. Defaults to false. type: boolean secretRef: description: CHAP Secret for iSCSI target and initiator authentication properties: name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string type: object targetPortal: description: iSCSI Target Portal. The Portal is either an IP or ip_addr:port if the port is other than default (typically TCP ports 860 and 3260). type: string required: - iqn - lun - targetPortal type: object name: description: 'Volume''s name. Must be a DNS_LABEL and unique within the pod. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' type: string nfs: description: 'NFS represents an NFS mount on the host that shares a pod''s lifetime More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' properties: path: description: 'Path that is exported by the NFS server. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' type: string readOnly: description: 'ReadOnly here will force the NFS export to be mounted with read-only permissions. Defaults to false. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' type: boolean server: description: 'Server is the hostname or IP address of the NFS server. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' type: string required: - path - server type: object persistentVolumeClaim: description: 'PersistentVolumeClaimVolumeSource represents a reference to a PersistentVolumeClaim in the same namespace. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' properties: claimName: description: 'ClaimName is the name of a PersistentVolumeClaim in the same namespace as the pod using this volume. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' type: string readOnly: description: Will force the ReadOnly setting in VolumeMounts. Default false. type: boolean required: - claimName type: object photonPersistentDisk: description: PhotonPersistentDisk represents a PhotonController persistent disk attached and mounted on kubelets host machine properties: fsType: description: Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. type: string pdID: description: ID that identifies Photon Controller persistent disk type: string required: - pdID type: object portworxVolume: description: PortworxVolume represents a portworx volume attached and mounted on kubelets host machine properties: fsType: description: FSType represents the filesystem type to mount Must be a filesystem type supported by the host operating system. Ex. "ext4", "xfs". Implicitly inferred to be "ext4" if unspecified. type: string readOnly: description: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts. type: boolean volumeID: description: VolumeID uniquely identifies a Portworx volume type: string required: - volumeID type: object projected: description: Items for all in one resources secrets, configmaps, and downward API properties: defaultMode: description: Mode bits to use on created files by default. Must be a value between 0 and 0777. Directories within the path are not affected by this setting. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set. format: int32 type: integer sources: description: list of volume projections items: description: Projection that may be projected along with other supported volume types properties: configMap: description: information about the configMap data to project properties: items: description: If unspecified, each key-value pair in the Data field of the referenced ConfigMap will be projected into the volume as a file whose name is the key and content is the value. If specified, the listed keys will be projected into the specified paths, and unlisted keys will not be present. If a key is specified which is not present in the ConfigMap, the volume setup will error unless it is marked optional. Paths must be relative and may not contain the '..' path or start with '..'. items: description: Maps a string key to a path within a volume. properties: key: description: The key to project. type: string mode: description: 'Optional: mode bits to use on this file, must be a value between 0 and 0777. If not specified, the volume defaultMode will be used. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.' format: int32 type: integer path: description: The relative path of the file to map the key to. May not be an absolute path. May not contain the path element '..'. May not start with the string '..'. type: string required: - key - path type: object type: array name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the ConfigMap or its keys must be defined type: boolean type: object downwardAPI: description: information about the downwardAPI data to project properties: items: description: Items is a list of DownwardAPIVolume file items: description: DownwardAPIVolumeFile represents information to create the file containing the pod field properties: fieldRef: description: 'Required: Selects a field of the pod: only annotations, labels, name and namespace are supported.' properties: apiVersion: description: Version of the schema the FieldPath is written in terms of, defaults to "v1". type: string fieldPath: description: Path of the field to select in the specified API version. type: string required: - fieldPath type: object mode: description: 'Optional: mode bits to use on this file, must be a value between 0 and 0777. If not specified, the volume defaultMode will be used. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.' format: int32 type: integer path: description: 'Required: Path is the relative path name of the file to be created. Must not be absolute or contain the ''..'' path. Must be utf-8 encoded. The first item of the relative path must not start with ''..''' type: string resourceFieldRef: description: 'Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported.' properties: containerName: description: 'Container name: required for volumes, optional for env vars' type: string divisor: anyOf: - type: integer - type: string description: Specifies the output format of the exposed resources, defaults to "1" pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true resource: description: 'Required: resource to select' type: string required: - resource type: object required: - path type: object type: array type: object secret: description: information about the secret data to project properties: items: description: If unspecified, each key-value pair in the Data field of the referenced Secret will be projected into the volume as a file whose name is the key and content is the value. If specified, the listed keys will be projected into the specified paths, and unlisted keys will not be present. If a key is specified which is not present in the Secret, the volume setup will error unless it is marked optional. Paths must be relative and may not contain the '..' path or start with '..'. items: description: Maps a string key to a path within a volume. properties: key: description: The key to project. type: string mode: description: 'Optional: mode bits to use on this file, must be a value between 0 and 0777. If not specified, the volume defaultMode will be used. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.' format: int32 type: integer path: description: The relative path of the file to map the key to. May not be an absolute path. May not contain the path element '..'. May not start with the string '..'. type: string required: - key - path type: object type: array name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean type: object serviceAccountToken: description: information about the serviceAccountToken data to project properties: audience: description: Audience is the intended audience of the token. A recipient of a token must identify itself with an identifier specified in the audience of the token, and otherwise should reject the token. The audience defaults to the identifier of the apiserver. type: string expirationSeconds: description: ExpirationSeconds is the requested duration of validity of the service account token. As the token approaches expiration, the kubelet volume plugin will proactively rotate the service account token. The kubelet will start trying to rotate the token if the token is older than 80 percent of its time to live or if the token is older than 24 hours.Defaults to 1 hour and must be at least 10 minutes. format: int64 type: integer path: description: Path is the path relative to the mount point of the file to project the token into. type: string required: - path type: object type: object type: array required: - sources type: object quobyte: description: Quobyte represents a Quobyte mount on the host that shares a pod's lifetime properties: group: description: Group to map volume access to Default is no group type: string readOnly: description: ReadOnly here will force the Quobyte volume to be mounted with read-only permissions. Defaults to false. type: boolean registry: description: Registry represents a single or multiple Quobyte Registry services specified as a string as host:port pair (multiple entries are separated with commas) which acts as the central registry for volumes type: string tenant: description: Tenant owning the given Quobyte volume in the Backend Used with dynamically provisioned Quobyte volumes, value is set by the plugin type: string user: description: User to map volume access to Defaults to serivceaccount user type: string volume: description: Volume is a string that references an already created Quobyte volume by name. type: string required: - registry - volume type: object rbd: description: 'RBD represents a Rados Block Device mount on the host that shares a pod''s lifetime. More info: https://examples.k8s.io/volumes/rbd/README.md' properties: fsType: description: 'Filesystem type of the volume that you want to mount. Tip: Ensure that the filesystem type is supported by the host operating system. Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd TODO: how do we prevent errors in the filesystem from compromising the machine' type: string image: description: 'The rados image name. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' type: string keyring: description: 'Keyring is the path to key ring for RBDUser. Default is /etc/ceph/keyring. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' type: string monitors: description: 'A collection of Ceph monitors. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' items: type: string type: array pool: description: 'The rados pool name. Default is rbd. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' type: string readOnly: description: 'ReadOnly here will force the ReadOnly setting in VolumeMounts. Defaults to false. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' type: boolean secretRef: description: 'SecretRef is name of the authentication secret for RBDUser. If provided overrides keyring. Default is nil. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' properties: name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string type: object user: description: 'The rados user name. Default is admin. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' type: string required: - image - monitors type: object scaleIO: description: ScaleIO represents a ScaleIO persistent volume attached and mounted on Kubernetes nodes. properties: fsType: description: Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. "ext4", "xfs", "ntfs". Default is "xfs". type: string gateway: description: The host address of the ScaleIO API Gateway. type: string protectionDomain: description: The name of the ScaleIO Protection Domain for the configured storage. type: string readOnly: description: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts. type: boolean secretRef: description: SecretRef references to the secret for ScaleIO user and other sensitive information. If this is not provided, Login operation will fail. properties: name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string type: object sslEnabled: description: Flag to enable/disable SSL communication with Gateway, default false type: boolean storageMode: description: Indicates whether the storage for a volume should be ThickProvisioned or ThinProvisioned. Default is ThinProvisioned. type: string storagePool: description: The ScaleIO Storage Pool associated with the protection domain. type: string system: description: The name of the storage system as configured in ScaleIO. type: string volumeName: description: The name of a volume already created in the ScaleIO system that is associated with this volume source. type: string required: - gateway - secretRef - system type: object secret: description: 'Secret represents a secret that should populate this volume. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' properties: defaultMode: description: 'Optional: mode bits to use on created files by default. Must be a value between 0 and 0777. Defaults to 0644. Directories within the path are not affected by this setting. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.' format: int32 type: integer items: description: If unspecified, each key-value pair in the Data field of the referenced Secret will be projected into the volume as a file whose name is the key and content is the value. If specified, the listed keys will be projected into the specified paths, and unlisted keys will not be present. If a key is specified which is not present in the Secret, the volume setup will error unless it is marked optional. Paths must be relative and may not contain the '..' path or start with '..'. items: description: Maps a string key to a path within a volume. properties: key: description: The key to project. type: string mode: description: 'Optional: mode bits to use on this file, must be a value between 0 and 0777. If not specified, the volume defaultMode will be used. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.' format: int32 type: integer path: description: The relative path of the file to map the key to. May not be an absolute path. May not contain the path element '..'. May not start with the string '..'. type: string required: - key - path type: object type: array optional: description: Specify whether the Secret or its keys must be defined type: boolean secretName: description: 'Name of the secret in the pod''s namespace to use. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' type: string type: object storageos: description: StorageOS represents a StorageOS volume attached and mounted on Kubernetes nodes. properties: fsType: description: Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. type: string readOnly: description: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts. type: boolean secretRef: description: SecretRef specifies the secret to use for obtaining the StorageOS API credentials. If not specified, default values will be attempted. properties: name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string type: object volumeName: description: VolumeName is the human-readable name of the StorageOS volume. Volume names are only unique within a namespace. type: string volumeNamespace: description: VolumeNamespace specifies the scope of the volume within StorageOS. If no namespace is specified then the Pod's namespace will be used. This allows the Kubernetes name scoping to be mirrored within StorageOS for tighter integration. Set VolumeName to any name to override the default behaviour. Set to "default" if you are not using namespaces within StorageOS. Namespaces that do not pre-exist within StorageOS will be created. type: string type: object vsphereVolume: description: VsphereVolume represents a vSphere volume attached and mounted on kubelets host machine properties: fsType: description: Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. type: string storagePolicyID: description: Storage Policy Based Management (SPBM) profile ID associated with the StoragePolicyName. type: string storagePolicyName: description: Storage Policy Based Management (SPBM) profile name. type: string volumePath: description: Path that identifies vSphere volume vmdk type: string required: - volumePath type: object required: - name type: object type: array walCompression: description: Enable compression of the write-ahead log using Snappy. This flag is only available in versions of Prometheus >= 2.11.0. type: boolean web: description: WebSpec defines the web command line flags when starting Prometheus. properties: pageTitle: description: The prometheus web page title type: string type: object type: object status: description: 'Most recent observed status of the Prometheus cluster. Read-only. Not included when requesting from the apiserver, only from the Prometheus Operator API itself. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#spec-and-status' properties: availableReplicas: description: Total number of available pods (ready for at least minReadySeconds) targeted by this Prometheus deployment. format: int32 type: integer paused: description: Represents whether any actions on the underlying managed objects are being performed. Only delete actions will be performed. type: boolean replicas: description: Total number of non-terminated pods targeted by this Prometheus deployment (their labels match the selector). format: int32 type: integer unavailableReplicas: description: Total number of unavailable pods targeted by this Prometheus deployment. format: int32 type: integer updatedReplicas: description: Total number of non-terminated pods targeted by this Prometheus deployment that have the desired version spec. format: int32 type: integer required: - availableReplicas - paused - replicas - unavailableReplicas - updatedReplicas type: object required: - spec type: object served: true storage: true subresources: {} status: acceptedNames: kind: "" plural: "" conditions: [] storedVersions: [] --- --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.4.1 creationTimestamp: null name: prometheusrules.monitoring.coreos.com spec: group: monitoring.coreos.com names: kind: PrometheusRule listKind: PrometheusRuleList plural: prometheusrules singular: prometheusrule scope: Namespaced versions: - name: v1 schema: openAPIV3Schema: description: PrometheusRule defines recording and alerting rules for a Prometheus instance properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object spec: description: Specification of desired alerting rule definitions for Prometheus. properties: groups: description: Content of Prometheus rule file items: description: 'RuleGroup is a list of sequentially evaluated recording and alerting rules. Note: PartialResponseStrategy is only used by ThanosRuler and will be ignored by Prometheus instances. Valid values for this field are ''warn'' or ''abort''. More info: https://github.com/thanos-io/thanos/blob/master/docs/components/rule.md#partial-response' properties: interval: type: string name: type: string partial_response_strategy: type: string rules: items: description: Rule describes an alerting or recording rule. properties: alert: type: string annotations: additionalProperties: type: string type: object expr: anyOf: - type: integer - type: string x-kubernetes-int-or-string: true for: type: string labels: additionalProperties: type: string type: object record: type: string required: - expr type: object type: array required: - name - rules type: object type: array type: object required: - spec type: object served: true storage: true status: acceptedNames: kind: "" plural: "" conditions: [] storedVersions: [] --- --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.4.1 creationTimestamp: null name: servicemonitors.monitoring.coreos.com spec: group: monitoring.coreos.com names: categories: - prometheus-operator kind: ServiceMonitor listKind: ServiceMonitorList plural: servicemonitors singular: servicemonitor scope: Namespaced versions: - name: v1 schema: openAPIV3Schema: description: ServiceMonitor defines monitoring for a set of services. properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object spec: description: Specification of desired Service selection for target discovery by Prometheus. properties: endpoints: description: A list of endpoints allowed as part of this ServiceMonitor. items: description: Endpoint defines a scrapeable endpoint serving Prometheus metrics. properties: basicAuth: description: 'BasicAuth allow an endpoint to authenticate over basic authentication More info: https://prometheus.io/docs/operating/configuration/#endpoints' properties: password: description: The secret in the service monitor namespace that contains the password for authentication. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object username: description: The secret in the service monitor namespace that contains the username for authentication. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object type: object bearerTokenFile: description: File to read bearer token for scraping targets. type: string bearerTokenSecret: description: Secret to mount to read bearer token for scraping targets. The secret needs to be in the same namespace as the service monitor and accessible by the Prometheus Operator. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object honorLabels: description: HonorLabels chooses the metric's labels on collisions with target labels. type: boolean honorTimestamps: description: HonorTimestamps controls whether Prometheus respects the timestamps present in scraped data. type: boolean interval: description: Interval at which metrics should be scraped type: string metricRelabelings: description: MetricRelabelConfigs to apply to samples before ingestion. items: description: 'RelabelConfig allows dynamic rewriting of the label set, being applied to samples before ingestion. It defines ``-section of Prometheus configuration. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#metric_relabel_configs' properties: action: description: Action to perform based on regex matching. Default is 'replace' type: string modulus: description: Modulus to take of the hash of the source label values. format: int64 type: integer regex: description: Regular expression against which the extracted value is matched. Default is '(.*)' type: string replacement: description: Replacement value against which a regex replace is performed if the regular expression matches. Regex capture groups are available. Default is '$1' type: string separator: description: Separator placed between concatenated source label values. default is ';'. type: string sourceLabels: description: The source labels select values from existing labels. Their content is concatenated using the configured separator and matched against the configured regular expression for the replace, keep, and drop actions. items: type: string type: array targetLabel: description: Label to which the resulting value is written in a replace action. It is mandatory for replace actions. Regex capture groups are available. type: string type: object type: array params: additionalProperties: items: type: string type: array description: Optional HTTP URL parameters type: object path: description: HTTP path to scrape for metrics. type: string port: description: Name of the service port this endpoint refers to. Mutually exclusive with targetPort. type: string proxyUrl: description: ProxyURL eg http://proxyserver:2195 Directs scrapes to proxy through this endpoint. type: string relabelings: description: 'RelabelConfigs to apply to samples before scraping. Prometheus Operator automatically adds relabelings for a few standard Kubernetes fields and replaces original scrape job name with __tmp_prometheus_job_name. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config' items: description: 'RelabelConfig allows dynamic rewriting of the label set, being applied to samples before ingestion. It defines ``-section of Prometheus configuration. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#metric_relabel_configs' properties: action: description: Action to perform based on regex matching. Default is 'replace' type: string modulus: description: Modulus to take of the hash of the source label values. format: int64 type: integer regex: description: Regular expression against which the extracted value is matched. Default is '(.*)' type: string replacement: description: Replacement value against which a regex replace is performed if the regular expression matches. Regex capture groups are available. Default is '$1' type: string separator: description: Separator placed between concatenated source label values. default is ';'. type: string sourceLabels: description: The source labels select values from existing labels. Their content is concatenated using the configured separator and matched against the configured regular expression for the replace, keep, and drop actions. items: type: string type: array targetLabel: description: Label to which the resulting value is written in a replace action. It is mandatory for replace actions. Regex capture groups are available. type: string type: object type: array scheme: description: HTTP scheme to use for scraping. type: string scrapeTimeout: description: Timeout after which the scrape is ended type: string targetPort: anyOf: - type: integer - type: string description: Name or number of the target port of the Pod behind the Service, the port must be specified with container port property. Mutually exclusive with port. x-kubernetes-int-or-string: true tlsConfig: description: TLS configuration to use when scraping the endpoint properties: ca: description: Struct containing the CA cert to use for the targets. properties: configMap: description: ConfigMap containing data to use for the targets. properties: key: description: The key to select. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the ConfigMap or its key must be defined type: boolean required: - key type: object secret: description: Secret containing data to use for the targets. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object type: object caFile: description: Path to the CA cert in the Prometheus container to use for the targets. type: string cert: description: Struct containing the client cert file for the targets. properties: configMap: description: ConfigMap containing data to use for the targets. properties: key: description: The key to select. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the ConfigMap or its key must be defined type: boolean required: - key type: object secret: description: Secret containing data to use for the targets. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object type: object certFile: description: Path to the client cert file in the Prometheus container for the targets. type: string insecureSkipVerify: description: Disable target certificate validation. type: boolean keyFile: description: Path to the client key file in the Prometheus container for the targets. type: string keySecret: description: Secret containing the client key file for the targets. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object serverName: description: Used to verify the hostname for the targets. type: string type: object type: object type: array jobLabel: description: The label to use to retrieve the job name from. type: string namespaceSelector: description: Selector to select which namespaces the Endpoints objects are discovered from. properties: any: description: Boolean describing whether all namespaces are selected in contrast to a list restricting them. type: boolean matchNames: description: List of namespace names. items: type: string type: array type: object podTargetLabels: description: PodTargetLabels transfers labels on the Kubernetes Pod onto the target. items: type: string type: array sampleLimit: description: SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. format: int64 type: integer selector: description: Selector to select Endpoints objects. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. items: type: string type: array required: - key - operator type: object type: array matchLabels: additionalProperties: type: string description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object targetLabels: description: TargetLabels transfers labels on the Kubernetes Service onto the target. items: type: string type: array targetLimit: description: TargetLimit defines a limit on the number of scraped targets that will be accepted. format: int64 type: integer required: - endpoints - selector type: object required: - spec type: object served: true storage: true status: acceptedNames: kind: "" plural: "" conditions: [] storedVersions: [] --- --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.4.1 creationTimestamp: null name: thanosrulers.monitoring.coreos.com spec: group: monitoring.coreos.com names: categories: - prometheus-operator kind: ThanosRuler listKind: ThanosRulerList plural: thanosrulers singular: thanosruler scope: Namespaced versions: - name: v1 schema: openAPIV3Schema: description: ThanosRuler defines a ThanosRuler deployment. properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object spec: description: 'Specification of the desired behavior of the ThanosRuler cluster. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#spec-and-status' properties: affinity: description: If specified, the pod's scheduling constraints. properties: nodeAffinity: description: Describes node affinity scheduling rules for the pod. properties: preferredDuringSchedulingIgnoredDuringExecution: description: The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding "weight" to the sum if the node matches the corresponding matchExpressions; the node(s) with the highest sum are the most preferred. items: description: An empty preferred scheduling term matches all objects with implicit weight 0 (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). properties: preference: description: A node selector term, associated with the corresponding weight. properties: matchExpressions: description: A list of node selector requirements by node's labels. items: description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: The label key that the selector applies to. type: string operator: description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. type: string values: description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. items: type: string type: array required: - key - operator type: object type: array matchFields: description: A list of node selector requirements by node's fields. items: description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: The label key that the selector applies to. type: string operator: description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. type: string values: description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. items: type: string type: array required: - key - operator type: object type: array type: object weight: description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100. format: int32 type: integer required: - preference - weight type: object type: array requiredDuringSchedulingIgnoredDuringExecution: description: If the affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to an update), the system may or may not try to eventually evict the pod from its node. properties: nodeSelectorTerms: description: Required. A list of node selector terms. The terms are ORed. items: description: A null or empty node selector term matches no objects. The requirements of them are ANDed. The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. properties: matchExpressions: description: A list of node selector requirements by node's labels. items: description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: The label key that the selector applies to. type: string operator: description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. type: string values: description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. items: type: string type: array required: - key - operator type: object type: array matchFields: description: A list of node selector requirements by node's fields. items: description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: The label key that the selector applies to. type: string operator: description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. type: string values: description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. items: type: string type: array required: - key - operator type: object type: array type: object type: array required: - nodeSelectorTerms type: object type: object podAffinity: description: Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)). properties: preferredDuringSchedulingIgnoredDuringExecution: description: The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. items: description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) properties: podAffinityTerm: description: Required. A pod affinity term, associated with the corresponding weight. properties: labelSelector: description: A label query over a set of resources, in this case pods. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. items: type: string type: array required: - key - operator type: object type: array matchLabels: additionalProperties: type: string description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object namespaces: description: namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means "this pod's namespace" items: type: string type: array topologyKey: description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. type: string required: - topologyKey type: object weight: description: weight associated with matching the corresponding podAffinityTerm, in the range 1-100. format: int32 type: integer required: - podAffinityTerm - weight type: object type: array requiredDuringSchedulingIgnoredDuringExecution: description: If the affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied. items: description: Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key matches that of any node on which a pod of the set of pods is running properties: labelSelector: description: A label query over a set of resources, in this case pods. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. items: type: string type: array required: - key - operator type: object type: array matchLabels: additionalProperties: type: string description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object namespaces: description: namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means "this pod's namespace" items: type: string type: array topologyKey: description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. type: string required: - topologyKey type: object type: array type: object podAntiAffinity: description: Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)). properties: preferredDuringSchedulingIgnoredDuringExecution: description: The scheduler will prefer to schedule pods to nodes that satisfy the anti-affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling anti-affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. items: description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) properties: podAffinityTerm: description: Required. A pod affinity term, associated with the corresponding weight. properties: labelSelector: description: A label query over a set of resources, in this case pods. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. items: type: string type: array required: - key - operator type: object type: array matchLabels: additionalProperties: type: string description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object namespaces: description: namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means "this pod's namespace" items: type: string type: array topologyKey: description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. type: string required: - topologyKey type: object weight: description: weight associated with matching the corresponding podAffinityTerm, in the range 1-100. format: int32 type: integer required: - podAffinityTerm - weight type: object type: array requiredDuringSchedulingIgnoredDuringExecution: description: If the anti-affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the anti-affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied. items: description: Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key matches that of any node on which a pod of the set of pods is running properties: labelSelector: description: A label query over a set of resources, in this case pods. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. items: type: string type: array required: - key - operator type: object type: array matchLabels: additionalProperties: type: string description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object namespaces: description: namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means "this pod's namespace" items: type: string type: array topologyKey: description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. type: string required: - topologyKey type: object type: array type: object type: object alertDropLabels: description: AlertDropLabels configure the label names which should be dropped in ThanosRuler alerts. If `labels` field is not provided, `thanos_ruler_replica` will be dropped in alerts by default. items: type: string type: array alertQueryUrl: description: The external Query URL the Thanos Ruler will set in the 'Source' field of all alerts. Maps to the '--alert.query-url' CLI arg. type: string alertmanagersConfig: description: Define configuration for connecting to alertmanager. Only available with thanos v0.10.0 and higher. Maps to the `alertmanagers.config` arg. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object alertmanagersUrl: description: 'Define URLs to send alerts to Alertmanager. For Thanos v0.10.0 and higher, AlertManagersConfig should be used instead. Note: this field will be ignored if AlertManagersConfig is specified. Maps to the `alertmanagers.url` arg.' items: type: string type: array containers: description: 'Containers allows injecting additional containers or modifying operator generated containers. This can be used to allow adding an authentication proxy to a ThanosRuler pod or to change the behavior of an operator generated container. Containers described here modify an operator generated container if they share the same name and modifications are done via a strategic merge patch. The current container names are: `thanos-ruler` and `config-reloader`. Overriding containers is entirely outside the scope of what the maintainers will support and by doing so, you accept that this behaviour may break at any time without notice.' items: description: A single application container that you want to run within a pod. properties: args: description: 'Arguments to the entrypoint. The docker image''s CMD is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container''s environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' items: type: string type: array command: description: 'Entrypoint array. Not executed within a shell. The docker image''s ENTRYPOINT is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container''s environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' items: type: string type: array env: description: List of environment variables to set in the container. Cannot be updated. items: description: EnvVar represents an environment variable present in a Container. properties: name: description: Name of the environment variable. Must be a C_IDENTIFIER. type: string value: description: 'Variable references $(VAR_NAME) are expanded using the previous defined environment variables in the container and any service environment variables. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Defaults to "".' type: string valueFrom: description: Source for the environment variable's value. Cannot be used if value is not empty. properties: configMapKeyRef: description: Selects a key of a ConfigMap. properties: key: description: The key to select. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the ConfigMap or its key must be defined type: boolean required: - key type: object fieldRef: description: 'Selects a field of the pod: supports metadata.name, metadata.namespace, metadata.labels, metadata.annotations, spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs.' properties: apiVersion: description: Version of the schema the FieldPath is written in terms of, defaults to "v1". type: string fieldPath: description: Path of the field to select in the specified API version. type: string required: - fieldPath type: object resourceFieldRef: description: 'Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported.' properties: containerName: description: 'Container name: required for volumes, optional for env vars' type: string divisor: anyOf: - type: integer - type: string description: Specifies the output format of the exposed resources, defaults to "1" pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true resource: description: 'Required: resource to select' type: string required: - resource type: object secretKeyRef: description: Selects a key of a secret in the pod's namespace properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object type: object required: - name type: object type: array envFrom: description: List of sources to populate environment variables in the container. The keys defined within a source must be a C_IDENTIFIER. All invalid keys will be reported as an event when the container is starting. When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated. items: description: EnvFromSource represents the source of a set of ConfigMaps properties: configMapRef: description: The ConfigMap to select from properties: name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the ConfigMap must be defined type: boolean type: object prefix: description: An optional identifier to prepend to each key in the ConfigMap. Must be a C_IDENTIFIER. type: string secretRef: description: The Secret to select from properties: name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret must be defined type: boolean type: object type: object type: array image: description: 'Docker image name. More info: https://kubernetes.io/docs/concepts/containers/images This field is optional to allow higher level config management to default or override container images in workload controllers like Deployments and StatefulSets.' type: string imagePullPolicy: description: 'Image pull policy. One of Always, Never, IfNotPresent. Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. Cannot be updated. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images' type: string lifecycle: description: Actions that the management system should take in response to container lifecycle events. Cannot be updated. properties: postStart: description: 'PostStart is called immediately after a container is created. If the handler fails, the container is terminated and restarted according to its restart policy. Other management of the container blocks until the hook completes. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' properties: exec: description: One and only one of the following should be specified. Exec specifies the action to take. properties: command: description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array type: object httpGet: description: HTTPGet specifies the http request to perform. properties: host: description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. type: string httpHeaders: description: Custom headers to set in the request. HTTP allows repeated headers. items: description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: description: The header field name type: string value: description: The header field value type: string required: - name - value type: object type: array path: description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: description: Scheme to use for connecting to the host. Defaults to HTTP. type: string required: - port type: object tcpSocket: description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' properties: host: description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object type: object preStop: description: 'PreStop is called immediately before a container is terminated due to an API request or management event such as liveness/startup probe failure, preemption, resource contention, etc. The handler is not called if the container crashes or exits. The reason for termination is passed to the handler. The Pod''s termination grace period countdown begins before the PreStop hooked is executed. Regardless of the outcome of the handler, the container will eventually terminate within the Pod''s termination grace period. Other management of the container blocks until the hook completes or until the termination grace period is reached. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' properties: exec: description: One and only one of the following should be specified. Exec specifies the action to take. properties: command: description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array type: object httpGet: description: HTTPGet specifies the http request to perform. properties: host: description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. type: string httpHeaders: description: Custom headers to set in the request. HTTP allows repeated headers. items: description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: description: The header field name type: string value: description: The header field value type: string required: - name - value type: object type: array path: description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: description: Scheme to use for connecting to the host. Defaults to HTTP. type: string required: - port type: object tcpSocket: description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' properties: host: description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object type: object type: object livenessProbe: description: 'Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' properties: exec: description: One and only one of the following should be specified. Exec specifies the action to take. properties: command: description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array type: object failureThreshold: description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1. format: int32 type: integer httpGet: description: HTTPGet specifies the http request to perform. properties: host: description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. type: string httpHeaders: description: Custom headers to set in the request. HTTP allows repeated headers. items: description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: description: The header field name type: string value: description: The header field value type: string required: - name - value type: object type: array path: description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: description: Scheme to use for connecting to the host. Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer periodSeconds: description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1. format: int32 type: integer successThreshold: description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. format: int32 type: integer tcpSocket: description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' properties: host: description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object timeoutSeconds: description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer type: object name: description: Name of the container specified as a DNS_LABEL. Each container in a pod must have a unique name (DNS_LABEL). Cannot be updated. type: string ports: description: List of ports to expose from the container. Exposing a port here gives the system additional information about the network connections a container uses, but is primarily informational. Not specifying a port here DOES NOT prevent that port from being exposed. Any port which is listening on the default "0.0.0.0" address inside a container will be accessible from the network. Cannot be updated. items: description: ContainerPort represents a network port in a single container. properties: containerPort: description: Number of port to expose on the pod's IP address. This must be a valid port number, 0 < x < 65536. format: int32 type: integer hostIP: description: What host IP to bind the external port to. type: string hostPort: description: Number of port to expose on the host. If specified, this must be a valid port number, 0 < x < 65536. If HostNetwork is specified, this must match ContainerPort. Most containers do not need this. format: int32 type: integer name: description: If specified, this must be an IANA_SVC_NAME and unique within the pod. Each named port in a pod must have a unique name. Name for the port that can be referred to by services. type: string protocol: default: TCP description: Protocol for port. Must be UDP, TCP, or SCTP. Defaults to "TCP". type: string required: - containerPort type: object type: array x-kubernetes-list-map-keys: - containerPort - protocol x-kubernetes-list-type: map readinessProbe: description: 'Periodic probe of container service readiness. Container will be removed from service endpoints if the probe fails. Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' properties: exec: description: One and only one of the following should be specified. Exec specifies the action to take. properties: command: description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array type: object failureThreshold: description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1. format: int32 type: integer httpGet: description: HTTPGet specifies the http request to perform. properties: host: description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. type: string httpHeaders: description: Custom headers to set in the request. HTTP allows repeated headers. items: description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: description: The header field name type: string value: description: The header field value type: string required: - name - value type: object type: array path: description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: description: Scheme to use for connecting to the host. Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer periodSeconds: description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1. format: int32 type: integer successThreshold: description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. format: int32 type: integer tcpSocket: description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' properties: host: description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object timeoutSeconds: description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer type: object resources: description: 'Compute Resources required by this container. Cannot be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' properties: limits: additionalProperties: anyOf: - type: integer - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' type: object requests: additionalProperties: anyOf: - type: integer - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' type: object type: object securityContext: description: 'Security options the pod should run with. More info: https://kubernetes.io/docs/concepts/policy/security-context/ More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/' properties: allowPrivilegeEscalation: description: 'AllowPrivilegeEscalation controls whether a process can gain more privileges than its parent process. This bool directly controls if the no_new_privs flag will be set on the container process. AllowPrivilegeEscalation is true always when the container is: 1) run as Privileged 2) has CAP_SYS_ADMIN' type: boolean capabilities: description: The capabilities to add/drop when running containers. Defaults to the default set of capabilities granted by the container runtime. properties: add: description: Added capabilities items: description: Capability represent POSIX capabilities type type: string type: array drop: description: Removed capabilities items: description: Capability represent POSIX capabilities type type: string type: array type: object privileged: description: Run container in privileged mode. Processes in privileged containers are essentially equivalent to root on the host. Defaults to false. type: boolean procMount: description: procMount denotes the type of proc mount to use for the containers. The default is DefaultProcMount which uses the container runtime defaults for readonly paths and masked paths. This requires the ProcMountType feature flag to be enabled. type: string readOnlyRootFilesystem: description: Whether this container has a read-only root filesystem. Default is false. type: boolean runAsGroup: description: The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. format: int64 type: integer runAsNonRoot: description: Indicates that the container must run as a non-root user. If true, the Kubelet will validate the image at runtime to ensure that it does not run as UID 0 (root) and fail to start the container if it does. If unset or false, no such validation will be performed. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. type: boolean runAsUser: description: The UID to run the entrypoint of the container process. Defaults to user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. format: int64 type: integer seLinuxOptions: description: The SELinux context to be applied to the container. If unspecified, the container runtime will allocate a random SELinux context for each container. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. properties: level: description: Level is SELinux level label that applies to the container. type: string role: description: Role is a SELinux role label that applies to the container. type: string type: description: Type is a SELinux type label that applies to the container. type: string user: description: User is a SELinux user label that applies to the container. type: string type: object windowsOptions: description: The Windows specific settings applied to all containers. If unspecified, the options from the PodSecurityContext will be used. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. properties: gmsaCredentialSpec: description: GMSACredentialSpec is where the GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the GMSA credential spec named by the GMSACredentialSpecName field. type: string gmsaCredentialSpecName: description: GMSACredentialSpecName is the name of the GMSA credential spec to use. type: string runAsUserName: description: The UserName in Windows to run the entrypoint of the container process. Defaults to the user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. type: string type: object type: object startupProbe: description: 'StartupProbe indicates that the Pod has successfully initialized. If specified, no other probes are executed until this completes successfully. If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. This can be used to provide different probe parameters at the beginning of a Pod''s lifecycle, when it might take a long time to load data or warm a cache, than during steady-state operation. This cannot be updated. This is a beta feature enabled by the StartupProbe feature flag. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' properties: exec: description: One and only one of the following should be specified. Exec specifies the action to take. properties: command: description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array type: object failureThreshold: description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1. format: int32 type: integer httpGet: description: HTTPGet specifies the http request to perform. properties: host: description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. type: string httpHeaders: description: Custom headers to set in the request. HTTP allows repeated headers. items: description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: description: The header field name type: string value: description: The header field value type: string required: - name - value type: object type: array path: description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: description: Scheme to use for connecting to the host. Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer periodSeconds: description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1. format: int32 type: integer successThreshold: description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. format: int32 type: integer tcpSocket: description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' properties: host: description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object timeoutSeconds: description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer type: object stdin: description: Whether this container should allocate a buffer for stdin in the container runtime. If this is not set, reads from stdin in the container will always result in EOF. Default is false. type: boolean stdinOnce: description: Whether the container runtime should close the stdin channel after it has been opened by a single attach. When stdin is true the stdin stream will remain open across multiple attach sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the first client attaches to stdin, and then remains open and accepts data until the client disconnects, at which time stdin is closed and remains closed until the container is restarted. If this flag is false, a container processes that reads from stdin will never receive an EOF. Default is false type: boolean terminationMessagePath: description: 'Optional: Path at which the file to which the container''s termination message will be written is mounted into the container''s filesystem. Message written is intended to be brief final status, such as an assertion failure message. Will be truncated by the node if greater than 4096 bytes. The total message length across all containers will be limited to 12kb. Defaults to /dev/termination-log. Cannot be updated.' type: string terminationMessagePolicy: description: Indicate how the termination message should be populated. File will use the contents of terminationMessagePath to populate the container status message on both success and failure. FallbackToLogsOnError will use the last chunk of container log output if the termination message file is empty and the container exited with an error. The log output is limited to 2048 bytes or 80 lines, whichever is smaller. Defaults to File. Cannot be updated. type: string tty: description: Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. Default is false. type: boolean volumeDevices: description: volumeDevices is the list of block devices to be used by the container. items: description: volumeDevice describes a mapping of a raw block device within a container. properties: devicePath: description: devicePath is the path inside of the container that the device will be mapped to. type: string name: description: name must match the name of a persistentVolumeClaim in the pod type: string required: - devicePath - name type: object type: array volumeMounts: description: Pod volumes to mount into the container's filesystem. Cannot be updated. items: description: VolumeMount describes a mounting of a Volume within a container. properties: mountPath: description: Path within the container at which the volume should be mounted. Must not contain ':'. type: string mountPropagation: description: mountPropagation determines how mounts are propagated from the host to container and the other way around. When not set, MountPropagationNone is used. This field is beta in 1.10. type: string name: description: This must match the Name of a Volume. type: string readOnly: description: Mounted read-only if true, read-write otherwise (false or unspecified). Defaults to false. type: boolean subPath: description: Path within the volume from which the container's volume should be mounted. Defaults to "" (volume's root). type: string subPathExpr: description: Expanded path within the volume from which the container's volume should be mounted. Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. Defaults to "" (volume's root). SubPathExpr and SubPath are mutually exclusive. type: string required: - mountPath - name type: object type: array workingDir: description: Container's working directory. If not specified, the container runtime's default will be used, which might be configured in the container image. Cannot be updated. type: string required: - name type: object type: array enforcedNamespaceLabel: description: EnforcedNamespaceLabel enforces adding a namespace label of origin for each alert and metric that is user created. The label value will always be the namespace of the object that is being created. type: string evaluationInterval: description: Interval between consecutive evaluations. type: string externalPrefix: description: The external URL the Thanos Ruler instances will be available under. This is necessary to generate correct URLs. This is necessary if Thanos Ruler is not served from root of a DNS name. type: string grpcServerTlsConfig: description: 'GRPCServerTLSConfig configures the gRPC server from which Thanos Querier reads recorded rule data. Note: Currently only the CAFile, CertFile, and KeyFile fields are supported. Maps to the ''--grpc-server-tls-*'' CLI args.' properties: ca: description: Struct containing the CA cert to use for the targets. properties: configMap: description: ConfigMap containing data to use for the targets. properties: key: description: The key to select. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the ConfigMap or its key must be defined type: boolean required: - key type: object secret: description: Secret containing data to use for the targets. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object type: object caFile: description: Path to the CA cert in the Prometheus container to use for the targets. type: string cert: description: Struct containing the client cert file for the targets. properties: configMap: description: ConfigMap containing data to use for the targets. properties: key: description: The key to select. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the ConfigMap or its key must be defined type: boolean required: - key type: object secret: description: Secret containing data to use for the targets. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object type: object certFile: description: Path to the client cert file in the Prometheus container for the targets. type: string insecureSkipVerify: description: Disable target certificate validation. type: boolean keyFile: description: Path to the client key file in the Prometheus container for the targets. type: string keySecret: description: Secret containing the client key file for the targets. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object serverName: description: Used to verify the hostname for the targets. type: string type: object image: description: Thanos container image URL. type: string imagePullSecrets: description: An optional list of references to secrets in the same namespace to use for pulling thanos images from registries see http://kubernetes.io/docs/user-guide/images#specifying-imagepullsecrets-on-a-pod items: description: LocalObjectReference contains enough information to let you locate the referenced object inside the same namespace. properties: name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string type: object type: array initContainers: description: 'InitContainers allows adding initContainers to the pod definition. Those can be used to e.g. fetch secrets for injection into the ThanosRuler configuration from external sources. Any errors during the execution of an initContainer will lead to a restart of the Pod. More info: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ Using initContainers for any use case other then secret fetching is entirely outside the scope of what the maintainers will support and by doing so, you accept that this behaviour may break at any time without notice.' items: description: A single application container that you want to run within a pod. properties: args: description: 'Arguments to the entrypoint. The docker image''s CMD is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container''s environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' items: type: string type: array command: description: 'Entrypoint array. Not executed within a shell. The docker image''s ENTRYPOINT is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container''s environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' items: type: string type: array env: description: List of environment variables to set in the container. Cannot be updated. items: description: EnvVar represents an environment variable present in a Container. properties: name: description: Name of the environment variable. Must be a C_IDENTIFIER. type: string value: description: 'Variable references $(VAR_NAME) are expanded using the previous defined environment variables in the container and any service environment variables. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Defaults to "".' type: string valueFrom: description: Source for the environment variable's value. Cannot be used if value is not empty. properties: configMapKeyRef: description: Selects a key of a ConfigMap. properties: key: description: The key to select. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the ConfigMap or its key must be defined type: boolean required: - key type: object fieldRef: description: 'Selects a field of the pod: supports metadata.name, metadata.namespace, metadata.labels, metadata.annotations, spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs.' properties: apiVersion: description: Version of the schema the FieldPath is written in terms of, defaults to "v1". type: string fieldPath: description: Path of the field to select in the specified API version. type: string required: - fieldPath type: object resourceFieldRef: description: 'Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported.' properties: containerName: description: 'Container name: required for volumes, optional for env vars' type: string divisor: anyOf: - type: integer - type: string description: Specifies the output format of the exposed resources, defaults to "1" pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true resource: description: 'Required: resource to select' type: string required: - resource type: object secretKeyRef: description: Selects a key of a secret in the pod's namespace properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object type: object required: - name type: object type: array envFrom: description: List of sources to populate environment variables in the container. The keys defined within a source must be a C_IDENTIFIER. All invalid keys will be reported as an event when the container is starting. When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated. items: description: EnvFromSource represents the source of a set of ConfigMaps properties: configMapRef: description: The ConfigMap to select from properties: name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the ConfigMap must be defined type: boolean type: object prefix: description: An optional identifier to prepend to each key in the ConfigMap. Must be a C_IDENTIFIER. type: string secretRef: description: The Secret to select from properties: name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret must be defined type: boolean type: object type: object type: array image: description: 'Docker image name. More info: https://kubernetes.io/docs/concepts/containers/images This field is optional to allow higher level config management to default or override container images in workload controllers like Deployments and StatefulSets.' type: string imagePullPolicy: description: 'Image pull policy. One of Always, Never, IfNotPresent. Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. Cannot be updated. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images' type: string lifecycle: description: Actions that the management system should take in response to container lifecycle events. Cannot be updated. properties: postStart: description: 'PostStart is called immediately after a container is created. If the handler fails, the container is terminated and restarted according to its restart policy. Other management of the container blocks until the hook completes. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' properties: exec: description: One and only one of the following should be specified. Exec specifies the action to take. properties: command: description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array type: object httpGet: description: HTTPGet specifies the http request to perform. properties: host: description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. type: string httpHeaders: description: Custom headers to set in the request. HTTP allows repeated headers. items: description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: description: The header field name type: string value: description: The header field value type: string required: - name - value type: object type: array path: description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: description: Scheme to use for connecting to the host. Defaults to HTTP. type: string required: - port type: object tcpSocket: description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' properties: host: description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object type: object preStop: description: 'PreStop is called immediately before a container is terminated due to an API request or management event such as liveness/startup probe failure, preemption, resource contention, etc. The handler is not called if the container crashes or exits. The reason for termination is passed to the handler. The Pod''s termination grace period countdown begins before the PreStop hooked is executed. Regardless of the outcome of the handler, the container will eventually terminate within the Pod''s termination grace period. Other management of the container blocks until the hook completes or until the termination grace period is reached. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' properties: exec: description: One and only one of the following should be specified. Exec specifies the action to take. properties: command: description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array type: object httpGet: description: HTTPGet specifies the http request to perform. properties: host: description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. type: string httpHeaders: description: Custom headers to set in the request. HTTP allows repeated headers. items: description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: description: The header field name type: string value: description: The header field value type: string required: - name - value type: object type: array path: description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: description: Scheme to use for connecting to the host. Defaults to HTTP. type: string required: - port type: object tcpSocket: description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' properties: host: description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object type: object type: object livenessProbe: description: 'Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' properties: exec: description: One and only one of the following should be specified. Exec specifies the action to take. properties: command: description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array type: object failureThreshold: description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1. format: int32 type: integer httpGet: description: HTTPGet specifies the http request to perform. properties: host: description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. type: string httpHeaders: description: Custom headers to set in the request. HTTP allows repeated headers. items: description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: description: The header field name type: string value: description: The header field value type: string required: - name - value type: object type: array path: description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: description: Scheme to use for connecting to the host. Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer periodSeconds: description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1. format: int32 type: integer successThreshold: description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. format: int32 type: integer tcpSocket: description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' properties: host: description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object timeoutSeconds: description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer type: object name: description: Name of the container specified as a DNS_LABEL. Each container in a pod must have a unique name (DNS_LABEL). Cannot be updated. type: string ports: description: List of ports to expose from the container. Exposing a port here gives the system additional information about the network connections a container uses, but is primarily informational. Not specifying a port here DOES NOT prevent that port from being exposed. Any port which is listening on the default "0.0.0.0" address inside a container will be accessible from the network. Cannot be updated. items: description: ContainerPort represents a network port in a single container. properties: containerPort: description: Number of port to expose on the pod's IP address. This must be a valid port number, 0 < x < 65536. format: int32 type: integer hostIP: description: What host IP to bind the external port to. type: string hostPort: description: Number of port to expose on the host. If specified, this must be a valid port number, 0 < x < 65536. If HostNetwork is specified, this must match ContainerPort. Most containers do not need this. format: int32 type: integer name: description: If specified, this must be an IANA_SVC_NAME and unique within the pod. Each named port in a pod must have a unique name. Name for the port that can be referred to by services. type: string protocol: default: TCP description: Protocol for port. Must be UDP, TCP, or SCTP. Defaults to "TCP". type: string required: - containerPort type: object type: array x-kubernetes-list-map-keys: - containerPort - protocol x-kubernetes-list-type: map readinessProbe: description: 'Periodic probe of container service readiness. Container will be removed from service endpoints if the probe fails. Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' properties: exec: description: One and only one of the following should be specified. Exec specifies the action to take. properties: command: description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array type: object failureThreshold: description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1. format: int32 type: integer httpGet: description: HTTPGet specifies the http request to perform. properties: host: description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. type: string httpHeaders: description: Custom headers to set in the request. HTTP allows repeated headers. items: description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: description: The header field name type: string value: description: The header field value type: string required: - name - value type: object type: array path: description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: description: Scheme to use for connecting to the host. Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer periodSeconds: description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1. format: int32 type: integer successThreshold: description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. format: int32 type: integer tcpSocket: description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' properties: host: description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object timeoutSeconds: description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer type: object resources: description: 'Compute Resources required by this container. Cannot be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' properties: limits: additionalProperties: anyOf: - type: integer - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' type: object requests: additionalProperties: anyOf: - type: integer - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' type: object type: object securityContext: description: 'Security options the pod should run with. More info: https://kubernetes.io/docs/concepts/policy/security-context/ More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/' properties: allowPrivilegeEscalation: description: 'AllowPrivilegeEscalation controls whether a process can gain more privileges than its parent process. This bool directly controls if the no_new_privs flag will be set on the container process. AllowPrivilegeEscalation is true always when the container is: 1) run as Privileged 2) has CAP_SYS_ADMIN' type: boolean capabilities: description: The capabilities to add/drop when running containers. Defaults to the default set of capabilities granted by the container runtime. properties: add: description: Added capabilities items: description: Capability represent POSIX capabilities type type: string type: array drop: description: Removed capabilities items: description: Capability represent POSIX capabilities type type: string type: array type: object privileged: description: Run container in privileged mode. Processes in privileged containers are essentially equivalent to root on the host. Defaults to false. type: boolean procMount: description: procMount denotes the type of proc mount to use for the containers. The default is DefaultProcMount which uses the container runtime defaults for readonly paths and masked paths. This requires the ProcMountType feature flag to be enabled. type: string readOnlyRootFilesystem: description: Whether this container has a read-only root filesystem. Default is false. type: boolean runAsGroup: description: The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. format: int64 type: integer runAsNonRoot: description: Indicates that the container must run as a non-root user. If true, the Kubelet will validate the image at runtime to ensure that it does not run as UID 0 (root) and fail to start the container if it does. If unset or false, no such validation will be performed. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. type: boolean runAsUser: description: The UID to run the entrypoint of the container process. Defaults to user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. format: int64 type: integer seLinuxOptions: description: The SELinux context to be applied to the container. If unspecified, the container runtime will allocate a random SELinux context for each container. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. properties: level: description: Level is SELinux level label that applies to the container. type: string role: description: Role is a SELinux role label that applies to the container. type: string type: description: Type is a SELinux type label that applies to the container. type: string user: description: User is a SELinux user label that applies to the container. type: string type: object windowsOptions: description: The Windows specific settings applied to all containers. If unspecified, the options from the PodSecurityContext will be used. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. properties: gmsaCredentialSpec: description: GMSACredentialSpec is where the GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the GMSA credential spec named by the GMSACredentialSpecName field. type: string gmsaCredentialSpecName: description: GMSACredentialSpecName is the name of the GMSA credential spec to use. type: string runAsUserName: description: The UserName in Windows to run the entrypoint of the container process. Defaults to the user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. type: string type: object type: object startupProbe: description: 'StartupProbe indicates that the Pod has successfully initialized. If specified, no other probes are executed until this completes successfully. If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. This can be used to provide different probe parameters at the beginning of a Pod''s lifecycle, when it might take a long time to load data or warm a cache, than during steady-state operation. This cannot be updated. This is a beta feature enabled by the StartupProbe feature flag. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' properties: exec: description: One and only one of the following should be specified. Exec specifies the action to take. properties: command: description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array type: object failureThreshold: description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1. format: int32 type: integer httpGet: description: HTTPGet specifies the http request to perform. properties: host: description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. type: string httpHeaders: description: Custom headers to set in the request. HTTP allows repeated headers. items: description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: description: The header field name type: string value: description: The header field value type: string required: - name - value type: object type: array path: description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: description: Scheme to use for connecting to the host. Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer periodSeconds: description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1. format: int32 type: integer successThreshold: description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. format: int32 type: integer tcpSocket: description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' properties: host: description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object timeoutSeconds: description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer type: object stdin: description: Whether this container should allocate a buffer for stdin in the container runtime. If this is not set, reads from stdin in the container will always result in EOF. Default is false. type: boolean stdinOnce: description: Whether the container runtime should close the stdin channel after it has been opened by a single attach. When stdin is true the stdin stream will remain open across multiple attach sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the first client attaches to stdin, and then remains open and accepts data until the client disconnects, at which time stdin is closed and remains closed until the container is restarted. If this flag is false, a container processes that reads from stdin will never receive an EOF. Default is false type: boolean terminationMessagePath: description: 'Optional: Path at which the file to which the container''s termination message will be written is mounted into the container''s filesystem. Message written is intended to be brief final status, such as an assertion failure message. Will be truncated by the node if greater than 4096 bytes. The total message length across all containers will be limited to 12kb. Defaults to /dev/termination-log. Cannot be updated.' type: string terminationMessagePolicy: description: Indicate how the termination message should be populated. File will use the contents of terminationMessagePath to populate the container status message on both success and failure. FallbackToLogsOnError will use the last chunk of container log output if the termination message file is empty and the container exited with an error. The log output is limited to 2048 bytes or 80 lines, whichever is smaller. Defaults to File. Cannot be updated. type: string tty: description: Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. Default is false. type: boolean volumeDevices: description: volumeDevices is the list of block devices to be used by the container. items: description: volumeDevice describes a mapping of a raw block device within a container. properties: devicePath: description: devicePath is the path inside of the container that the device will be mapped to. type: string name: description: name must match the name of a persistentVolumeClaim in the pod type: string required: - devicePath - name type: object type: array volumeMounts: description: Pod volumes to mount into the container's filesystem. Cannot be updated. items: description: VolumeMount describes a mounting of a Volume within a container. properties: mountPath: description: Path within the container at which the volume should be mounted. Must not contain ':'. type: string mountPropagation: description: mountPropagation determines how mounts are propagated from the host to container and the other way around. When not set, MountPropagationNone is used. This field is beta in 1.10. type: string name: description: This must match the Name of a Volume. type: string readOnly: description: Mounted read-only if true, read-write otherwise (false or unspecified). Defaults to false. type: boolean subPath: description: Path within the volume from which the container's volume should be mounted. Defaults to "" (volume's root). type: string subPathExpr: description: Expanded path within the volume from which the container's volume should be mounted. Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. Defaults to "" (volume's root). SubPathExpr and SubPath are mutually exclusive. type: string required: - mountPath - name type: object type: array workingDir: description: Container's working directory. If not specified, the container runtime's default will be used, which might be configured in the container image. Cannot be updated. type: string required: - name type: object type: array labels: additionalProperties: type: string description: Labels configure the external label pairs to ThanosRuler. If not provided, default replica label `thanos_ruler_replica` will be added as a label and be dropped in alerts. type: object listenLocal: description: ListenLocal makes the Thanos ruler listen on loopback, so that it does not bind against the Pod IP. type: boolean logFormat: description: Log format for ThanosRuler to be configured with. type: string logLevel: description: Log level for ThanosRuler to be configured with. type: string nodeSelector: additionalProperties: type: string description: Define which Nodes the Pods are scheduled on. type: object objectStorageConfig: description: ObjectStorageConfig configures object storage in Thanos. Alternative to ObjectStorageConfigFile, and lower order priority. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object objectStorageConfigFile: description: ObjectStorageConfigFile specifies the path of the object storage configuration file. When used alongside with ObjectStorageConfig, ObjectStorageConfigFile takes precedence. type: string paused: description: When a ThanosRuler deployment is paused, no actions except for deletion will be performed on the underlying objects. type: boolean podMetadata: description: PodMetadata contains Labels and Annotations gets propagated to the thanos ruler pods. properties: annotations: additionalProperties: type: string description: 'Annotations is an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should be preserved when modifying objects. More info: http://kubernetes.io/docs/user-guide/annotations' type: object labels: additionalProperties: type: string description: 'Map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. More info: http://kubernetes.io/docs/user-guide/labels' type: object name: description: 'Name must be unique within a namespace. Is required when creating resources, although some resources may allow a client to request the generation of an appropriate name automatically. Name is primarily intended for creation idempotence and configuration definition. Cannot be updated. More info: http://kubernetes.io/docs/user-guide/identifiers#names' type: string type: object portName: description: Port name used for the pods and governing service. This defaults to web type: string priorityClassName: description: Priority class assigned to the Pods type: string prometheusRulesExcludedFromEnforce: description: PrometheusRulesExcludedFromEnforce - list of Prometheus rules to be excluded from enforcing of adding namespace labels. Works only if enforcedNamespaceLabel set to true. Make sure both ruleNamespace and ruleName are set for each pair items: description: PrometheusRuleExcludeConfig enables users to configure excluded PrometheusRule names and their namespaces to be ignored while enforcing namespace label for alerts and metrics. properties: ruleName: description: RuleNamespace - name of excluded rule type: string ruleNamespace: description: RuleNamespace - namespace of excluded rule type: string required: - ruleName - ruleNamespace type: object type: array queryConfig: description: Define configuration for connecting to thanos query instances. If this is defined, the QueryEndpoints field will be ignored. Maps to the `query.config` CLI argument. Only available with thanos v0.11.0 and higher. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object queryEndpoints: description: QueryEndpoints defines Thanos querier endpoints from which to query metrics. Maps to the --query flag of thanos ruler. items: type: string type: array replicas: description: Number of thanos ruler instances to deploy. format: int32 type: integer resources: description: Resources defines the resource requirements for single Pods. If not provided, no requests/limits will be set properties: limits: additionalProperties: anyOf: - type: integer - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' type: object requests: additionalProperties: anyOf: - type: integer - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' type: object type: object retention: description: Time duration ThanosRuler shall retain data for. Default is '24h', and must match the regular expression `[0-9]+(ms|s|m|h|d|w|y)` (milliseconds seconds minutes hours days weeks years). type: string routePrefix: description: The route prefix ThanosRuler registers HTTP handlers for. This allows thanos UI to be served on a sub-path. type: string ruleNamespaceSelector: description: Namespaces to be selected for Rules discovery. If unspecified, only the same namespace as the ThanosRuler object is in is used. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. items: type: string type: array required: - key - operator type: object type: array matchLabels: additionalProperties: type: string description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object ruleSelector: description: A label selector to select which PrometheusRules to mount for alerting and recording. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. items: type: string type: array required: - key - operator type: object type: array matchLabels: additionalProperties: type: string description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object securityContext: description: SecurityContext holds pod-level security attributes and common container settings. This defaults to the default PodSecurityContext. properties: fsGroup: description: "A special supplemental group that applies to all containers in a pod. Some volume types allow the Kubelet to change the ownership of that volume to be owned by the pod: \n 1. The owning GID will be the FSGroup 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) 3. The permission bits are OR'd with rw-rw---- \n If unset, the Kubelet will not modify the ownership and permissions of any volume." format: int64 type: integer fsGroupChangePolicy: description: 'fsGroupChangePolicy defines behavior of changing ownership and permission of the volume before being exposed inside Pod. This field will only apply to volume types which support fsGroup based ownership(and permissions). It will have no effect on ephemeral volume types such as: secret, configmaps and emptydir. Valid values are "OnRootMismatch" and "Always". If not specified defaults to "Always".' type: string runAsGroup: description: The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence for that container. format: int64 type: integer runAsNonRoot: description: Indicates that the container must run as a non-root user. If true, the Kubelet will validate the image at runtime to ensure that it does not run as UID 0 (root) and fail to start the container if it does. If unset or false, no such validation will be performed. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. type: boolean runAsUser: description: The UID to run the entrypoint of the container process. Defaults to user specified in image metadata if unspecified. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence for that container. format: int64 type: integer seLinuxOptions: description: The SELinux context to be applied to all containers. If unspecified, the container runtime will allocate a random SELinux context for each container. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence for that container. properties: level: description: Level is SELinux level label that applies to the container. type: string role: description: Role is a SELinux role label that applies to the container. type: string type: description: Type is a SELinux type label that applies to the container. type: string user: description: User is a SELinux user label that applies to the container. type: string type: object supplementalGroups: description: A list of groups applied to the first process run in each container, in addition to the container's primary GID. If unspecified, no groups will be added to any container. items: format: int64 type: integer type: array sysctls: description: Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported sysctls (by the container runtime) might fail to launch. items: description: Sysctl defines a kernel parameter to be set properties: name: description: Name of a property to set type: string value: description: Value of a property to set type: string required: - name - value type: object type: array windowsOptions: description: The Windows specific settings applied to all containers. If unspecified, the options within a container's SecurityContext will be used. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. properties: gmsaCredentialSpec: description: GMSACredentialSpec is where the GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the GMSA credential spec named by the GMSACredentialSpecName field. type: string gmsaCredentialSpecName: description: GMSACredentialSpecName is the name of the GMSA credential spec to use. type: string runAsUserName: description: The UserName in Windows to run the entrypoint of the container process. Defaults to the user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. type: string type: object type: object serviceAccountName: description: ServiceAccountName is the name of the ServiceAccount to use to run the Thanos Ruler Pods. type: string storage: description: Storage spec to specify how storage shall be used. properties: disableMountSubPath: description: 'Deprecated: subPath usage will be disabled by default in a future release, this option will become unnecessary. DisableMountSubPath allows to remove any subPath usage in volume mounts.' type: boolean emptyDir: description: 'EmptyDirVolumeSource to be used by the Prometheus StatefulSets. If specified, used in place of any volumeClaimTemplate. More info: https://kubernetes.io/docs/concepts/storage/volumes/#emptydir' properties: medium: description: 'What type of storage medium should back this directory. The default is "" which means to use the node''s default medium. Must be an empty string (default) or Memory. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' type: string sizeLimit: anyOf: - type: integer - type: string description: 'Total amount of local storage required for this EmptyDir volume. The size limit is also applicable for memory medium. The maximum usage on memory medium EmptyDir would be the minimum value between the SizeLimit specified here and the sum of memory limits of all containers in a pod. The default is nil which means that the limit is undefined. More info: http://kubernetes.io/docs/user-guide/volumes#emptydir' pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true type: object volumeClaimTemplate: description: A PVC spec to be used by the Prometheus StatefulSets. properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: description: EmbeddedMetadata contains metadata relevant to an EmbeddedResource. properties: annotations: additionalProperties: type: string description: 'Annotations is an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should be preserved when modifying objects. More info: http://kubernetes.io/docs/user-guide/annotations' type: object labels: additionalProperties: type: string description: 'Map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. More info: http://kubernetes.io/docs/user-guide/labels' type: object name: description: 'Name must be unique within a namespace. Is required when creating resources, although some resources may allow a client to request the generation of an appropriate name automatically. Name is primarily intended for creation idempotence and configuration definition. Cannot be updated. More info: http://kubernetes.io/docs/user-guide/identifiers#names' type: string type: object spec: description: 'Spec defines the desired characteristics of a volume requested by a pod author. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' properties: accessModes: description: 'AccessModes contains the desired access modes the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1' items: type: string type: array dataSource: description: 'This field can be used to specify either: * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot - Beta) * An existing PVC (PersistentVolumeClaim) * An existing custom resource/object that implements data population (Alpha) In order to use VolumeSnapshot object types, the appropriate feature gate must be enabled (VolumeSnapshotDataSource or AnyVolumeDataSource) If the provisioner or an external controller can support the specified data source, it will create a new volume based on the contents of the specified data source. If the specified data source is not supported, the volume will not be created and the failure will be reported as an event. In the future, we plan to support more data source types and the behavior of the provisioner may change.' properties: apiGroup: description: APIGroup is the group for the resource being referenced. If APIGroup is not specified, the specified Kind must be in the core API group. For any other third-party types, APIGroup is required. type: string kind: description: Kind is the type of resource being referenced type: string name: description: Name is the name of resource being referenced type: string required: - kind - name type: object resources: description: 'Resources represents the minimum resources the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources' properties: limits: additionalProperties: anyOf: - type: integer - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' type: object requests: additionalProperties: anyOf: - type: integer - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' type: object type: object selector: description: A label query over volumes to consider for binding. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. items: type: string type: array required: - key - operator type: object type: array matchLabels: additionalProperties: type: string description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object storageClassName: description: 'Name of the StorageClass required by the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1' type: string volumeMode: description: volumeMode defines what type of volume is required by the claim. Value of Filesystem is implied when not included in claim spec. type: string volumeName: description: VolumeName is the binding reference to the PersistentVolume backing this claim. type: string type: object status: description: 'Status represents the current information/status of a persistent volume claim. Read-only. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' properties: accessModes: description: 'AccessModes contains the actual access modes the volume backing the PVC has. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1' items: type: string type: array capacity: additionalProperties: anyOf: - type: integer - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true description: Represents the actual resources of the underlying volume. type: object conditions: description: Current Condition of persistent volume claim. If underlying persistent volume is being resized then the Condition will be set to 'ResizeStarted'. items: description: PersistentVolumeClaimCondition contails details about state of pvc properties: lastProbeTime: description: Last time we probed the condition. format: date-time type: string lastTransitionTime: description: Last time the condition transitioned from one status to another. format: date-time type: string message: description: Human-readable message indicating details about last transition. type: string reason: description: Unique, this should be a short, machine understandable string that gives the reason for condition's last transition. If it reports "ResizeStarted" that means the underlying persistent volume is being resized. type: string status: type: string type: description: PersistentVolumeClaimConditionType is a valid value of PersistentVolumeClaimCondition.Type type: string required: - status - type type: object type: array phase: description: Phase represents the current phase of PersistentVolumeClaim. type: string type: object type: object type: object tolerations: description: If specified, the pod's tolerations. items: description: The pod this Toleration is attached to tolerates any taint that matches the triple using the matching operator . properties: effect: description: Effect indicates the taint effect to match. Empty means match all taint effects. When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. type: string key: description: Key is the taint key that the toleration applies to. Empty means match all taint keys. If the key is empty, operator must be Exists; this combination means to match all values and all keys. type: string operator: description: Operator represents a key's relationship to the value. Valid operators are Exists and Equal. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category. type: string tolerationSeconds: description: TolerationSeconds represents the period of time the toleration (which must be of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, it is not set, which means tolerate the taint forever (do not evict). Zero and negative values will be treated as 0 (evict immediately) by the system. format: int64 type: integer value: description: Value is the taint value the toleration matches to. If the operator is Exists, the value should be empty, otherwise just a regular string. type: string type: object type: array topologySpreadConstraints: description: If specified, the pod's topology spread constraints. items: description: TopologySpreadConstraint specifies how to spread matching pods among the given topology. properties: labelSelector: description: LabelSelector is used to find matching pods. Pods that match this label selector are counted to determine the number of pods in their corresponding topology domain. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. items: type: string type: array required: - key - operator type: object type: array matchLabels: additionalProperties: type: string description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object maxSkew: description: 'MaxSkew describes the degree to which pods may be unevenly distributed. It''s the maximum permitted difference between the number of matching pods in any two topology domains of a given topology type. For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same labelSelector spread as 1/1/0: | zone1 | zone2 | zone3 | | P | P | | - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 1/1/1; scheduling it onto zone1(zone2) would make the ActualSkew(2-0) on zone1(zone2) violate MaxSkew(1). - if MaxSkew is 2, incoming pod can be scheduled onto any zone. It''s a required field. Default value is 1 and 0 is not allowed.' format: int32 type: integer topologyKey: description: TopologyKey is the key of node labels. Nodes that have a label with this key and identical values are considered to be in the same topology. We consider each as a "bucket", and try to put balanced number of pods into each bucket. It's a required field. type: string whenUnsatisfiable: description: 'WhenUnsatisfiable indicates how to deal with a pod if it doesn''t satisfy the spread constraint. - DoNotSchedule (default) tells the scheduler not to schedule it - ScheduleAnyway tells the scheduler to still schedule it It''s considered as "Unsatisfiable" if and only if placing incoming pod on any topology violates "MaxSkew". For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same labelSelector spread as 3/1/1: | zone1 | zone2 | zone3 | | P P P | P | P | If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler won''t make it *more* imbalanced. It''s a required field.' type: string required: - maxSkew - topologyKey - whenUnsatisfiable type: object type: array tracingConfig: description: TracingConfig configures tracing in Thanos. This is an experimental feature, it may change in any upcoming release in a breaking way. properties: key: description: The key of the secret to select from. Must be a valid secret key. type: string name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean required: - key type: object volumes: description: Volumes allows configuration of additional volumes on the output StatefulSet definition. Volumes specified will be appended to other volumes that are generated as a result of StorageSpec objects. items: description: Volume represents a named volume in a pod that may be accessed by any container in the pod. properties: awsElasticBlockStore: description: 'AWSElasticBlockStore represents an AWS Disk resource that is attached to a kubelet''s host machine and then exposed to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' properties: fsType: description: 'Filesystem type of the volume that you want to mount. Tip: Ensure that the filesystem type is supported by the host operating system. Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore TODO: how do we prevent errors in the filesystem from compromising the machine' type: string partition: description: 'The partition in the volume that you want to mount. If omitted, the default is to mount by volume name. Examples: For volume /dev/sda1, you specify the partition as "1". Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty).' format: int32 type: integer readOnly: description: 'Specify "true" to force and set the ReadOnly property in VolumeMounts to "true". If omitted, the default is "false". More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' type: boolean volumeID: description: 'Unique ID of the persistent disk resource in AWS (Amazon EBS volume). More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' type: string required: - volumeID type: object azureDisk: description: AzureDisk represents an Azure Data Disk mount on the host and bind mount to the pod. properties: cachingMode: description: 'Host Caching mode: None, Read Only, Read Write.' type: string diskName: description: The Name of the data disk in the blob storage type: string diskURI: description: The URI the data disk in the blob storage type: string fsType: description: Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. type: string kind: description: 'Expected values Shared: multiple blob disks per storage account Dedicated: single blob disk per storage account Managed: azure managed data disk (only in managed availability set). defaults to shared' type: string readOnly: description: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts. type: boolean required: - diskName - diskURI type: object azureFile: description: AzureFile represents an Azure File Service mount on the host and bind mount to the pod. properties: readOnly: description: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts. type: boolean secretName: description: the name of secret that contains Azure Storage Account Name and Key type: string shareName: description: Share Name type: string required: - secretName - shareName type: object cephfs: description: CephFS represents a Ceph FS mount on the host that shares a pod's lifetime properties: monitors: description: 'Required: Monitors is a collection of Ceph monitors More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' items: type: string type: array path: description: 'Optional: Used as the mounted root, rather than the full Ceph tree, default is /' type: string readOnly: description: 'Optional: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts. More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' type: boolean secretFile: description: 'Optional: SecretFile is the path to key ring for User, default is /etc/ceph/user.secret More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' type: string secretRef: description: 'Optional: SecretRef is reference to the authentication secret for User, default is empty. More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' properties: name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string type: object user: description: 'Optional: User is the rados user name, default is admin More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' type: string required: - monitors type: object cinder: description: 'Cinder represents a cinder volume attached and mounted on kubelets host machine. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' properties: fsType: description: 'Filesystem type to mount. Must be a filesystem type supported by the host operating system. Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' type: string readOnly: description: 'Optional: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' type: boolean secretRef: description: 'Optional: points to a secret object containing parameters used to connect to OpenStack.' properties: name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string type: object volumeID: description: 'volume id used to identify the volume in cinder. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' type: string required: - volumeID type: object configMap: description: ConfigMap represents a configMap that should populate this volume properties: defaultMode: description: 'Optional: mode bits to use on created files by default. Must be a value between 0 and 0777. Defaults to 0644. Directories within the path are not affected by this setting. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.' format: int32 type: integer items: description: If unspecified, each key-value pair in the Data field of the referenced ConfigMap will be projected into the volume as a file whose name is the key and content is the value. If specified, the listed keys will be projected into the specified paths, and unlisted keys will not be present. If a key is specified which is not present in the ConfigMap, the volume setup will error unless it is marked optional. Paths must be relative and may not contain the '..' path or start with '..'. items: description: Maps a string key to a path within a volume. properties: key: description: The key to project. type: string mode: description: 'Optional: mode bits to use on this file, must be a value between 0 and 0777. If not specified, the volume defaultMode will be used. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.' format: int32 type: integer path: description: The relative path of the file to map the key to. May not be an absolute path. May not contain the path element '..'. May not start with the string '..'. type: string required: - key - path type: object type: array name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the ConfigMap or its keys must be defined type: boolean type: object csi: description: CSI (Container Storage Interface) represents storage that is handled by an external CSI driver (Alpha feature). properties: driver: description: Driver is the name of the CSI driver that handles this volume. Consult with your admin for the correct name as registered in the cluster. type: string fsType: description: Filesystem type to mount. Ex. "ext4", "xfs", "ntfs". If not provided, the empty value is passed to the associated CSI driver which will determine the default filesystem to apply. type: string nodePublishSecretRef: description: NodePublishSecretRef is a reference to the secret object containing sensitive information to pass to the CSI driver to complete the CSI NodePublishVolume and NodeUnpublishVolume calls. This field is optional, and may be empty if no secret is required. If the secret object contains more than one secret, all secret references are passed. properties: name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string type: object readOnly: description: Specifies a read-only configuration for the volume. Defaults to false (read/write). type: boolean volumeAttributes: additionalProperties: type: string description: VolumeAttributes stores driver-specific properties that are passed to the CSI driver. Consult your driver's documentation for supported values. type: object required: - driver type: object downwardAPI: description: DownwardAPI represents downward API about the pod that should populate this volume properties: defaultMode: description: 'Optional: mode bits to use on created files by default. Must be a value between 0 and 0777. Defaults to 0644. Directories within the path are not affected by this setting. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.' format: int32 type: integer items: description: Items is a list of downward API volume file items: description: DownwardAPIVolumeFile represents information to create the file containing the pod field properties: fieldRef: description: 'Required: Selects a field of the pod: only annotations, labels, name and namespace are supported.' properties: apiVersion: description: Version of the schema the FieldPath is written in terms of, defaults to "v1". type: string fieldPath: description: Path of the field to select in the specified API version. type: string required: - fieldPath type: object mode: description: 'Optional: mode bits to use on this file, must be a value between 0 and 0777. If not specified, the volume defaultMode will be used. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.' format: int32 type: integer path: description: 'Required: Path is the relative path name of the file to be created. Must not be absolute or contain the ''..'' path. Must be utf-8 encoded. The first item of the relative path must not start with ''..''' type: string resourceFieldRef: description: 'Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported.' properties: containerName: description: 'Container name: required for volumes, optional for env vars' type: string divisor: anyOf: - type: integer - type: string description: Specifies the output format of the exposed resources, defaults to "1" pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true resource: description: 'Required: resource to select' type: string required: - resource type: object required: - path type: object type: array type: object emptyDir: description: 'EmptyDir represents a temporary directory that shares a pod''s lifetime. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' properties: medium: description: 'What type of storage medium should back this directory. The default is "" which means to use the node''s default medium. Must be an empty string (default) or Memory. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' type: string sizeLimit: anyOf: - type: integer - type: string description: 'Total amount of local storage required for this EmptyDir volume. The size limit is also applicable for memory medium. The maximum usage on memory medium EmptyDir would be the minimum value between the SizeLimit specified here and the sum of memory limits of all containers in a pod. The default is nil which means that the limit is undefined. More info: http://kubernetes.io/docs/user-guide/volumes#emptydir' pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true type: object fc: description: FC represents a Fibre Channel resource that is attached to a kubelet's host machine and then exposed to the pod. properties: fsType: description: 'Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. TODO: how do we prevent errors in the filesystem from compromising the machine' type: string lun: description: 'Optional: FC target lun number' format: int32 type: integer readOnly: description: 'Optional: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts.' type: boolean targetWWNs: description: 'Optional: FC target worldwide names (WWNs)' items: type: string type: array wwids: description: 'Optional: FC volume world wide identifiers (wwids) Either wwids or combination of targetWWNs and lun must be set, but not both simultaneously.' items: type: string type: array type: object flexVolume: description: FlexVolume represents a generic volume resource that is provisioned/attached using an exec based plugin. properties: driver: description: Driver is the name of the driver to use for this volume. type: string fsType: description: Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. "ext4", "xfs", "ntfs". The default filesystem depends on FlexVolume script. type: string options: additionalProperties: type: string description: 'Optional: Extra command options if any.' type: object readOnly: description: 'Optional: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts.' type: boolean secretRef: description: 'Optional: SecretRef is reference to the secret object containing sensitive information to pass to the plugin scripts. This may be empty if no secret object is specified. If the secret object contains more than one secret, all secrets are passed to the plugin scripts.' properties: name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string type: object required: - driver type: object flocker: description: Flocker represents a Flocker volume attached to a kubelet's host machine. This depends on the Flocker control service being running properties: datasetName: description: Name of the dataset stored as metadata -> name on the dataset for Flocker should be considered as deprecated type: string datasetUUID: description: UUID of the dataset. This is unique identifier of a Flocker dataset type: string type: object gcePersistentDisk: description: 'GCEPersistentDisk represents a GCE Disk resource that is attached to a kubelet''s host machine and then exposed to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' properties: fsType: description: 'Filesystem type of the volume that you want to mount. Tip: Ensure that the filesystem type is supported by the host operating system. Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk TODO: how do we prevent errors in the filesystem from compromising the machine' type: string partition: description: 'The partition in the volume that you want to mount. If omitted, the default is to mount by volume name. Examples: For volume /dev/sda1, you specify the partition as "1". Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' format: int32 type: integer pdName: description: 'Unique name of the PD resource in GCE. Used to identify the disk in GCE. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' type: string readOnly: description: 'ReadOnly here will force the ReadOnly setting in VolumeMounts. Defaults to false. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' type: boolean required: - pdName type: object gitRepo: description: 'GitRepo represents a git repository at a particular revision. DEPRECATED: GitRepo is deprecated. To provision a container with a git repo, mount an EmptyDir into an InitContainer that clones the repo using git, then mount the EmptyDir into the Pod''s container.' properties: directory: description: Target directory name. Must not contain or start with '..'. If '.' is supplied, the volume directory will be the git repository. Otherwise, if specified, the volume will contain the git repository in the subdirectory with the given name. type: string repository: description: Repository URL type: string revision: description: Commit hash for the specified revision. type: string required: - repository type: object glusterfs: description: 'Glusterfs represents a Glusterfs mount on the host that shares a pod''s lifetime. More info: https://examples.k8s.io/volumes/glusterfs/README.md' properties: endpoints: description: 'EndpointsName is the endpoint name that details Glusterfs topology. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' type: string path: description: 'Path is the Glusterfs volume path. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' type: string readOnly: description: 'ReadOnly here will force the Glusterfs volume to be mounted with read-only permissions. Defaults to false. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' type: boolean required: - endpoints - path type: object hostPath: description: 'HostPath represents a pre-existing file or directory on the host machine that is directly exposed to the container. This is generally used for system agents or other privileged things that are allowed to see the host machine. Most containers will NOT need this. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath --- TODO(jonesdl) We need to restrict who can use host directory mounts and who can/can not mount host directories as read/write.' properties: path: description: 'Path of the directory on the host. If the path is a symlink, it will follow the link to the real path. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath' type: string type: description: 'Type for HostPath Volume Defaults to "" More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath' type: string required: - path type: object iscsi: description: 'ISCSI represents an ISCSI Disk resource that is attached to a kubelet''s host machine and then exposed to the pod. More info: https://examples.k8s.io/volumes/iscsi/README.md' properties: chapAuthDiscovery: description: whether support iSCSI Discovery CHAP authentication type: boolean chapAuthSession: description: whether support iSCSI Session CHAP authentication type: boolean fsType: description: 'Filesystem type of the volume that you want to mount. Tip: Ensure that the filesystem type is supported by the host operating system. Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi TODO: how do we prevent errors in the filesystem from compromising the machine' type: string initiatorName: description: Custom iSCSI Initiator Name. If initiatorName is specified with iscsiInterface simultaneously, new iSCSI interface : will be created for the connection. type: string iqn: description: Target iSCSI Qualified Name. type: string iscsiInterface: description: iSCSI Interface Name that uses an iSCSI transport. Defaults to 'default' (tcp). type: string lun: description: iSCSI Target Lun number. format: int32 type: integer portals: description: iSCSI Target Portal List. The portal is either an IP or ip_addr:port if the port is other than default (typically TCP ports 860 and 3260). items: type: string type: array readOnly: description: ReadOnly here will force the ReadOnly setting in VolumeMounts. Defaults to false. type: boolean secretRef: description: CHAP Secret for iSCSI target and initiator authentication properties: name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string type: object targetPortal: description: iSCSI Target Portal. The Portal is either an IP or ip_addr:port if the port is other than default (typically TCP ports 860 and 3260). type: string required: - iqn - lun - targetPortal type: object name: description: 'Volume''s name. Must be a DNS_LABEL and unique within the pod. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' type: string nfs: description: 'NFS represents an NFS mount on the host that shares a pod''s lifetime More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' properties: path: description: 'Path that is exported by the NFS server. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' type: string readOnly: description: 'ReadOnly here will force the NFS export to be mounted with read-only permissions. Defaults to false. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' type: boolean server: description: 'Server is the hostname or IP address of the NFS server. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' type: string required: - path - server type: object persistentVolumeClaim: description: 'PersistentVolumeClaimVolumeSource represents a reference to a PersistentVolumeClaim in the same namespace. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' properties: claimName: description: 'ClaimName is the name of a PersistentVolumeClaim in the same namespace as the pod using this volume. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' type: string readOnly: description: Will force the ReadOnly setting in VolumeMounts. Default false. type: boolean required: - claimName type: object photonPersistentDisk: description: PhotonPersistentDisk represents a PhotonController persistent disk attached and mounted on kubelets host machine properties: fsType: description: Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. type: string pdID: description: ID that identifies Photon Controller persistent disk type: string required: - pdID type: object portworxVolume: description: PortworxVolume represents a portworx volume attached and mounted on kubelets host machine properties: fsType: description: FSType represents the filesystem type to mount Must be a filesystem type supported by the host operating system. Ex. "ext4", "xfs". Implicitly inferred to be "ext4" if unspecified. type: string readOnly: description: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts. type: boolean volumeID: description: VolumeID uniquely identifies a Portworx volume type: string required: - volumeID type: object projected: description: Items for all in one resources secrets, configmaps, and downward API properties: defaultMode: description: Mode bits to use on created files by default. Must be a value between 0 and 0777. Directories within the path are not affected by this setting. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set. format: int32 type: integer sources: description: list of volume projections items: description: Projection that may be projected along with other supported volume types properties: configMap: description: information about the configMap data to project properties: items: description: If unspecified, each key-value pair in the Data field of the referenced ConfigMap will be projected into the volume as a file whose name is the key and content is the value. If specified, the listed keys will be projected into the specified paths, and unlisted keys will not be present. If a key is specified which is not present in the ConfigMap, the volume setup will error unless it is marked optional. Paths must be relative and may not contain the '..' path or start with '..'. items: description: Maps a string key to a path within a volume. properties: key: description: The key to project. type: string mode: description: 'Optional: mode bits to use on this file, must be a value between 0 and 0777. If not specified, the volume defaultMode will be used. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.' format: int32 type: integer path: description: The relative path of the file to map the key to. May not be an absolute path. May not contain the path element '..'. May not start with the string '..'. type: string required: - key - path type: object type: array name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the ConfigMap or its keys must be defined type: boolean type: object downwardAPI: description: information about the downwardAPI data to project properties: items: description: Items is a list of DownwardAPIVolume file items: description: DownwardAPIVolumeFile represents information to create the file containing the pod field properties: fieldRef: description: 'Required: Selects a field of the pod: only annotations, labels, name and namespace are supported.' properties: apiVersion: description: Version of the schema the FieldPath is written in terms of, defaults to "v1". type: string fieldPath: description: Path of the field to select in the specified API version. type: string required: - fieldPath type: object mode: description: 'Optional: mode bits to use on this file, must be a value between 0 and 0777. If not specified, the volume defaultMode will be used. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.' format: int32 type: integer path: description: 'Required: Path is the relative path name of the file to be created. Must not be absolute or contain the ''..'' path. Must be utf-8 encoded. The first item of the relative path must not start with ''..''' type: string resourceFieldRef: description: 'Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported.' properties: containerName: description: 'Container name: required for volumes, optional for env vars' type: string divisor: anyOf: - type: integer - type: string description: Specifies the output format of the exposed resources, defaults to "1" pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true resource: description: 'Required: resource to select' type: string required: - resource type: object required: - path type: object type: array type: object secret: description: information about the secret data to project properties: items: description: If unspecified, each key-value pair in the Data field of the referenced Secret will be projected into the volume as a file whose name is the key and content is the value. If specified, the listed keys will be projected into the specified paths, and unlisted keys will not be present. If a key is specified which is not present in the Secret, the volume setup will error unless it is marked optional. Paths must be relative and may not contain the '..' path or start with '..'. items: description: Maps a string key to a path within a volume. properties: key: description: The key to project. type: string mode: description: 'Optional: mode bits to use on this file, must be a value between 0 and 0777. If not specified, the volume defaultMode will be used. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.' format: int32 type: integer path: description: The relative path of the file to map the key to. May not be an absolute path. May not contain the path element '..'. May not start with the string '..'. type: string required: - key - path type: object type: array name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string optional: description: Specify whether the Secret or its key must be defined type: boolean type: object serviceAccountToken: description: information about the serviceAccountToken data to project properties: audience: description: Audience is the intended audience of the token. A recipient of a token must identify itself with an identifier specified in the audience of the token, and otherwise should reject the token. The audience defaults to the identifier of the apiserver. type: string expirationSeconds: description: ExpirationSeconds is the requested duration of validity of the service account token. As the token approaches expiration, the kubelet volume plugin will proactively rotate the service account token. The kubelet will start trying to rotate the token if the token is older than 80 percent of its time to live or if the token is older than 24 hours.Defaults to 1 hour and must be at least 10 minutes. format: int64 type: integer path: description: Path is the path relative to the mount point of the file to project the token into. type: string required: - path type: object type: object type: array required: - sources type: object quobyte: description: Quobyte represents a Quobyte mount on the host that shares a pod's lifetime properties: group: description: Group to map volume access to Default is no group type: string readOnly: description: ReadOnly here will force the Quobyte volume to be mounted with read-only permissions. Defaults to false. type: boolean registry: description: Registry represents a single or multiple Quobyte Registry services specified as a string as host:port pair (multiple entries are separated with commas) which acts as the central registry for volumes type: string tenant: description: Tenant owning the given Quobyte volume in the Backend Used with dynamically provisioned Quobyte volumes, value is set by the plugin type: string user: description: User to map volume access to Defaults to serivceaccount user type: string volume: description: Volume is a string that references an already created Quobyte volume by name. type: string required: - registry - volume type: object rbd: description: 'RBD represents a Rados Block Device mount on the host that shares a pod''s lifetime. More info: https://examples.k8s.io/volumes/rbd/README.md' properties: fsType: description: 'Filesystem type of the volume that you want to mount. Tip: Ensure that the filesystem type is supported by the host operating system. Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd TODO: how do we prevent errors in the filesystem from compromising the machine' type: string image: description: 'The rados image name. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' type: string keyring: description: 'Keyring is the path to key ring for RBDUser. Default is /etc/ceph/keyring. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' type: string monitors: description: 'A collection of Ceph monitors. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' items: type: string type: array pool: description: 'The rados pool name. Default is rbd. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' type: string readOnly: description: 'ReadOnly here will force the ReadOnly setting in VolumeMounts. Defaults to false. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' type: boolean secretRef: description: 'SecretRef is name of the authentication secret for RBDUser. If provided overrides keyring. Default is nil. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' properties: name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string type: object user: description: 'The rados user name. Default is admin. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' type: string required: - image - monitors type: object scaleIO: description: ScaleIO represents a ScaleIO persistent volume attached and mounted on Kubernetes nodes. properties: fsType: description: Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. "ext4", "xfs", "ntfs". Default is "xfs". type: string gateway: description: The host address of the ScaleIO API Gateway. type: string protectionDomain: description: The name of the ScaleIO Protection Domain for the configured storage. type: string readOnly: description: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts. type: boolean secretRef: description: SecretRef references to the secret for ScaleIO user and other sensitive information. If this is not provided, Login operation will fail. properties: name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string type: object sslEnabled: description: Flag to enable/disable SSL communication with Gateway, default false type: boolean storageMode: description: Indicates whether the storage for a volume should be ThickProvisioned or ThinProvisioned. Default is ThinProvisioned. type: string storagePool: description: The ScaleIO Storage Pool associated with the protection domain. type: string system: description: The name of the storage system as configured in ScaleIO. type: string volumeName: description: The name of a volume already created in the ScaleIO system that is associated with this volume source. type: string required: - gateway - secretRef - system type: object secret: description: 'Secret represents a secret that should populate this volume. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' properties: defaultMode: description: 'Optional: mode bits to use on created files by default. Must be a value between 0 and 0777. Defaults to 0644. Directories within the path are not affected by this setting. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.' format: int32 type: integer items: description: If unspecified, each key-value pair in the Data field of the referenced Secret will be projected into the volume as a file whose name is the key and content is the value. If specified, the listed keys will be projected into the specified paths, and unlisted keys will not be present. If a key is specified which is not present in the Secret, the volume setup will error unless it is marked optional. Paths must be relative and may not contain the '..' path or start with '..'. items: description: Maps a string key to a path within a volume. properties: key: description: The key to project. type: string mode: description: 'Optional: mode bits to use on this file, must be a value between 0 and 0777. If not specified, the volume defaultMode will be used. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.' format: int32 type: integer path: description: The relative path of the file to map the key to. May not be an absolute path. May not contain the path element '..'. May not start with the string '..'. type: string required: - key - path type: object type: array optional: description: Specify whether the Secret or its keys must be defined type: boolean secretName: description: 'Name of the secret in the pod''s namespace to use. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' type: string type: object storageos: description: StorageOS represents a StorageOS volume attached and mounted on Kubernetes nodes. properties: fsType: description: Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. type: string readOnly: description: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts. type: boolean secretRef: description: SecretRef specifies the secret to use for obtaining the StorageOS API credentials. If not specified, default values will be attempted. properties: name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' type: string type: object volumeName: description: VolumeName is the human-readable name of the StorageOS volume. Volume names are only unique within a namespace. type: string volumeNamespace: description: VolumeNamespace specifies the scope of the volume within StorageOS. If no namespace is specified then the Pod's namespace will be used. This allows the Kubernetes name scoping to be mirrored within StorageOS for tighter integration. Set VolumeName to any name to override the default behaviour. Set to "default" if you are not using namespaces within StorageOS. Namespaces that do not pre-exist within StorageOS will be created. type: string type: object vsphereVolume: description: VsphereVolume represents a vSphere volume attached and mounted on kubelets host machine properties: fsType: description: Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. type: string storagePolicyID: description: Storage Policy Based Management (SPBM) profile ID associated with the StoragePolicyName. type: string storagePolicyName: description: Storage Policy Based Management (SPBM) profile name. type: string volumePath: description: Path that identifies vSphere volume vmdk type: string required: - volumePath type: object required: - name type: object type: array type: object status: description: 'Most recent observed status of the ThanosRuler cluster. Read-only. Not included when requesting from the apiserver, only from the ThanosRuler Operator API itself. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#spec-and-status' properties: availableReplicas: description: Total number of available pods (ready for at least minReadySeconds) targeted by this ThanosRuler deployment. format: int32 type: integer paused: description: Represents whether any actions on the underlying managed objects are being performed. Only delete actions will be performed. type: boolean replicas: description: Total number of non-terminated pods targeted by this ThanosRuler deployment (their labels match the selector). format: int32 type: integer unavailableReplicas: description: Total number of unavailable pods targeted by this ThanosRuler deployment. format: int32 type: integer updatedReplicas: description: Total number of non-terminated pods targeted by this ThanosRuler deployment that have the desired version spec. format: int32 type: integer required: - availableReplicas - paused - replicas - unavailableReplicas - updatedReplicas type: object required: - spec type: object served: true storage: true status: acceptedNames: kind: "" plural: "" conditions: [] storedVersions: [] --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: labels: app.kubernetes.io/component: controller app.kubernetes.io/name: prometheus-operator app.kubernetes.io/version: 0.48.1 name: prometheus-operator roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: prometheus-operator subjects: - kind: ServiceAccount name: prometheus-operator namespace: prometheus --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: app.kubernetes.io/component: controller app.kubernetes.io/name: prometheus-operator app.kubernetes.io/version: 0.48.1 name: prometheus-operator rules: - apiGroups: - monitoring.coreos.com resources: - alertmanagers - alertmanagers/finalizers - alertmanagerconfigs - prometheuses - prometheuses/finalizers - thanosrulers - thanosrulers/finalizers - servicemonitors - podmonitors - probes - prometheusrules verbs: - '*' - apiGroups: - apps resources: - statefulsets verbs: - '*' - apiGroups: - "" resources: - configmaps - secrets verbs: - '*' - apiGroups: - "" resources: - pods verbs: - list - delete - apiGroups: - "" resources: - services - services/finalizers - endpoints verbs: - get - create - update - delete - apiGroups: - "" resources: - nodes verbs: - list - watch - apiGroups: - "" resources: - namespaces verbs: - get - list - watch - apiGroups: - networking.k8s.io resources: - ingresses verbs: - get - list - watch --- apiVersion: apps/v1 kind: Deployment metadata: labels: app.kubernetes.io/component: controller app.kubernetes.io/name: prometheus-operator app.kubernetes.io/version: 0.48.1 name: prometheus-operator namespace: prometheus spec: replicas: 1 selector: matchLabels: app.kubernetes.io/component: controller app.kubernetes.io/name: prometheus-operator template: metadata: annotations: kubectl.kubernetes.io/default-container: prometheus-operator labels: app.kubernetes.io/component: controller app.kubernetes.io/name: prometheus-operator app.kubernetes.io/version: 0.48.1 spec: containers: - args: - --kubelet-service=kube-system/kubelet - --prometheus-config-reloader=$CORTEX_IMAGE_PROMETHEUS_CONFIG_RELOADER image: $CORTEX_IMAGE_PROMETHEUS_OPERATOR name: prometheus-operator ports: - containerPort: 8080 name: http resources: limits: cpu: 200m memory: 200Mi requests: cpu: 100m memory: 100Mi securityContext: allowPrivilegeEscalation: false nodeSelector: kubernetes.io/os: linux prometheus: "true" tolerations: - key: prometheus operator: Exists effect: NoSchedule securityContext: runAsNonRoot: true runAsUser: 65534 serviceAccountName: prometheus-operator --- apiVersion: v1 kind: ServiceAccount metadata: labels: app.kubernetes.io/component: controller app.kubernetes.io/name: prometheus-operator app.kubernetes.io/version: 0.48.1 name: prometheus-operator namespace: prometheus --- apiVersion: v1 kind: Service metadata: labels: app.kubernetes.io/component: controller app.kubernetes.io/name: prometheus-operator app.kubernetes.io/version: 0.48.1 name: prometheus-operator namespace: prometheus spec: clusterIP: None ports: - name: http port: 8080 targetPort: http selector: app.kubernetes.io/component: controller app.kubernetes.io/name: prometheus-operator ================================================ FILE: manager/manifests/prometheus-statsd-exporter.yaml ================================================ # Copyright 2013 The Prometheus Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Modifications Copyright 2022 Cortex Labs, Inc. apiVersion: v1 kind: ConfigMap metadata: name: prometheus-statsd-exporter-config namespace: prometheus data: statsd-mapping.yaml: | defaults: observer_type: histogram --- apiVersion: apps/v1 kind: Deployment metadata: name: prometheus-statsd-exporter namespace: prometheus spec: replicas: 1 selector: matchLabels: name: prometheus-statsd-exporter template: metadata: labels: name: prometheus-statsd-exporter spec: containers: - name: prometheus-statsd-exporter image: $CORTEX_IMAGE_PROMETHEUS_STATSD_EXPORTER imagePullPolicy: Always args: - --web.listen-address=:9102 - --web.telemetry-path=/metrics - --statsd.listen-udp=:9125 - --statsd.listen-tcp=:9125 - --statsd.cache-size=1000 - --statsd.event-queue-size=10000 - --statsd.event-flush-threshold=1000 - --statsd.event-flush-interval=200ms - --statsd.mapping-config=/etc/prometheus-statsd-exporter/statsd-mapping.yaml ports: - name: metrics containerPort: 9102 protocol: TCP - name: statsd-udp containerPort: 9125 protocol: UDP livenessProbe: httpGet: path: /metrics port: metrics initialDelaySeconds: 30 periodSeconds: 30 resources: limits: memory: 100Mi requests: cpu: 100m memory: 100Mi volumeMounts: - name: statsd-mapping-config mountPath: /etc/prometheus-statsd-exporter nodeSelector: prometheus: "true" tolerations: - key: prometheus operator: Exists effect: NoSchedule volumes: - name: statsd-mapping-config configMap: name: prometheus-statsd-exporter-config items: - key: statsd-mapping.yaml path: statsd-mapping.yaml terminationGracePeriodSeconds: 60 --- apiVersion: v1 kind: Service metadata: namespace: prometheus name: prometheus-statsd-exporter labels: cortex.dev/name: prometheus-statsd-exporter spec: selector: name: prometheus-statsd-exporter ports: - port: 9125 name: statsd-udp protocol: UDP - port: 9102 name: metrics protocol: TCP ================================================ FILE: manager/refresh.sh ================================================ #!/bin/bash # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -e CORTEX_VERSION_MINOR=master cluster_config_out_path="$1" mkdir -p "$(dirname "$cluster_config_out_path")" if ! eksctl utils describe-stacks --cluster=$CORTEX_CLUSTER_NAME --region=$CORTEX_REGION >/dev/null 2>&1; then echo "error: there is no cluster named \"$CORTEX_CLUSTER_NAME\" in $CORTEX_REGION; please update your configuration to point to an existing cortex cluster or create a cortex cluster with \`cortex cluster up\`" exit 1 fi eksctl utils write-kubeconfig --cluster=$CORTEX_CLUSTER_NAME --region=$CORTEX_REGION --verbose=0 | (grep -v "saved kubeconfig as" || true) out=$(kubectl get pods 2>&1 || true); if [[ "$out" == *"must be logged in to the server"* ]]; then echo "error: your aws iam user does not have access to this cluster; to grant access, see https://docs.cortexlabs.com/v/${CORTEX_VERSION_MINOR}/"; exit 1; fi kubectl get -n=default configmap cluster-config -o json | jq -r '.data."cluster.yaml"' >> $cluster_config_out_path ================================================ FILE: manager/render_template.py ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import sys import yaml import os import pathlib from jinja2 import Environment, FileSystemLoader # python render_template.py [CLUSTER_CONFIG_PATH] TEMPLATE_PATH if __name__ == "__main__": if len(sys.argv) == 3: yaml_file_path = sys.argv[1] template_path = pathlib.Path(sys.argv[2]) elif len(sys.argv) == 2: yaml_file_path = None template_path = pathlib.Path(sys.argv[1]) else: raise RuntimeError(f"incorrect number of parameters ({len(sys.argv)})") file_loader = FileSystemLoader(str(template_path.parent)) env = Environment(loader=file_loader) env.trim_blocks = True env.lstrip_blocks = True env.rstrip_blocks = True template = env.get_template(str(template_path.name)) if yaml_file_path: with open(yaml_file_path, "r") as f: yaml_data = yaml.safe_load(f) print(template.render(config=yaml_data, env=os.environ)) else: print(template.render(env=os.environ)) ================================================ FILE: manager/requirements.txt ================================================ boto3 jinja2 pyyaml yq click==8.0.4 ================================================ FILE: manager/uninstall.sh ================================================ #!/bin/bash # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -e EKSCTL_TIMEOUT=45m function main() { echo aws eks --region $CORTEX_REGION update-kubeconfig --name $CORTEX_CLUSTER_NAME >/dev/null eksctl delete cluster --wait --name=$CORTEX_CLUSTER_NAME --region=$CORTEX_REGION --disable-nodegroup-eviction --timeout=$EKSCTL_TIMEOUT echo -e "\n✓ done spinning down the cluster" } main ================================================ FILE: manager/upgrade_kube_proxy_mode.py ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Usage: python create_user.py $KUBE_PROXY_CONFIG.yaml import yaml import sys def main(): kube_proxy_config_file = sys.argv[1] with open(kube_proxy_config_file, "r") as f: kube_proxy_config = yaml.safe_load(f) kube_proxy_config["mode"] = "ipvs" # IP Virtual Server kube_proxy_config["ipvs"]["scheduler"] = "rr" # round robin print(yaml.dump(kube_proxy_config, indent=2)) if __name__ == "__main__": main() ================================================ FILE: pkg/activator/activator.go ================================================ /* Copyright 2018 The Knative Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. Modifications Copyright 2022 Cortex Labs, Inc. */ package activator import ( "context" "sync" "github.com/cortexlabs/cortex/pkg/autoscaler" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/telemetry" "github.com/cortexlabs/cortex/pkg/types/userconfig" "github.com/cortexlabs/cortex/pkg/workloads" "go.uber.org/zap" istionetworkingclient "istio.io/client-go/pkg/clientset/versioned/typed/networking/v1beta1" kapps "k8s.io/api/apps/v1" kmeta "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/cache" ) type ctxValue string const APINameCtxKey ctxValue = "apiName" type StatsReporter interface { AddAPI(apiName string) RemoveAPI(apiName string) } type Activator interface { Try(ctx context.Context, fn func() error) error } type activator struct { activatorsMux sync.Mutex trackersMux sync.Mutex autoscalerClient autoscaler.Client apiActivators map[string]*apiActivator readinessTrackers map[string]*readinessTracker istioClient istionetworkingclient.VirtualServiceInterface reporter StatsReporter logger *zap.SugaredLogger } func New( istioClient istionetworkingclient.VirtualServiceInterface, deploymentInformer cache.SharedIndexInformer, virtualServiceInformer cache.SharedIndexInformer, autoscalerClient autoscaler.Client, reporter StatsReporter, logger *zap.SugaredLogger, ) Activator { log := logger.With(zap.String("apiKind", userconfig.RealtimeAPIKind.String())) act := &activator{ apiActivators: make(map[string]*apiActivator), readinessTrackers: make(map[string]*readinessTracker), istioClient: istioClient, logger: log, autoscalerClient: autoscalerClient, reporter: reporter, } virtualServiceInformer.AddEventHandler( cache.ResourceEventHandlerFuncs{ AddFunc: act.addAPI, UpdateFunc: act.updateAPI, DeleteFunc: act.removeAPI, }, ) deploymentInformer.AddEventHandler( cache.ResourceEventHandlerFuncs{ AddFunc: act.updateReadinessTracker, UpdateFunc: func(_, newObj interface{}) { act.updateReadinessTracker(newObj) }, DeleteFunc: act.removeReadinessTracker, }, ) return act } func (a *activator) Try(ctx context.Context, fn func() error) error { apiNameValue := ctx.Value(APINameCtxKey) apiName, ok := apiNameValue.(string) if !ok || apiName == "" { return errors.ErrorUnexpected("failed to get the api name from context") } act, err := a.getOrCreateAPIActivator(ctx, apiName) if err != nil { return err } tracker := a.getOrCreateReadinessTracker(apiName) if act.inFlight() == 0 { go a.awakenAPI(apiName) } return act.try(ctx, fn, tracker) } func (a *activator) getOrCreateAPIActivator(ctx context.Context, apiName string) (*apiActivator, error) { a.activatorsMux.Lock() defer a.activatorsMux.Unlock() act, ok := a.apiActivators[apiName] if ok { return act, nil } vs, err := a.istioClient.Get(ctx, workloads.K8sName(apiName), kmeta.GetOptions{}) if err != nil { return nil, errors.WithStack(err) } maxQueueLength, maxConcurrency, err := userconfig.ConcurrencyFromAnnotations(vs) if err != nil { return nil, err } apiAct := newAPIActivator(maxQueueLength, maxConcurrency) a.apiActivators[apiName] = apiAct return apiAct, nil } func (a *activator) getOrCreateReadinessTracker(apiName string) *readinessTracker { a.trackersMux.Lock() defer a.trackersMux.Unlock() tracker, ok := a.readinessTrackers[apiName] if ok { return tracker } tracker = newReadinessTracker() a.readinessTrackers[apiName] = tracker return tracker } func (a *activator) addAPI(obj interface{}) { apiMetadata, err := getAPIMeta(obj) if err != nil { a.logger.Errorw("error during virtual service informer add callback", zap.Error(err)) telemetry.Error(err) return } if apiMetadata.apiKind != userconfig.RealtimeAPIKind { return } apiName := apiMetadata.apiName a.activatorsMux.Lock() if a.apiActivators[apiName] == nil { a.logger.Debugw("adding new api activator", zap.String("apiName", apiName)) a.apiActivators[apiName] = newAPIActivator(apiMetadata.maxQueueLength, apiMetadata.maxConcurrency) } a.activatorsMux.Unlock() a.reporter.AddAPI(apiName) } func (a *activator) updateAPI(oldObj interface{}, newObj interface{}) { apiMetadata, err := getAPIMeta(newObj) if err != nil { a.logger.Errorw("error during virtual service informer update callback", zap.Error(err)) telemetry.Error(err) return } if apiMetadata.apiKind != userconfig.RealtimeAPIKind { return } apiName := apiMetadata.apiName oldAPIMetatada, err := getAPIMeta(oldObj) if err != nil { a.logger.Errorw("error during virtual service informer update callback", zap.Error(err)) telemetry.Error(err) return } if oldAPIMetatada.maxConcurrency != apiMetadata.maxConcurrency || oldAPIMetatada.maxQueueLength != apiMetadata.maxQueueLength { a.logger.Debugw("updating api activator", zap.String("apiName", apiName)) a.activatorsMux.Lock() a.apiActivators[apiName].updateQueueParams(apiMetadata.maxQueueLength, apiMetadata.maxConcurrency) a.activatorsMux.Unlock() } } func (a *activator) removeAPI(obj interface{}) { apiMetadata, err := getAPIMeta(obj) if err != nil { a.logger.Errorw("error during virtual service informer delete callback", zap.Error(err)) telemetry.Error(err) return } if apiMetadata.apiKind != userconfig.RealtimeAPIKind { return } a.logger.Debugw("deleting api activator", zap.String("apiName", apiMetadata.apiName)) a.activatorsMux.Lock() delete(a.apiActivators, apiMetadata.apiName) a.activatorsMux.Unlock() a.reporter.RemoveAPI(apiMetadata.apiName) } func (a *activator) awakenAPI(apiName string) { err := a.autoscalerClient.Awaken( userconfig.Resource{ Name: apiName, Kind: userconfig.RealtimeAPIKind, // only realtime apis are supported as of now, so we can assume the kind }, ) if err != nil { a.logger.Errorw("failed to awake api", zap.Error(err), zap.String("apiName", apiName)) } } func (a *activator) updateReadinessTracker(obj interface{}) { deployment, ok := obj.(*kapps.Deployment) if !ok { return } api, err := getAPIMeta(obj) if err != nil { a.logger.Errorw("error during deployment informer callback", zap.Error(err)) telemetry.Error(err) return } if api.apiKind != userconfig.RealtimeAPIKind { return } tracker := a.getOrCreateReadinessTracker(api.apiName) tracker.Update(deployment) a.logger.Debugw("updated readiness tracker", zap.Bool("ready", deployment.Status.ReadyReplicas > 0), zap.String("apiName", api.apiName), ) } func (a *activator) removeReadinessTracker(obj interface{}) { api, err := getAPIMeta(obj) if err != nil { a.logger.Errorw("error during deployment informer callback", zap.Error(err)) telemetry.Error(err) return } if api.apiKind != userconfig.RealtimeAPIKind { return } a.trackersMux.Lock() defer a.trackersMux.Unlock() delete(a.readinessTrackers, api.apiName) } ================================================ FILE: pkg/activator/activator_test.go ================================================ /* Copyright 2018 The Knative Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. Modifications Copyright 2022 Cortex Labs, Inc. */ package activator import ( "context" "errors" "testing" "time" "github.com/cortexlabs/cortex/pkg/proxy" "github.com/cortexlabs/cortex/pkg/types/userconfig" "github.com/stretchr/testify/require" "go.uber.org/zap" ) type autoscalerClientMock struct{} func (m autoscalerClientMock) AddAPI(_ userconfig.Resource) error { return nil } func (m autoscalerClientMock) Awaken(_ userconfig.Resource) error { return nil } func newLogger(t *testing.T) *zap.SugaredLogger { t.Helper() config := zap.NewDevelopmentConfig() config.Level = zap.NewAtomicLevelAt(zap.ErrorLevel) logger, err := config.Build() require.NoError(t, err) logr := logger.Sugar() return logr } func TestActivator_Try(t *testing.T) { t.Parallel() log := newLogger(t) apiName := "test" act := &activator{ autoscalerClient: autoscalerClientMock{}, apiActivators: map[string]*apiActivator{ apiName: newAPIActivator(1, 1), }, readinessTrackers: map[string]*readinessTracker{ apiName: {ready: true}, }, logger: log, } ctx := context.Background() ctx = context.WithValue(ctx, APINameCtxKey, apiName) errCh := make(chan error) waitCh := make(chan struct{}) for i := 0; i < 3; i++ { go func() { ctx, cancel := context.WithTimeout(ctx, time.Second) defer cancel() errCh <- act.Try(ctx, func() error { <-waitCh return nil }) }() } err := <-errCh require.Error(t, err) require.True(t, errors.Is(err, proxy.ErrRequestQueueFull)) for i := 0; i < 2; i++ { waitCh <- struct{}{} require.NoError(t, <-errCh) } } ================================================ FILE: pkg/activator/api_activator.go ================================================ /* Copyright 2018 The Knative Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. Modifications Copyright 2022 Cortex Labs, Inc. */ package activator import ( "context" "sync" "github.com/cortexlabs/cortex/pkg/consts" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/proxy" kapps "k8s.io/api/apps/v1" ) type apiActivator struct { breaker *proxy.Breaker } func newAPIActivator(maxQueueLength, maxConcurrency int) *apiActivator { breaker := proxy.NewBreaker(proxy.BreakerParams{ QueueDepth: maxQueueLength, MaxConcurrency: maxConcurrency, InitialCapacity: maxConcurrency, }) return &apiActivator{breaker: breaker} } // try waits for the readinessTracker to be ready and then attempts to execute the passed callback. // If the readinessTracker does not reach a ready state, it will timeout. func (a *apiActivator) try(ctx context.Context, fn func() error, tracker *readinessTracker) error { var execErr error if err := a.breaker.Maybe(ctx, func() { ctx, cancel := context.WithTimeout(ctx, consts.WaitForReadyReplicasTimeout) defer cancel() if !tracker.IsReady() { loop: for { select { case <-ctx.Done(): execErr = errors.Wrap(ctx.Err(), "no ready replicas available") return case <-tracker.Wait(): break loop } } } execErr = fn() }); err != nil { return err } return execErr } // updateQueueParams updates the breaker queue parameters (not thread safe) func (a *apiActivator) updateQueueParams(maxQueueLength, maxConcurrency int) { a.breaker.UpdateConcurrency(maxConcurrency) a.breaker.UpdateQueueLength(maxQueueLength) } // inFlight returns the amount of in-flight requests of the breaker func (a *apiActivator) inFlight() int64 { return a.breaker.InFlight() } type readinessTracker struct { mux sync.RWMutex c chan bool ready bool } func newReadinessTracker() *readinessTracker { return &readinessTracker{ c: make(chan bool), } } func (t *readinessTracker) Update(deployment *kapps.Deployment) { t.mux.Lock() defer t.mux.Unlock() // if changing from unready to ready state, wake up waiting goroutines if !t.ready && deployment.Status.ReadyReplicas > 0 { close(t.c) // wake up all of the goroutines waiting on this channel t.c = make(chan bool) // make a new channel ready to be used } t.ready = deployment.Status.ReadyReplicas > 0 } func (t *readinessTracker) IsReady() bool { t.mux.RLock() defer t.mux.RUnlock() return t.ready } func (t *readinessTracker) Wait() chan bool { t.mux.RLock() defer t.mux.RUnlock() return t.c } ================================================ FILE: pkg/activator/api_activator_test.go ================================================ /* Copyright 2018 The Knative Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. Modifications Copyright 2022 Cortex Labs, Inc. */ package activator import ( "context" "testing" "github.com/cortexlabs/cortex/pkg/proxy" "github.com/pkg/errors" "github.com/stretchr/testify/require" ) func TestApiActivator_Try(t *testing.T) { t.Parallel() act := newAPIActivator(1, 1) errCh := make(chan error) waitCh := make(chan struct{}) for i := 0; i < 3; i++ { go func() { errCh <- act.try(context.Background(), func() error { <-waitCh return nil }, &readinessTracker{ready: true}) }() } err := <-errCh require.Error(t, err) require.True(t, errors.Is(err, proxy.ErrRequestQueueFull)) for i := 0; i < 2; i++ { waitCh <- struct{}{} require.NoError(t, <-errCh) } } ================================================ FILE: pkg/activator/handler.go ================================================ /* Copyright 2018 The Knative Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. Modifications Copyright 2022 Cortex Labs, Inc. */ package activator import ( "context" stderrors "errors" "fmt" "net/http" "net/http/httputil" "net/url" "github.com/cortexlabs/cortex/pkg/consts" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/telemetry" "github.com/cortexlabs/cortex/pkg/proxy" "go.uber.org/zap" ) type Handler struct { activator Activator logger *zap.SugaredLogger } func NewHandler(act Activator, logger *zap.SugaredLogger) *Handler { return &Handler{ activator: act, logger: logger, } } func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { if hasCortexProbeHeader(r) { w.WriteHeader(http.StatusOK) return } apiName := r.Header.Get(consts.CortexAPINameHeader) if apiName == "" { http.Error(w, fmt.Sprintf("missing %s", consts.CortexAPINameHeader), http.StatusInternalServerError) return } ctx := r.Context() ctx = context.WithValue(ctx, APINameCtxKey, apiName) if err := h.activator.Try(ctx, func() error { return h.proxyRequest(w, r) }); err != nil { h.logger.Errorw("activator try error", zap.Error(err)) if stderrors.Is(err, context.DeadlineExceeded) || stderrors.Is(err, proxy.ErrRequestQueueFull) { http.Error(w, err.Error(), http.StatusServiceUnavailable) } else { w.WriteHeader(http.StatusInternalServerError) telemetry.Error(err) } } } func (h *Handler) proxyRequest(w http.ResponseWriter, r *http.Request) error { target := r.Header.Get(consts.CortexTargetServiceHeader) if target == "" { return errors.ErrorUnexpected("missing header", consts.CortexTargetServiceHeader) } targetURL, err := url.Parse(target) if err != nil { return errors.WithStack(err) } // delete activator specific headers r.Header.Del(consts.CortexAPINameHeader) r.Header.Del(consts.CortexTargetServiceHeader) reverseProxy := httputil.NewSingleHostReverseProxy(targetURL) reverseProxy.ServeHTTP(w, r) return nil } func hasCortexProbeHeader(r *http.Request) bool { return r.Header.Get(consts.CortexProbeHeader) != "" } ================================================ FILE: pkg/activator/handler_test.go ================================================ /* Copyright 2018 The Knative Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. Modifications Copyright 2022 Cortex Labs, Inc. */ package activator import ( "net/http" "net/http/httptest" "testing" "github.com/cortexlabs/cortex/pkg/consts" "github.com/stretchr/testify/require" ) func TestActivatorHandler_ServeHTTP(t *testing.T) { t.Parallel() log := newLogger(t) apiName := "test" act := &activator{ autoscalerClient: autoscalerClientMock{}, apiActivators: map[string]*apiActivator{ apiName: newAPIActivator(1, 1), }, readinessTrackers: map[string]*readinessTracker{ apiName: {ready: true}, }, logger: log, } ah := NewHandler(act, log) var callCount int server := httptest.NewServer( http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { callCount++ w.WriteHeader(http.StatusOK) }), ) w := httptest.NewRecorder() r := httptest.NewRequest("POST", "http://fake.cortex.dev/api", nil) r.Header.Set(consts.CortexAPINameHeader, apiName) r.Header.Set(consts.CortexTargetServiceHeader, server.URL) ah.ServeHTTP(w, r) require.Equal(t, http.StatusOK, w.Code) require.Equal(t, 1, callCount) } ================================================ FILE: pkg/activator/helpers.go ================================================ /* Copyright 2018 The Knative Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. Modifications Copyright 2022 Cortex Labs, Inc. */ package activator import ( "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/types/userconfig" "k8s.io/apimachinery/pkg/api/meta" ) type apiMeta struct { apiName string apiKind userconfig.Kind labels map[string]string annotations map[string]string maxConcurrency int maxQueueLength int } func getAPIMeta(obj interface{}) (apiMeta, error) { resource, err := meta.Accessor(obj) if err != nil { return apiMeta{}, err } labels := resource.GetLabels() apiKind, ok := labels["apiKind"] if !ok { return apiMeta{}, err } apiName, ok := labels["apiName"] if !ok { return apiMeta{}, errors.ErrorUnexpected("got a virtual service without apiName label") } maxQueueLength, maxConcurrency, err := userconfig.ConcurrencyFromAnnotations(resource) if err != nil { return apiMeta{}, err } return apiMeta{ apiName: apiName, apiKind: userconfig.KindFromString(apiKind), labels: labels, annotations: resource.GetAnnotations(), maxConcurrency: maxConcurrency, maxQueueLength: maxQueueLength, }, nil } ================================================ FILE: pkg/activator/request_stats.go ================================================ /* Copyright 2018 The Knative Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. Modifications Copyright 2022 Cortex Labs, Inc. */ package activator import ( "net/http" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/client_golang/prometheus/promhttp" ) type PrometheusStatsReporter struct { handler http.Handler inFlightRequests *prometheus.GaugeVec } func NewPrometheusStatsReporter() *PrometheusStatsReporter { inFlightRequestsGauge := promauto.NewGaugeVec(prometheus.GaugeOpts{ Name: "cortex_in_flight_requests", Help: "The number of in-flight requests for a cortex API", }, []string{"api_name"}) return &PrometheusStatsReporter{ handler: promhttp.Handler(), inFlightRequests: inFlightRequestsGauge, } } func (r *PrometheusStatsReporter) AddAPI(apiName string) { r.inFlightRequests.WithLabelValues(apiName).Set(0) } func (r *PrometheusStatsReporter) RemoveAPI(apiName string) { r.inFlightRequests.DeleteLabelValues(apiName) } func (r *PrometheusStatsReporter) ServeHTTP(w http.ResponseWriter, req *http.Request) { r.handler.ServeHTTP(w, req) } ================================================ FILE: pkg/async-gateway/endpoint.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package gateway import ( "encoding/json" "fmt" "net/http" "github.com/cortexlabs/cortex/pkg/consts" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/telemetry" "github.com/cortexlabs/cortex/pkg/types/async" "github.com/gorilla/mux" "go.uber.org/zap" ) // Endpoint wraps an async-gateway Service with HTTP logic type Endpoint struct { service Service logger *zap.SugaredLogger } // NewEndpoint creates and initializes a new Endpoint struct func NewEndpoint(svc Service, logger *zap.SugaredLogger) *Endpoint { return &Endpoint{ service: svc, logger: logger, } } // CreateWorkload is a handler for the async-gateway service workload creation route func (e *Endpoint) CreateWorkload(w http.ResponseWriter, r *http.Request) { requestID := r.Header.Get("x-request-id") if requestID == "" { respondPlainText(w, http.StatusBadRequest, "error: missing x-request-id key in request header") return } apiName := r.Header.Get(consts.CortexAPINameHeader) if requestID == "" { respondPlainText(w, http.StatusBadRequest, fmt.Sprintf("error: missing %s key in request header", consts.CortexAPINameHeader)) return } r.Header.Del(consts.CortexAPINameHeader) queueURL := r.Header.Get(consts.CortexQueueURLHeader) if queueURL == "" { respondPlainText(w, http.StatusBadRequest, fmt.Sprintf("error: missing %s key in request header", consts.CortexQueueURLHeader)) return } r.Header.Del(consts.CortexQueueURLHeader) body := r.Body defer func() { _ = r.Body.Close() }() log := e.logger.With(zap.String("id", requestID), zap.String("apiName", apiName)) id, err := e.service.CreateWorkload(requestID, apiName, queueURL, body, r.Header) if err != nil { respondPlainText(w, http.StatusInternalServerError, fmt.Sprintf("error: %v", err)) logErrorWithTelemetry(log, errors.Wrap(err, "failed to create workload")) return } if err = respondJSON(w, http.StatusOK, CreateWorkloadResponse{ID: id}); err != nil { logErrorWithTelemetry(log, errors.Wrap(err, "failed to encode json response")) return } } // GetWorkload is a handler for the async-gateway service workload retrieval route func (e *Endpoint) GetWorkload(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) id, ok := vars["id"] if !ok { respondPlainText(w, http.StatusBadRequest, "error: missing request id in url path") return } apiName := r.Header.Get(consts.CortexAPINameHeader) if apiName == "" { respondPlainText(w, http.StatusBadRequest, fmt.Sprintf("error: missing %s key in request header", consts.CortexAPINameHeader)) return } r.Header.Del(consts.CortexAPINameHeader) log := e.logger.With(zap.String("id", id), zap.String("apiName", apiName)) res, err := e.service.GetWorkload(id, apiName) if err != nil { respondPlainText(w, http.StatusInternalServerError, fmt.Sprintf("error: %v", err)) logErrorWithTelemetry(log, errors.Wrap(err, "failed to get workload")) return } if res.Status == async.StatusNotFound { respondPlainText(w, http.StatusNotFound, fmt.Sprintf("error: id %s not found", res.ID)) logErrorWithTelemetry(log, errors.ErrorUnexpected(fmt.Sprintf("error: id %s not found", res.ID))) return } if err = respondJSON(w, http.StatusOK, res); err != nil { logErrorWithTelemetry(log, errors.Wrap(err, "failed to encode json response")) return } } func respondPlainText(w http.ResponseWriter, statusCode int, message string) { w.Header().Set("Content-Type", "text/plain") w.WriteHeader(statusCode) _, _ = w.Write([]byte(message)) } func respondJSON(w http.ResponseWriter, statusCode int, s interface{}) error { w.Header().Set("Content-Type", "application/json") w.WriteHeader(statusCode) return json.NewEncoder(w).Encode(s) } func logErrorWithTelemetry(log *zap.SugaredLogger, err error) { telemetry.Error(err) log.Error(err) } ================================================ FILE: pkg/async-gateway/queue.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package gateway import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" awssqs "github.com/aws/aws-sdk-go/service/sqs" ) // Queue is an interface to abstract communication with event queues type Queue interface { SendMessage(message string, uniqueID string) error } type sqs struct { queueURL string client *awssqs.SQS } // NewSQS creates a new SQS client that satisfies the Queue interface func NewSQS(queueURL string, sess *session.Session) Queue { client := awssqs.New(sess) return &sqs{queueURL: queueURL, client: client} } // SendMessage sends a string func (q *sqs) SendMessage(message string, uniqueID string) error { _, err := q.client.SendMessage(&awssqs.SendMessageInput{ MessageBody: aws.String(message), MessageDeduplicationId: aws.String(uniqueID), MessageGroupId: aws.String(uniqueID), QueueUrl: aws.String(q.queueURL), }) return err } ================================================ FILE: pkg/async-gateway/service.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package gateway import ( "bytes" "encoding/json" "fmt" "io" "net/http" "strings" "github.com/aws/aws-sdk-go/aws/session" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/types/async" "go.uber.org/zap" ) // Service provides an interface to the async-gateway business logic type Service interface { CreateWorkload(id string, apiName string, queueURL string, payload io.Reader, headers http.Header) (string, error) GetWorkload(id string, apiName string) (GetWorkloadResponse, error) } type service struct { logger *zap.SugaredLogger storage Storage clusterUID string session session.Session } // NewService creates a new async-gateway service func NewService(clusterUID string, storage Storage, logger *zap.SugaredLogger, session session.Session) Service { return &service{ logger: logger, storage: storage, clusterUID: clusterUID, session: session, } } // CreateWorkload enqueues an async workload request and uploads the request payload to S3 func (s *service) CreateWorkload(id string, apiName string, queueURL string, payload io.Reader, headers http.Header) (string, error) { prefix := async.StoragePath(s.clusterUID, apiName) log := s.logger.With(zap.String("id", id), zap.String("apiName", apiName)) buf := &bytes.Buffer{} if err := json.NewEncoder(buf).Encode(headers); err != nil { return "", errors.Wrap(err, "failed to dump headers") } headersPath := async.HeadersPath(prefix, id) log.Debugw("uploading headers", zap.String("path", headersPath)) if err := s.storage.Upload(headersPath, buf, "application/json"); err != nil { return "", errors.Wrap(err, "failed to upload headers") } contentType := headers.Get("Content-Type") payloadPath := async.PayloadPath(prefix, id) log.Debugw("uploading payload", zap.String("path", payloadPath)) if err := s.storage.Upload(payloadPath, payload, contentType); err != nil { return "", errors.Wrap(err, "failed to upload payload") } log.Debug("sending message to queue") queue := NewSQS(queueURL, &s.session) if err := queue.SendMessage(id, id); err != nil { return "", errors.Wrap(err, "failed to send message to queue") } statusPath := fmt.Sprintf("%s/%s/status/%s", prefix, id, async.StatusInQueue) log.Debug(fmt.Sprintf("setting status to %s", async.StatusInQueue)) if err := s.storage.Upload(statusPath, strings.NewReader(""), "text/plain"); err != nil { return "", errors.Wrap(err, "failed to upload workload status") } return id, nil } // GetWorkload retrieves the status and result, if available, of a given workload func (s *service) GetWorkload(id string, apiName string) (GetWorkloadResponse, error) { log := s.logger.With(zap.String("id", id), zap.String("apiName", apiName)) st, err := s.getStatus(id, apiName) if err != nil { return GetWorkloadResponse{}, err } if st != async.StatusCompleted { return GetWorkloadResponse{ ID: id, Status: st, }, nil } // attempt to download user result prefix := async.StoragePath(s.clusterUID, apiName) resultPath := async.ResultPath(prefix, id) log.Debug("downloading user result", zap.String("path", resultPath)) resultBuf, err := s.storage.Download(resultPath) if err != nil { return GetWorkloadResponse{}, err } var userResponse UserResponse if err = json.Unmarshal(resultBuf, &userResponse); err != nil { return GetWorkloadResponse{}, err } log.Debug("getting workload timestamp") timestamp, err := s.storage.GetLastModified(resultPath) if err != nil { return GetWorkloadResponse{}, err } return GetWorkloadResponse{ ID: id, Status: st, Result: &userResponse, Timestamp: ×tamp, }, nil } func (s *service) getStatus(id string, apiName string) (async.Status, error) { prefix := async.StoragePath(s.clusterUID, apiName) log := s.logger.With(zap.String("id", id)) // download workload status statusPrefixPath := async.StatusPrefixPath(prefix, id) log.Debug("checking status", zap.String("path", statusPrefixPath)) files, err := s.storage.List(statusPrefixPath) if err != nil { return "", err } if len(files) == 0 { return async.StatusNotFound, nil } // determine request status st := async.StatusInQueue for _, file := range files { fileStatus := async.Status(file) if !fileStatus.Valid() { st = fileStatus return "", fmt.Errorf("invalid workload status: %s", st) } if fileStatus == async.StatusInProgress { st = fileStatus } if fileStatus == async.StatusCompleted || fileStatus == async.StatusFailed { st = fileStatus break } } return st, nil } ================================================ FILE: pkg/async-gateway/storage.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package gateway import ( "io" "path" "strings" "time" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" awss3 "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3/s3manager" ) // Storage is an interface that abstracts cloud storage uploading type Storage interface { Upload(key string, payload io.Reader, contentType string) error Download(key string) ([]byte, error) List(key string) ([]string, error) GetLastModified(key string) (time.Time, error) } type s3 struct { uploader *s3manager.Uploader downloader *s3manager.Downloader client *awss3.S3 bucket string } // NewS3 creates a new S3 client that satisfies the Storage interface func NewS3(sess *session.Session, bucket string) Storage { uploader := s3manager.NewUploader(sess) downloader := s3manager.NewDownloader(sess) client := awss3.New(sess) return &s3{ uploader: uploader, bucket: bucket, downloader: downloader, client: client, } } // Upload uploads binary data to S3 func (s *s3) Upload(key string, payload io.Reader, contentType string) error { _, err := s.uploader.Upload(&s3manager.UploadInput{ Key: aws.String(key), Bucket: aws.String(s.bucket), ContentType: aws.String(contentType), Body: payload, }) return err } // Download downloads a file from S3 into memory func (s *s3) Download(key string) ([]byte, error) { buff := &aws.WriteAtBuffer{} input := awss3.GetObjectInput{ Key: aws.String(key), Bucket: aws.String(s.bucket), } _, err := s.downloader.Download(buff, &input) if err != nil { return nil, err } return buff.Bytes(), nil } // List lists a set of files from a given S3 path. // Works only for one level deep sub-paths. func (s *s3) List(key string) ([]string, error) { if key != "" && !strings.HasSuffix(key, "/") { key += "/" } result, err := s.client.ListObjectsV2(&awss3.ListObjectsV2Input{ Prefix: aws.String(key), Bucket: aws.String(s.bucket), }) if err != nil { return nil, err } files := []string{} for _, obj := range result.Contents { _, file := path.Split(*obj.Key) files = append(files, file) } return files, nil } // GetLastModified retrieves the last modified timestamp of an S3 object func (s *s3) GetLastModified(key string) (time.Time, error) { input := awss3.GetObjectInput{ Key: aws.String(key), Bucket: aws.String(s.bucket), } obj, err := s.client.GetObject(&input) if err != nil { return time.Time{}, err } return *obj.LastModified, nil } ================================================ FILE: pkg/async-gateway/types.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package gateway import ( "time" "github.com/cortexlabs/cortex/pkg/types/async" ) // UserResponse represents the user's API response, which has to be JSON serializable type UserResponse = map[string]interface{} //CreateWorkloadResponse represents the response returned to the user on workload creation type CreateWorkloadResponse struct { ID string `json:"id"` } // GetWorkloadResponse represents the workload response that is returned to the user type GetWorkloadResponse struct { ID string `json:"id"` Status async.Status `json:"status"` Result *UserResponse `json:"result,omitempty"` Timestamp *time.Time `json:"timestamp,omitempty"` } ================================================ FILE: pkg/autoscaler/async_scaler.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package autoscaler import ( "context" "fmt" "time" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/k8s" "github.com/cortexlabs/cortex/pkg/lib/pointer" "github.com/cortexlabs/cortex/pkg/types/userconfig" "github.com/cortexlabs/cortex/pkg/workloads" promv1 "github.com/prometheus/client_golang/api/prometheus/v1" "github.com/prometheus/common/model" ) type AsyncScaler struct { k8s *k8s.Client prometheus promv1.API } func NewAsyncScaler(k8sClient *k8s.Client, promClient promv1.API) *AsyncScaler { return &AsyncScaler{ k8s: k8sClient, prometheus: promClient, } } func (s *AsyncScaler) Scale(apiName string, request int32) error { deployment, err := s.k8s.GetDeployment(workloads.K8sName(apiName)) if err != nil { return err } if deployment == nil { return errors.ErrorUnexpected("unable to find k8s deployment", apiName) } if deployment.Spec.Replicas != nil && *deployment.Spec.Replicas == request { return nil } deployment.Spec.Replicas = pointer.Int32(request) if _, err = s.k8s.UpdateDeployment(deployment); err != nil { return err } return nil } func (s *AsyncScaler) GetInFlightRequests(apiName string, window time.Duration) (*float64, error) { windowSeconds := int64(window.Seconds()) // PromQL query: // sum(sum_over_time(cortex_async_in_flight{api_name=""}[60s])) / // sum(count_over_time(cortex_async_in_flight{api_name=""}[60s])) query := fmt.Sprintf( "sum(sum_over_time(cortex_async_in_flight{api_name=\"%s\"}[%ds])) / "+ "max(count_over_time(cortex_async_in_flight{api_name=\"%s\"}[%ds]))", apiName, windowSeconds, apiName, windowSeconds, ) ctx, cancel := context.WithTimeout(context.Background(), _prometheusQueryTimeoutSeconds*time.Second) defer cancel() valuesQuery, _, err := s.prometheus.Query(ctx, query, time.Now()) if err != nil { return nil, err } values, ok := valuesQuery.(model.Vector) if !ok { return nil, errors.ErrorUnexpected("failed to convert prometheus metric to vector") } if values.Len() != 0 { return pointer.Float64(float64(values[0].Value)), nil } return nil, nil } func (s *AsyncScaler) GetAutoscalingSpec(apiName string) (*userconfig.Autoscaling, error) { deployment, err := s.k8s.GetDeployment(workloads.K8sName(apiName)) if err != nil { return nil, err } if deployment == nil { return nil, errors.ErrorUnexpected("unable to find k8s deployment", apiName) } autoscalingSpec, err := userconfig.AutoscalingFromAnnotations(deployment) if err != nil { return nil, err } return autoscalingSpec, nil } func (s *AsyncScaler) CurrentRequestedReplicas(apiName string) (int32, error) { deployment, err := s.k8s.GetDeployment(workloads.K8sName(apiName)) if err != nil { return 0, err } if deployment == nil { return 0, errors.ErrorUnexpected("unable to find k8s deployment", apiName) } if deployment.Spec.Replicas == nil { return 0, errors.ErrorUnexpected("k8s deployment doesn't have the replicas field set", apiName) } return *deployment.Spec.Replicas, nil } ================================================ FILE: pkg/autoscaler/autoscaler.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package autoscaler import ( "fmt" "math" "time" "github.com/cortexlabs/cortex/pkg/lib/cron" "github.com/cortexlabs/cortex/pkg/lib/errors" libmath "github.com/cortexlabs/cortex/pkg/lib/math" "github.com/cortexlabs/cortex/pkg/lib/pointer" "github.com/cortexlabs/cortex/pkg/lib/telemetry" libtime "github.com/cortexlabs/cortex/pkg/lib/time" "github.com/cortexlabs/cortex/pkg/types/spec" "github.com/cortexlabs/cortex/pkg/types/userconfig" "go.uber.org/zap" ) const ( _prometheusQueryTimeoutSeconds = 10 ) type Scaler interface { Scale(apiName string, request int32) error GetInFlightRequests(apiName string, window time.Duration) (*float64, error) GetAutoscalingSpec(apiName string) (*userconfig.Autoscaling, error) CurrentRequestedReplicas(apiName string) (int32, error) } type Autoscaler struct { logger *zap.SugaredLogger crons map[string]cron.Cron scalers map[userconfig.Kind]Scaler recs map[string]*recommendations } func New(logger *zap.SugaredLogger) *Autoscaler { return &Autoscaler{ logger: logger, crons: make(map[string]cron.Cron), scalers: make(map[userconfig.Kind]Scaler), recs: make(map[string]*recommendations), } } func (a *Autoscaler) AddScaler(scaler Scaler, kind userconfig.Kind) { a.scalers[kind] = scaler } func (a *Autoscaler) Awaken(api userconfig.Resource) error { scaler, ok := a.scalers[api.Kind] if !ok { return errors.ErrorUnexpected( fmt.Sprintf("autoscaler does not have a scaler for the %s kind", api.Kind), ) } log := a.logger.With( zap.String("apiName", api.Name), zap.String("apiKind", api.Kind.String()), ) currentRequestedReplicas, err := scaler.CurrentRequestedReplicas(api.Name) if err != nil { return errors.Wrap(err, "failed to get current replicas") } if currentRequestedReplicas > 0 { return nil } log.Infof("autoscaling awake event") if err := scaler.Scale(api.Name, 1); err != nil { return errors.Wrap(err, "failed to scale api to one") } a.recs[api.Name].add(1) return nil } func (a *Autoscaler) AddAPI(api userconfig.Resource) error { if _, ok := a.crons[api.Name]; ok { return nil } errorHandler := func(err error) { log := a.logger.With( zap.String("apiName", api.Name), zap.String("apiKind", api.Kind.String()), ) log.Error(err) telemetry.Error(err) } autoscaleFn, err := a.autoscaleFn(api) if err != nil { return err } a.crons[api.Name] = cron.Run(autoscaleFn, errorHandler, spec.AutoscalingTickInterval) return nil } func (a *Autoscaler) RemoveAPI(api userconfig.Resource) { log := a.logger.With( zap.String("apiName", api.Name), zap.String("apiKind", api.Kind.String()), ) if autoscalerCron, ok := a.crons[api.Name]; ok { autoscalerCron.Cancel() delete(a.crons, api.Name) } delete(a.recs, api.Name) log.Info("autoscaler stop") } func (a *Autoscaler) Stop() { for apiName, apiCron := range a.crons { apiCron.Cancel() delete(a.crons, apiName) } } func (a *Autoscaler) autoscaleFn(api userconfig.Resource) (func() error, error) { log := a.logger.With( zap.String("apiName", api.Name), zap.String("apiKind", api.Kind.String()), ) scaler, ok := a.scalers[api.Kind] if !ok { return nil, errors.ErrorUnexpected( fmt.Sprintf("autoscaler does not have a scaler for the %s kind", api.Kind), ) } log.Info("autoscaler init") var startTime time.Time a.recs[api.Name] = newRecommendations() return func() error { autoscalingSpec, err := scaler.GetAutoscalingSpec(api.Name) if err != nil { return errors.Wrap(err, "failed to get autoscaling spec") } currentRequestedReplicas, err := scaler.CurrentRequestedReplicas(api.Name) if err != nil { return errors.Wrap(err, "failed to get current replicas") } if startTime.IsZero() { startTime = time.Now() } avgInFlight, err := scaler.GetInFlightRequests(api.Name, autoscalingSpec.Window) if err != nil { return errors.Wrap(err, "failed to get in-flight requests") } if avgInFlight == nil { log.Debug("autoscaler tick: metrics not available yet") return nil } rawRecommendation := *avgInFlight / *autoscalingSpec.TargetInFlight recommendation := int32(math.Ceil(rawRecommendation)) if rawRecommendation < float64(currentRequestedReplicas) && rawRecommendation > float64(currentRequestedReplicas)*(1-autoscalingSpec.DownscaleTolerance) { recommendation = currentRequestedReplicas } if rawRecommendation > float64(currentRequestedReplicas) && rawRecommendation < float64(currentRequestedReplicas)*(1+autoscalingSpec.UpscaleTolerance) { recommendation = currentRequestedReplicas } // always allow subtraction of 1 downscaleFactorFloor := libmath.MinInt32(currentRequestedReplicas-1, int32(math.Ceil(float64(currentRequestedReplicas)*autoscalingSpec.MaxDownscaleFactor))) if recommendation < downscaleFactorFloor { recommendation = downscaleFactorFloor } // always allow addition of 1 upscaleFactorCeil := libmath.MaxInt32(currentRequestedReplicas+1, int32(math.Ceil(float64(currentRequestedReplicas)*autoscalingSpec.MaxUpscaleFactor))) if recommendation > upscaleFactorCeil { recommendation = upscaleFactorCeil } if recommendation < autoscalingSpec.MinReplicas { recommendation = autoscalingSpec.MinReplicas } if recommendation > autoscalingSpec.MaxReplicas { recommendation = autoscalingSpec.MaxReplicas } recs := a.recs[api.Name] // Rule of thumb: any modifications that don't consider historical recommendations should be performed before // recording the recommendation, any modifications that use historical recommendations should be performed after recs.add(recommendation) // This is just for garbage collection recs.deleteOlderThan(libtime.MaxDuration(autoscalingSpec.DownscaleStabilizationPeriod, autoscalingSpec.UpscaleStabilizationPeriod)) request := recommendation var downscaleStabilizationFloor *int32 var upscaleStabilizationCeil *int32 if request < currentRequestedReplicas { downscaleStabilizationFloor = recs.maxSince(autoscalingSpec.DownscaleStabilizationPeriod) if downscaleStabilizationFloor != nil { downscaleStabilizationFloor = pointer.Int32(libmath.MinInt32(*downscaleStabilizationFloor, currentRequestedReplicas)) } if time.Since(startTime) < autoscalingSpec.DownscaleStabilizationPeriod { request = currentRequestedReplicas } else if downscaleStabilizationFloor != nil && request < *downscaleStabilizationFloor { request = *downscaleStabilizationFloor } } if request > currentRequestedReplicas { upscaleStabilizationCeil = recs.minSince(autoscalingSpec.UpscaleStabilizationPeriod) if upscaleStabilizationCeil != nil { upscaleStabilizationCeil = pointer.Int32(libmath.MaxInt32(*upscaleStabilizationCeil, currentRequestedReplicas)) } if time.Since(startTime) < autoscalingSpec.UpscaleStabilizationPeriod { request = currentRequestedReplicas } else if upscaleStabilizationCeil != nil && request > *upscaleStabilizationCeil { request = *upscaleStabilizationCeil } } log.Debugw("autoscaler tick", "autoscaling", map[string]interface{}{ "avg_in_flight": *avgInFlight, "target_in_flight": *autoscalingSpec.TargetInFlight, "raw_recommendation": rawRecommendation, "current_replicas": currentRequestedReplicas, "downscale_tolerance": autoscalingSpec.DownscaleTolerance, "upscale_tolerance": autoscalingSpec.UpscaleTolerance, "max_downscale_factor": autoscalingSpec.MaxDownscaleFactor, "downscale_factor_floor": downscaleFactorFloor, "max_upscale_factor": autoscalingSpec.MaxUpscaleFactor, "upscale_factor_ceil": upscaleFactorCeil, "min_replicas": autoscalingSpec.MinReplicas, "max_replicas": autoscalingSpec.MaxReplicas, "recommendation": recommendation, "downscale_stabilization_period": autoscalingSpec.DownscaleStabilizationPeriod.Seconds(), "downscale_stabilization_floor": downscaleStabilizationFloor, "upscale_stabilization_period": autoscalingSpec.UpscaleStabilizationPeriod.Seconds(), "upscale_stabilization_ceil": upscaleStabilizationCeil, "request": request, }, ) if currentRequestedReplicas != request { log.Infof("autoscaling event: %d -> %d", currentRequestedReplicas, request) if err = scaler.Scale(api.Name, request); err != nil { return err } } return nil }, nil } ================================================ FILE: pkg/autoscaler/autoscaler_test.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package autoscaler import ( "sync" "testing" "time" "github.com/cortexlabs/cortex/pkg/lib/cron" "github.com/cortexlabs/cortex/pkg/lib/pointer" "github.com/cortexlabs/cortex/pkg/types/userconfig" "github.com/stretchr/testify/require" "go.uber.org/zap" ) func newLogger(t *testing.T) *zap.SugaredLogger { t.Helper() config := zap.NewDevelopmentConfig() config.Level = zap.NewAtomicLevelAt(zap.ErrorLevel) logger, err := config.Build() require.NoError(t, err) logr := logger.Sugar() return logr } func generateRecommendationTimeline(t *testing.T, recs []int32, interval time.Duration) *recommendations { t.Helper() startTime := time.Now() recsTimeline := map[time.Time]int32{} for i := range recs { timestamp := startTime.Add(time.Duration(i) * interval) recsTimeline[timestamp] = recs[i] } return &recommendations{ timeline: recsTimeline, } } func TestAutoscaler_AutoscaleFn(t *testing.T) { t.Parallel() log := newLogger(t) interval := 250 * time.Millisecond cases := []struct { name string autoscalingSpec userconfig.Autoscaling inFlight float64 currentReplicas int32 recommendationTimeline []int32 expectedRequest *int32 }{ { name: "no scale below zero within stabilization period", autoscalingSpec: userconfig.Autoscaling{ MinReplicas: 0, MaxReplicas: 5, InitReplicas: 0, TargetInFlight: pointer.Float64(1), Window: 4 * interval, DownscaleStabilizationPeriod: time.Second, UpscaleStabilizationPeriod: time.Second, MaxDownscaleFactor: 0.75, MaxUpscaleFactor: 1.5, DownscaleTolerance: 0.05, UpscaleTolerance: 0.05, }, inFlight: 0, currentReplicas: 1, expectedRequest: nil, }, { name: "downscale no stabilization", autoscalingSpec: userconfig.Autoscaling{ MinReplicas: 1, MaxReplicas: 5, InitReplicas: 1, TargetInFlight: pointer.Float64(1), Window: 4 * interval, DownscaleStabilizationPeriod: 0, UpscaleStabilizationPeriod: 0, MaxDownscaleFactor: 0.5, MaxUpscaleFactor: 1.5, DownscaleTolerance: 0.05, UpscaleTolerance: 0.05, }, inFlight: 2, currentReplicas: 5, recommendationTimeline: []int32{5, 2, 2, 2}, expectedRequest: pointer.Int32(3), }, { name: "downscale with stabilization", autoscalingSpec: userconfig.Autoscaling{ MinReplicas: 1, MaxReplicas: 5, InitReplicas: 1, TargetInFlight: pointer.Float64(1), Window: 4 * interval, DownscaleStabilizationPeriod: time.Second, UpscaleStabilizationPeriod: time.Second, MaxDownscaleFactor: 0.5, MaxUpscaleFactor: 1.5, DownscaleTolerance: 0.05, UpscaleTolerance: 0.05, }, inFlight: 5, currentReplicas: 1, recommendationTimeline: []int32{5, 5, 2, 2}, expectedRequest: nil, }, { name: "upscale no stabilization", autoscalingSpec: userconfig.Autoscaling{ MinReplicas: 1, MaxReplicas: 5, InitReplicas: 1, TargetInFlight: pointer.Float64(1), Window: 4 * interval, DownscaleStabilizationPeriod: 0, UpscaleStabilizationPeriod: 0, MaxDownscaleFactor: 0.5, MaxUpscaleFactor: 1.5, DownscaleTolerance: 0.05, UpscaleTolerance: 0.05, }, inFlight: 3, currentReplicas: 1, expectedRequest: pointer.Int32(2), }, { name: "upscale with stabilization", autoscalingSpec: userconfig.Autoscaling{ MinReplicas: 1, MaxReplicas: 5, InitReplicas: 1, TargetInFlight: pointer.Float64(1), Window: 4 * interval, DownscaleStabilizationPeriod: time.Second, UpscaleStabilizationPeriod: time.Second, MaxDownscaleFactor: 0.5, MaxUpscaleFactor: 1.5, DownscaleTolerance: 0.05, UpscaleTolerance: 0.05, }, inFlight: 5, currentReplicas: 2, recommendationTimeline: []int32{2, 2, 2, 5}, expectedRequest: nil, }, { name: "no upscale below current replicas", autoscalingSpec: userconfig.Autoscaling{ MinReplicas: 0, MaxReplicas: 5, InitReplicas: 0, TargetInFlight: pointer.Float64(1), Window: 4 * interval, DownscaleStabilizationPeriod: time.Second, UpscaleStabilizationPeriod: time.Second, MaxDownscaleFactor: 0.75, MaxUpscaleFactor: 1.5, DownscaleTolerance: 0.05, UpscaleTolerance: 0.05, }, inFlight: 3, currentReplicas: 2, recommendationTimeline: []int32{0, 1, 2, 3}, expectedRequest: nil, }, } for i, tt := range cases { t.Run(tt.name, func(t *testing.T) { t.Parallel() var latestRequest *int32 scalerMock := &ScalerFunc{ ScaleFunc: func(apiName string, request int32) error { latestRequest = pointer.Int32(request) return nil }, GetInFlightRequestsFunc: func(apiName string, window time.Duration) (*float64, error) { return pointer.Float64(tt.inFlight), nil }, GetAutoscalingSpecFunc: func(apiName string) (*userconfig.Autoscaling, error) { return &cases[i].autoscalingSpec, nil }, CurrentRequestedReplicasFunc: func(apiName string) (int32, error) { return tt.currentReplicas, nil }, } autoScaler := &Autoscaler{ logger: log, crons: make(map[string]cron.Cron), scalers: make(map[userconfig.Kind]Scaler), recs: make(map[string]*recommendations), } autoScaler.AddScaler(scalerMock, userconfig.RealtimeAPIKind) apiName := "test" api := userconfig.Resource{ Name: apiName, Kind: userconfig.RealtimeAPIKind, } autoScaler.recs[apiName] = generateRecommendationTimeline(t, tt.recommendationTimeline, interval) autoscaleFn, err := autoScaler.autoscaleFn(api) require.NoError(t, err) time.Sleep(interval) err = autoscaleFn() require.NoError(t, err) require.Equal(t, tt.expectedRequest, latestRequest) }) } } func TestAutoscaler_Awake(t *testing.T) { t.Parallel() log := newLogger(t) mux := sync.RWMutex{} var latestRequest int32 downscaleStabilizationPeriod := 3 * time.Second scalerMock := &ScalerFunc{ ScaleFunc: func(apiName string, request int32) error { mux.Lock() defer mux.Unlock() latestRequest = request return nil }, GetInFlightRequestsFunc: func(apiName string, window time.Duration) (*float64, error) { return pointer.Float64(0), nil }, GetAutoscalingSpecFunc: func(apiName string) (*userconfig.Autoscaling, error) { return &userconfig.Autoscaling{ MinReplicas: 0, MaxReplicas: 1, InitReplicas: 1, TargetInFlight: pointer.Float64(1), Window: 500 * time.Millisecond, DownscaleStabilizationPeriod: downscaleStabilizationPeriod, MaxDownscaleFactor: 0.75, MaxUpscaleFactor: 1.5, }, nil }, CurrentRequestedReplicasFunc: func(apiName string) (int32, error) { return 0, nil }, } autoScaler := &Autoscaler{ logger: log, crons: make(map[string]cron.Cron), scalers: make(map[userconfig.Kind]Scaler), recs: make(map[string]*recommendations), } autoScaler.AddScaler(scalerMock, userconfig.RealtimeAPIKind) apiName := "test" api := userconfig.Resource{ Name: apiName, Kind: userconfig.RealtimeAPIKind, } autoscaleFn, err := autoScaler.autoscaleFn(api) require.NoError(t, err) ticker := time.NewTicker(250 * time.Millisecond) go func() { for { select { case <-ticker.C: err := autoscaleFn() require.NoError(t, err) } } }() err = autoScaler.Awaken(api) require.NoError(t, err) require.Never(t, func() bool { mux.RLock() defer mux.RUnlock() return latestRequest != 1 }, downscaleStabilizationPeriod, time.Second) } func TestAutoscaler_MinReplicas(t *testing.T) { t.Parallel() log := newLogger(t) mux := sync.RWMutex{} var latestRequest int32 minReplicas := int32(5) maxReplicas := int32(10) scalerMock := &ScalerFunc{ ScaleFunc: func(apiName string, request int32) error { mux.Lock() defer mux.Unlock() latestRequest = request return nil }, GetInFlightRequestsFunc: func(apiName string, window time.Duration) (*float64, error) { return pointer.Float64(0), nil }, GetAutoscalingSpecFunc: func(apiName string) (*userconfig.Autoscaling, error) { return &userconfig.Autoscaling{ MinReplicas: minReplicas, MaxReplicas: maxReplicas, InitReplicas: minReplicas, TargetInFlight: pointer.Float64(1), Window: 500 * time.Millisecond, MaxDownscaleFactor: 0.75, MaxUpscaleFactor: 1.5, }, nil }, CurrentRequestedReplicasFunc: func(apiName string) (int32, error) { return minReplicas + 1, nil }, } autoScaler := &Autoscaler{ logger: log, crons: make(map[string]cron.Cron), scalers: make(map[userconfig.Kind]Scaler), recs: make(map[string]*recommendations), } autoScaler.AddScaler(scalerMock, userconfig.RealtimeAPIKind) apiName := "test" api := userconfig.Resource{ Name: apiName, Kind: userconfig.RealtimeAPIKind, } autoscaleFn, err := autoScaler.autoscaleFn(api) require.NoError(t, err) ticker := time.NewTicker(250 * time.Millisecond) go func() { for { select { case <-ticker.C: err := autoscaleFn() require.NoError(t, err) } } }() require.Never(t, func() bool { mux.RLock() defer mux.RUnlock() return latestRequest < minReplicas }, 3*time.Second, time.Second) } func TestAutoscaler_MaxReplicas(t *testing.T) { t.Parallel() log := newLogger(t) mux := sync.RWMutex{} var latestRequest int32 minReplicas := int32(5) maxReplicas := int32(10) scalerMock := &ScalerFunc{ ScaleFunc: func(apiName string, request int32) error { mux.Lock() defer mux.Unlock() latestRequest = request return nil }, GetInFlightRequestsFunc: func(apiName string, window time.Duration) (*float64, error) { return pointer.Float64(20), nil }, GetAutoscalingSpecFunc: func(apiName string) (*userconfig.Autoscaling, error) { return &userconfig.Autoscaling{ MinReplicas: minReplicas, MaxReplicas: maxReplicas, InitReplicas: minReplicas, TargetInFlight: pointer.Float64(1), Window: 500 * time.Millisecond, MaxDownscaleFactor: 0.75, MaxUpscaleFactor: 1.5, }, nil }, CurrentRequestedReplicasFunc: func(apiName string) (int32, error) { return minReplicas, nil }, } autoScaler := &Autoscaler{ logger: log, crons: make(map[string]cron.Cron), scalers: make(map[userconfig.Kind]Scaler), recs: make(map[string]*recommendations), } autoScaler.AddScaler(scalerMock, userconfig.RealtimeAPIKind) apiName := "test" api := userconfig.Resource{ Name: apiName, Kind: userconfig.RealtimeAPIKind, } autoscaleFn, err := autoScaler.autoscaleFn(api) require.NoError(t, err) ticker := time.NewTicker(250 * time.Millisecond) go func() { for { select { case <-ticker.C: err := autoscaleFn() require.NoError(t, err) } } }() require.Never(t, func() bool { mux.RLock() defer mux.RUnlock() return latestRequest > maxReplicas }, 3*time.Second, time.Second) } ================================================ FILE: pkg/autoscaler/client.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package autoscaler import ( "bytes" "fmt" "io/ioutil" "net/http" "github.com/cortexlabs/cortex/pkg/lib/json" "github.com/cortexlabs/cortex/pkg/lib/urls" "github.com/cortexlabs/cortex/pkg/types/userconfig" ) type Client interface { Awaken(api userconfig.Resource) error } type client struct { httpClient *http.Client endpoint string } func NewClient(endpoint string) Client { return &client{ httpClient: &http.Client{}, endpoint: endpoint, } } func (c *client) Awaken(api userconfig.Resource) error { payload, err := json.Marshal(api) if err != nil { return err } response, err := c.httpClient.Post( urls.Join(c.endpoint, "/awaken"), "application/json", bytes.NewBuffer(payload), ) if err != nil { return err } defer func() { _ = response.Body.Close() }() if response.StatusCode != http.StatusOK { bodyBytes, _ := ioutil.ReadAll(response.Body) errMsg := fmt.Sprintf("failed to awake api (status code %d)", response.StatusCode) if bodyBytes != nil { errMsg = errMsg + fmt.Sprintf(": %s", string(bodyBytes)) } return fmt.Errorf(errMsg) } return nil } ================================================ FILE: pkg/autoscaler/handler.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package autoscaler import ( "encoding/json" "net/http" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/telemetry" "github.com/cortexlabs/cortex/pkg/types/userconfig" ) type Handler struct { autoscaler *Autoscaler } func NewHandler(autoscaler *Autoscaler) *Handler { return &Handler{autoscaler: autoscaler} } func (h *Handler) Awaken(w http.ResponseWriter, r *http.Request) { var api userconfig.Resource if err := json.NewDecoder(r.Body).Decode(&api); err != nil { http.Error(w, "failed to json decode request body", http.StatusBadRequest) return } defer func() { _ = r.Body.Close() }() if err := h.autoscaler.Awaken(api); err != nil { http.Error(w, errors.Wrap(err, "failed to awaken api").Error(), http.StatusInternalServerError) telemetry.Error(err) return } w.WriteHeader(http.StatusOK) } ================================================ FILE: pkg/autoscaler/realtime_scaler.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package autoscaler import ( "context" "fmt" "time" "github.com/cortexlabs/cortex/pkg/consts" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/k8s" "github.com/cortexlabs/cortex/pkg/lib/pointer" "github.com/cortexlabs/cortex/pkg/lib/telemetry" "github.com/cortexlabs/cortex/pkg/types/userconfig" "github.com/cortexlabs/cortex/pkg/workloads" promv1 "github.com/prometheus/client_golang/api/prometheus/v1" "github.com/prometheus/common/model" "go.uber.org/zap" kapps "k8s.io/api/apps/v1" kmeta "k8s.io/apimachinery/pkg/apis/meta/v1" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" ) type RealtimeScaler struct { k8s *k8s.Client prometheus promv1.API logger *zap.SugaredLogger } func NewRealtimeScaler(k8sClient *k8s.Client, promClient promv1.API, logger *zap.SugaredLogger) *RealtimeScaler { return &RealtimeScaler{ k8s: k8sClient, prometheus: promClient, logger: logger, } } func (s *RealtimeScaler) Scale(apiName string, request int32) error { ctx := context.Background() // we use the controller-runtime client to make use of the cache mechanism var deployment kapps.Deployment err := s.k8s.Get(ctx, ctrlclient.ObjectKey{ Namespace: s.k8s.Namespace, Name: workloads.K8sName(apiName), }, &deployment) if err != nil { return errors.Wrap(err, "failed to get deployment") } if deployment.Spec.Replicas == nil { return errors.Wrap(err, "k8s deployment doesn't have the replicas field set") } current := *deployment.Spec.Replicas if current == request { return nil } if request == 0 { if err = s.routeToActivator(&deployment); err != nil { return errors.Wrap(err, "failed to re-route traffic to activator") } } deployment.Spec.Replicas = pointer.Int32(request) if err = s.k8s.Update(ctx, &deployment); err != nil { return errors.Wrap(err, "failed to update deployment") } if current == 0 && request > 0 { go func() { if err := s.routeToService(&deployment); err != nil { s.logger.Errorw("failed to re-route traffic to API", zap.Error(err), zap.String("apiName", apiName), ) telemetry.Error(err) } }() } return nil } func (s *RealtimeScaler) GetInFlightRequests(apiName string, window time.Duration) (*float64, error) { windowSeconds := int64(window.Seconds()) // PromQL query: // sum(sum_over_time(cortex_in_flight_requests{api_name=""}[60s])) / // sum(count_over_time(cortex_in_flight_requests{api_name="", container!="activator"}[60s])) query := fmt.Sprintf( "sum(sum_over_time(cortex_in_flight_requests{api_name=\"%s\"}[%ds])) / "+ "max(count_over_time(cortex_in_flight_requests{api_name=\"%s\", container!=\"activator\"}[%ds]))", apiName, windowSeconds, apiName, windowSeconds, ) ctx, cancel := context.WithTimeout(context.Background(), _prometheusQueryTimeoutSeconds*time.Second) defer cancel() valuesQuery, _, err := s.prometheus.Query(ctx, query, time.Now()) if err != nil { return nil, err } values, ok := valuesQuery.(model.Vector) if !ok { return nil, errors.ErrorUnexpected("failed to convert prometheus metric to vector") } // no values available if values.Len() == 0 { return nil, nil } avgInflightRequests := float64(values[0].Value) return &avgInflightRequests, nil } func (s *RealtimeScaler) GetAutoscalingSpec(apiName string) (*userconfig.Autoscaling, error) { deployment, err := s.k8s.GetDeployment(workloads.K8sName(apiName)) if err != nil { return nil, errors.Wrap(err, "failed to get deployment") } if deployment == nil { return nil, errors.ErrorUnexpected("unable to find k8s deployment", apiName) } autoscalingSpec, err := userconfig.AutoscalingFromAnnotations(deployment) if err != nil { return nil, err } return autoscalingSpec, nil } func (s *RealtimeScaler) CurrentRequestedReplicas(apiName string) (int32, error) { ctx := context.Background() // we use the controller-runtime client to make use of the cache mechanism var deployment kapps.Deployment err := s.k8s.Get(ctx, ctrlclient.ObjectKey{ Namespace: s.k8s.Namespace, Name: workloads.K8sName(apiName), }, &deployment) if err != nil { return 0, errors.Wrap(err, "failed to get deployment") } if deployment.Spec.Replicas == nil { return 0, errors.Wrap(err, "k8s deployment doesn't have the replicas field set") } return *deployment.Spec.Replicas, nil } func (s *RealtimeScaler) routeToService(deployment *kapps.Deployment) error { ctx := context.Background() vs, err := s.k8s.GetVirtualService(deployment.Name) if err != nil { return errors.Wrap(err, "failed to get virtual service") } if len(vs.Spec.Http) < 1 { return errors.ErrorUnexpected("virtual service does not have any http entries") } if err = s.waitForReadyReplicas(ctx, deployment); err != nil { return errors.Wrap(err, "no ready replicas available") } for i := range vs.Spec.Http { if len(vs.Spec.Http[i].Route) != 2 { return errors.ErrorUnexpected("virtual service does not have the required number of 2 http routes") } vs.Spec.Http[i].Route[0].Weight = 100 // service traffic vs.Spec.Http[i].Route[1].Weight = 0 // activator traffic } vsClient := s.k8s.IstioClientSet().NetworkingV1beta1().VirtualServices(s.k8s.Namespace) if _, err = vsClient.Update(ctx, vs, kmeta.UpdateOptions{}); err != nil { return errors.Wrap(err, "failed to update virtual service") } return nil } func (s *RealtimeScaler) routeToActivator(deployment *kapps.Deployment) error { ctx := context.Background() vs, err := s.k8s.GetVirtualService(deployment.Name) if err != nil { return errors.Wrap(err, "failed to get virtual service") } if len(vs.Spec.Http) < 1 { return errors.ErrorUnexpected("virtual service does not have any http entries") } for i := range vs.Spec.Http { if len(vs.Spec.Http[i].Route) != 2 { return errors.ErrorUnexpected("virtual service does not have the required number of 2 http routes") } vs.Spec.Http[i].Route[0].Weight = 0 // service traffic vs.Spec.Http[i].Route[1].Weight = 100 // activator traffic } vsClient := s.k8s.IstioClientSet().NetworkingV1beta1().VirtualServices(s.k8s.Namespace) if _, err = vsClient.Update(ctx, vs, kmeta.UpdateOptions{}); err != nil { return errors.Wrap(err, "failed to update virtual service") } return nil } func (s *RealtimeScaler) waitForReadyReplicas(ctx context.Context, deployment *kapps.Deployment) error { watcher, err := s.k8s.ClientSet().AppsV1().Deployments(s.k8s.Namespace).Watch( ctx, kmeta.ListOptions{ FieldSelector: fmt.Sprintf("metadata.name=%s", deployment.Name), Watch: true, }, ) if err != nil { return errors.Wrap(err, "could not create deployment watcher") } defer watcher.Stop() ctx, cancel := context.WithTimeout(ctx, consts.WaitForReadyReplicasTimeout) defer cancel() for { select { case event := <-watcher.ResultChan(): deploy, ok := event.Object.(*kapps.Deployment) if !ok { continue } if deploy.Status.ReadyReplicas > 0 { return nil } case <-ctx.Done(): return ctx.Err() } } } ================================================ FILE: pkg/autoscaler/recommendations.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package autoscaler import ( "math" "sync" "time" ) type recommendations struct { mux sync.RWMutex timeline map[time.Time]int32 } func newRecommendations() *recommendations { return &recommendations{ timeline: make(map[time.Time]int32), } } func (r *recommendations) add(rec int32) { r.mux.Lock() defer r.mux.Unlock() r.timeline[time.Now()] = rec } func (r *recommendations) deleteOlderThan(period time.Duration) { r.mux.Lock() defer r.mux.Unlock() for t := range r.timeline { if time.Since(t) > period { delete(r.timeline, t) } } } // Returns nil if no recommendations in the period func (r *recommendations) maxSince(period time.Duration) *int32 { r.mux.RLock() defer r.mux.RUnlock() max := int32(math.MinInt32) foundRecommendation := false for t, rec := range r.timeline { if time.Since(t) <= period && rec > max { max = rec foundRecommendation = true } } if !foundRecommendation { return nil } return &max } // Returns nil if no recommendations in the period func (r *recommendations) minSince(period time.Duration) *int32 { r.mux.RLock() defer r.mux.RUnlock() min := int32(math.MaxInt32) foundRecommendation := false for t, rec := range r.timeline { if time.Since(t) <= period && rec < min { min = rec foundRecommendation = true } } if !foundRecommendation { return nil } return &min } ================================================ FILE: pkg/autoscaler/scaler_func.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package autoscaler import ( "time" "github.com/cortexlabs/cortex/pkg/types/userconfig" ) type ScalerFunc struct { ScaleFunc func(apiName string, request int32) error GetInFlightRequestsFunc func(apiName string, window time.Duration) (*float64, error) GetAutoscalingSpecFunc func(apiName string) (*userconfig.Autoscaling, error) CurrentRequestedReplicasFunc func(apiName string) (int32, error) } func (s *ScalerFunc) Scale(apiName string, request int32) error { if s.ScaleFunc == nil { return nil } return s.ScaleFunc(apiName, request) } func (s *ScalerFunc) GetInFlightRequests(apiName string, window time.Duration) (*float64, error) { if s.GetInFlightRequestsFunc == nil { return nil, nil } return s.GetInFlightRequestsFunc(apiName, window) } func (s *ScalerFunc) GetAutoscalingSpec(apiName string) (*userconfig.Autoscaling, error) { if s.GetAutoscalingSpecFunc == nil { return nil, nil } return s.GetAutoscalingSpecFunc(apiName) } func (s *ScalerFunc) CurrentRequestedReplicas(apiName string) (int32, error) { if s.CurrentRequestedReplicasFunc == nil { return 0, nil } return s.CurrentRequestedReplicasFunc(apiName) } ================================================ FILE: pkg/config/config.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package config import ( "fmt" "os" "strings" "github.com/DataDog/datadog-go/statsd" "github.com/cortexlabs/cortex/pkg/consts" batch "github.com/cortexlabs/cortex/pkg/crds/apis/batch/v1alpha1" "github.com/cortexlabs/cortex/pkg/lib/aws" cr "github.com/cortexlabs/cortex/pkg/lib/configreader" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/hash" "github.com/cortexlabs/cortex/pkg/lib/k8s" "github.com/cortexlabs/cortex/pkg/lib/telemetry" "github.com/cortexlabs/cortex/pkg/types/clusterconfig" promapi "github.com/prometheus/client_golang/api" promv1 "github.com/prometheus/client_golang/api/prometheus/v1" "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ) var ( OperatorMetadata *clusterconfig.OperatorMetadata ClusterConfig *clusterconfig.Config AWS *aws.Client K8s *k8s.Client K8sIstio *k8s.Client K8sAllNamspaces *k8s.Client MetricsClient *statsd.Client Prometheus promv1.API scheme = runtime.NewScheme() ) func init() { utilruntime.Must(clientgoscheme.AddToScheme(scheme)) utilruntime.Must(batch.AddToScheme(scheme)) } func InitConfigs(clusterConfig *clusterconfig.Config, operatorMetadata *clusterconfig.OperatorMetadata) { ClusterConfig = clusterConfig OperatorMetadata = operatorMetadata } func getClusterConfigFromConfigMap() (clusterconfig.Config, error) { configMapData, _, err := K8s.GetConfigMapData("cluster-config") if err != nil { return clusterconfig.Config{}, err } clusterConfig := clusterconfig.Config{} err = cr.ParseYAMLBytes(&clusterConfig, clusterconfig.FullConfigValidation, []byte(configMapData["cluster.yaml"])) if err != nil { return clusterconfig.Config{}, err } return clusterConfig, nil } func Init() error { var err error clusterConfigPath := os.Getenv("CORTEX_CLUSTER_CONFIG_PATH") if clusterConfigPath == "" { clusterConfigPath = consts.DefaultInClusterConfigPath } clusterConfig, err := clusterconfig.NewForFile(clusterConfigPath) if err != nil { return err } ClusterConfig = clusterConfig AWS, err = aws.NewForRegion(clusterConfig.Region) if err != nil { return err } accountID, hashedAccountID, err := AWS.CheckCredentials() if err != nil { return err } clusterConfig.AccountID = accountID OperatorMetadata = &clusterconfig.OperatorMetadata{ APIVersion: consts.CortexVersion, OperatorID: hashedAccountID, ClusterID: hash.String(clusterConfig.ClusterName + clusterConfig.Region + hashedAccountID), IsOperatorInCluster: strings.ToLower(os.Getenv("CORTEX_OPERATOR_IN_CLUSTER")) != "false", } if K8s, err = k8s.New(consts.DefaultNamespace, OperatorMetadata.IsOperatorInCluster, nil, scheme); err != nil { return err } if K8sIstio, err = k8s.New(consts.IstioNamespace, OperatorMetadata.IsOperatorInCluster, nil, scheme); err != nil { return err } if !OperatorMetadata.IsOperatorInCluster { cc, err := getClusterConfigFromConfigMap() if err != nil { return err } clusterConfig.Bucket = cc.Bucket clusterConfig.ClusterUID = cc.ClusterUID } exists, err := AWS.DoesBucketExist(clusterConfig.Bucket) if err != nil { return err } if !exists { return errors.ErrorUnexpected("the specified bucket does not exist", clusterConfig.Bucket) } err = telemetry.Init(telemetry.Config{ Enabled: clusterConfig.Telemetry, UserID: OperatorMetadata.OperatorID, Properties: map[string]string{ "cluster_id": OperatorMetadata.ClusterID, "operator_id": OperatorMetadata.OperatorID, }, Environment: "operator", LogErrors: true, BackoffMode: telemetry.BackoffDuplicateMessages, }) if err != nil { fmt.Println(errors.Message(err)) } prometheusURL := os.Getenv("CORTEX_PROMETHEUS_URL") if len(prometheusURL) == 0 { prometheusURL = fmt.Sprintf("http://prometheus.%s:9090", consts.PrometheusNamespace) } promClient, err := promapi.NewClient(promapi.Config{ Address: prometheusURL, }) if err != nil { return err } Prometheus = promv1.NewAPI(promClient) if K8sAllNamspaces, err = k8s.New("", OperatorMetadata.IsOperatorInCluster, nil, scheme); err != nil { return err } if OperatorMetadata.IsOperatorInCluster { MetricsClient, err = statsd.New(fmt.Sprintf("prometheus-statsd-exporter.%s:9125", consts.PrometheusNamespace)) if err != nil { return errors.Wrap(errors.WithStack(err), "unable to initialize metrics client") } } return nil } ================================================ FILE: pkg/consts/consts.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package consts import ( "os" "time" kresource "k8s.io/apimachinery/pkg/api/resource" ) var ( CortexVersion = "master" // CORTEX_VERSION CortexVersionMinor = "master" // CORTEX_VERSION_MINOR DefaultNamespace = "default" KubeSystemNamespace = "kube-system" IstioNamespace = "istio-system" PrometheusNamespace = "prometheus" LoggingNamespace = "logging" DefaultMaxQueueLength = int64(100) DefaultMaxConcurrency = int64(1) DefaultUserPodPortInt32 = int32(8080) ProxyPortStr = "8888" ProxyPortInt32 = int32(8888) ActivatorName = "activator" ActivatorPortInt32 = int32(8000) AdminPortName = "admin" AdminPortStr = "15000" AdminPortInt32 = int32(15000) AuthHeader = "X-Cortex-Authorization" CortexProxyCPU = kresource.MustParse("100m") CortexProxyMem = kresource.MustParse("100Mi") CortexDequeuerCPU = kresource.MustParse("100m") CortexDequeuerMem = kresource.MustParse("100Mi") /* CPU Pod Reservations: - FluentBit 100 - NodeExporter 50 (it has two containers) - KubeProxy 100 - AWS cni 10 */ CortexCPUPodReserved = kresource.MustParse("260m") /* CPU Node Reservations: - Reserved (150 + 150) see generate_eks.py for details */ CortexCPUK8sReserved = kresource.MustParse("300m") /* Memory Pod Reservations: - FluentBit 150 - NodeExporter 200 (it has two containers) */ CortexMemPodReserved = kresource.MustParse("350Mi") /* Memory Node Reservations: - Reserved (300 + 300 + 200) see generate_eks.py for details */ CortexMemK8sReserved = kresource.MustParse("800Mi") DefaultInClusterConfigPath = "/configs/cluster/cluster.yaml" MaxBucketLifecycleRules = 100 AsyncWorkloadsExpirationDays = int64(7) ReservedContainerPorts = []int32{ ProxyPortInt32, AdminPortInt32, } ReservedContainerNames = []string{ "dequeuer", "proxy", } UserAgentKey = "User-Agent" KubeProbeUserAgentPrefix = "kube-probe/" CortexAPINameHeader = "X-Cortex-API-Name" CortexTargetServiceHeader = "X-Cortex-Target-Service" CortexProbeHeader = "X-Cortex-Probe" CortexOriginHeader = "X-Cortex-Origin" CortexQueueURLHeader = "X-Cortex-Queue-URL" WaitForReadyReplicasTimeout = 20 * time.Minute ) func DefaultRegistry() string { if registryOverride := os.Getenv("CORTEX_DEV_DEFAULT_IMAGE_REGISTRY"); registryOverride != "" { return registryOverride } return "quay.io/cortexlabs" } ================================================ FILE: pkg/crds/Makefile ================================================ #!make # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Image URL to use all building/pushing image targets IMG ?= controller:latest # Produce CRDs that work back to Kubernetes 1.11 (no version conversion) CRD_OPTIONS ?= "crd:trivialVersions=true,preserveUnknownFields=false,crdVersions=v1" # Cortex cluster config path. Defaults to the dev config path CLUSTER_CONFIG ?= "../../dev/config/cluster.yaml" # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) ifeq (,$(shell go env GOBIN)) GOBIN=$(shell go env GOPATH)/bin else GOBIN=$(shell go env GOBIN) endif # Setting SHELL to bash allows bash commands to be executed by recipes. # This is a requirement for 'setup-envtest.sh' in the test target. # Options are set to exit when a recipe line exits non-zero or a piped command fails. SHELL = /usr/bin/env bash -o pipefail .SHELLFLAGS = -ec all: build ##@ General # The help target prints out all targets with their descriptions organized # beneath their categories. The categories are represented by '##@' and the # target descriptions by '##'. The awk commands is responsible for reading the # entire set of makefiles included in this invocation, looking for lines of the # file as xyz: ## something, and then pretty-format the target and help. Then, # if there's a line with ##@ something, that gets pretty-printed as a category. # More info on the usage of ANSI control characters for terminal formatting: # https://en.wikipedia.org/wiki/ANSI_escape_code#SGR_parameters # More info on the awk command: # http://linuxcommand.org/lc3_adv_awk.php help: ## Display this help. @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) ##@ Development manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects. $(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=manager-role webhook paths="./..." output:crd:artifacts:config=config/crd/bases generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." fmt: ## Run go fmt against code. go fmt ./... vet: ## Run go vet against code. go vet ./... ENVTEST_ASSETS_DIR=$(shell pwd)/testbin test: manifests generate fmt vet ## Run tests. mkdir -p ${ENVTEST_ASSETS_DIR} test -f ${ENVTEST_ASSETS_DIR}/setup-envtest.sh || curl -sSLo ${ENVTEST_ASSETS_DIR}/setup-envtest.sh https://raw.githubusercontent.com/kubernetes-sigs/controller-runtime/v0.7.2/hack/setup-envtest.sh source ${ENVTEST_ASSETS_DIR}/setup-envtest.sh; fetch_envtest_tools $(ENVTEST_ASSETS_DIR); setup_envtest_env $(ENVTEST_ASSETS_DIR); go test ./... -coverprofile cover.out ##@ Build build: generate fmt vet ## Build manager binary. go build -o bin/manager main.go run: manifests generate fmt vet ## Run a controller from your host. bash ./hack/run_manager.sh ${CLUSTER_CONFIG} docker-build: ## Build docker image with the manager. docker build -f ../../images/controller-manager/Dockerfile -t ${IMG} ../.. docker-push: ## Push docker image with the manager. docker push ${IMG} ##@ Deployment install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config. $(KUSTOMIZE) build config/crd | kubectl apply -f - uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. $(KUSTOMIZE) build config/crd | kubectl delete -f - deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} $(KUSTOMIZE) build config/default | kubectl apply -f - undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. $(KUSTOMIZE) build config/default | kubectl delete -f - CONTROLLER_GEN = $(shell pwd)/bin/controller-gen controller-gen: ## Download controller-gen locally if necessary. $(call go-get-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.4.1) KUSTOMIZE = $(shell pwd)/bin/kustomize kustomize: ## Download kustomize locally if necessary. $(call go-get-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/v3@v3.8.7) # go-get-tool will 'go get' any package $2 and install it to $1. PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST)))) define go-get-tool @[ -f $(1) ] || { \ set -e ;\ TMP_DIR=$$(mktemp -d) ;\ cd $$TMP_DIR ;\ go mod init tmp ;\ echo "Downloading $(2)" ;\ GOBIN=$(PROJECT_DIR)/bin go get $(2) ;\ rm -rf $$TMP_DIR ;\ } endef ================================================ FILE: pkg/crds/PROJECT ================================================ domain: cortex.dev layout: - go.kubebuilder.io/v3 multigroup: true projectName: operator repo: github.com/cortexlabs/cortex resources: - api: crdVersion: v1 namespaced: true controller: true domain: cortex.dev group: batch kind: BatchJob path: github.com/cortexlabs/cortex/pkg/crds/apis/batch/v1alpha1 version: v1alpha1 version: "3" ================================================ FILE: pkg/crds/apis/batch/v1alpha1/batchjob_metrics.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package v1alpha1 import ( "context" "fmt" "time" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/parallel" "github.com/cortexlabs/cortex/pkg/types/metrics" "github.com/cortexlabs/cortex/pkg/types/spec" promv1 "github.com/prometheus/client_golang/api/prometheus/v1" "github.com/prometheus/common/model" ) const ( _metricsRequestTimeoutSeconds = 10 ) // GetMetrics retrieves the BatchJob metrics from prometheus func GetMetrics(promAPIv1 promv1.API, jobKey spec.JobKey, t time.Time) (*metrics.BatchMetrics, error) { var ( jobBatchesSucceeded float64 jobBatchesFailed float64 avgTimePerBatch *float64 ) err := parallel.RunFirstErr( func() error { var err error jobBatchesSucceeded, err = getSucceededBatchesForJobMetric(promAPIv1, jobKey, t) return err }, func() error { var err error jobBatchesFailed, err = getFailedBatchesForJobMetric(promAPIv1, jobKey, t) return err }, func() error { var err error avgTimePerBatch, err = getAvgTimePerBatchMetric(promAPIv1, jobKey, t) return err }, ) if err != nil { return nil, err } return &metrics.BatchMetrics{ Succeeded: int(jobBatchesSucceeded), Failed: int(jobBatchesFailed), AverageTimePerBatch: avgTimePerBatch, }, nil } func getSucceededBatchesForJobMetric(promAPIv1 promv1.API, jobKey spec.JobKey, t time.Time) (float64, error) { query := fmt.Sprintf( "sum(cortex_batch_succeeded{api_name=\"%s\", job_id=\"%s\"})", jobKey.APIName, jobKey.ID, ) values, err := queryPrometheusVec(promAPIv1, query, t) if err != nil { return 0, err } if values.Len() == 0 { return 0, nil } succeededBatches := float64(values[0].Value) return succeededBatches, nil } func getFailedBatchesForJobMetric(promAPIv1 promv1.API, jobKey spec.JobKey, t time.Time) (float64, error) { query := fmt.Sprintf( "sum(cortex_batch_failed{api_name=\"%s\", job_id=\"%s\"})", jobKey.APIName, jobKey.ID, ) values, err := queryPrometheusVec(promAPIv1, query, t) if err != nil { return 0, err } if values.Len() == 0 { return 0, nil } failedBatches := float64(values[0].Value) return failedBatches, nil } func getAvgTimePerBatchMetric(promAPIv1 promv1.API, jobKey spec.JobKey, t time.Time) (*float64, error) { query := fmt.Sprintf( "sum(cortex_time_per_batch_sum{api_name=\"%s\", job_id=\"%s\"}) / sum(cortex_time_per_batch_count{api_name=\"%s\", job_id=\"%s\"})", jobKey.APIName, jobKey.ID, jobKey.APIName, jobKey.ID, ) values, err := queryPrometheusVec(promAPIv1, query, t) if err != nil { return nil, err } if values.Len() == 0 { return nil, nil } avgTimePerBatch := float64(values[0].Value) return &avgTimePerBatch, nil } func queryPrometheusVec(promAPIv1 promv1.API, query string, t time.Time) (model.Vector, error) { ctx, cancel := context.WithTimeout(context.Background(), _metricsRequestTimeoutSeconds*time.Second) defer cancel() valuesQuery, _, err := promAPIv1.Query(ctx, query, t) if err != nil { return nil, err } values, ok := valuesQuery.(model.Vector) if !ok { return nil, errors.ErrorUnexpected("failed to convert metric to vector") } return values, nil } ================================================ FILE: pkg/crds/apis/batch/v1alpha1/batchjob_types.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package v1alpha1 import ( "github.com/cortexlabs/cortex/pkg/types/status" kcore "k8s.io/api/core/v1" kmeta "k8s.io/apimachinery/pkg/apis/meta/v1" ) // BatchJobSpec defines the desired state of BatchJob type BatchJobSpec struct { // +kubebuilder:validation:Required // Reference to a cortex BatchAPI name APIName string `json:"api_name,omitempty"` // +kubebuilder:validation:Required // Reference to a cortex BatchAPI apiID APIID string `json:"api_id,omitempty"` // +kubebuilder:validation:Optional // +kubebuilder:default=1 // Number of workers for the batch job Workers int32 `json:"workers,omitempty"` // +kubebuilder:validation:Optional // YAML content of the user config Config *string `json:"config,omitempty"` // +kubebuilder:validation:Optional // Duration until a batch job times out Timeout *kmeta.Duration `json:"timeout,omitempty"` // +kubebuilder:validation:Optional // Configuration for the dead letter queue DeadLetterQueue *DeadLetterQueueSpec `json:"dead_letter_queue,omitempty"` // +kubebuilder:validation:Optional // Compute resource requirements Resources *kcore.ResourceRequirements `json:"resources,omitempty"` // +kubebuilder:validation:Optional // +nullable // Node groups selector NodeGroups []string `json:"node_groups"` // +kubebuilder:validation:Optional // +nullable // Readiness probes for the job (container name -> probe) Probes map[string]kcore.Probe `json:"probes"` // +kubebuilder:validation:Optional // Time to live for the resource. The controller will clean-up resources // that reached a final state when the TTL time is exceeded. TTL *kmeta.Duration `json:"ttl,omitempty"` } // DeadLetterQueueSpec defines the desired state for the dead letter queue in a BatchJob type DeadLetterQueueSpec struct { // +kubebuilder:validation:Required // arn of the dead letter queue e.g. arn:aws:sqs:us-west-2:123456789:failed.fifo ARN string `json:"arn,omitempty"` // +kubebuilder:validation:Optional // +kubebuilder:default=1 // +kubebuilder:validation:Minimum=1 // Number of times a batch is allowed to be handled by a worker before it is considered to be failed // and transferred to the dead letter queue (must be >= 1) MaxReceiveCount int32 `json:"max_receive_count,omitempty"` } // BatchJobStatus defines the observed state of BatchJob type BatchJobStatus struct { // Job ID ID string `json:"id,omitempty"` // Processing ending timestamp EndTime *kmeta.Time `json:"end_time,omitempty"` // URL for the used SQS queue QueueURL string `json:"queue_url,omitempty"` // Total batch count TotalBatchCount int `json:"total_batch_count,omitempty"` // +kubebuilder:validation:Type=string // Status of the batch job Status status.JobCode `json:"status,omitempty"` // Detailed worker counts with respective status WorkerCounts *status.WorkerCounts `json:"worker_counts,omitempty"` } // EnqueuingStatus is an enum for the different possible enqueuing status type EnqueuingStatus string // Possible EnqueuingStatus states const ( EnqueuingNotStarted EnqueuingStatus = "not_started" EnqueuingInProgress EnqueuingStatus = "in_progress" EnqueuingDone EnqueuingStatus = "done" EnqueuingFailed EnqueuingStatus = "failed" ) // +kubebuilder:object:root=true // +kubebuilder:subresource:status // +kubebuilder:printcolumn:JSONPath=".status.status",name="Status",type="string" // +kubebuilder:printcolumn:JSONPath=".status.queue_url",name="Queue URL",type="string" // BatchJob is the Schema for the batchjobs API type BatchJob struct { kmeta.TypeMeta `json:",inline"` kmeta.ObjectMeta `json:"metadata,omitempty"` Spec BatchJobSpec `json:"spec,omitempty"` Status BatchJobStatus `json:"status,omitempty"` } // +kubebuilder:object:root=true // BatchJobList contains a list of BatchJob type BatchJobList struct { kmeta.TypeMeta `json:",inline"` kmeta.ListMeta `json:"metadata,omitempty"` Items []BatchJob `json:"items"` } func init() { SchemeBuilder.Register(&BatchJob{}, &BatchJobList{}) } ================================================ FILE: pkg/crds/apis/batch/v1alpha1/groupversion_info.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ // Package v1alpha1 contains API Schema definitions for the batch v1alpha1 API group // +kubebuilder:object:generate=true // +groupName=batch.cortex.dev package v1alpha1 import ( "k8s.io/apimachinery/pkg/runtime/schema" "sigs.k8s.io/controller-runtime/pkg/scheme" ) var ( // GroupVersion is group version used to register these objects GroupVersion = schema.GroupVersion{Group: "batch.cortex.dev", Version: "v1alpha1"} // SchemeBuilder is used to add go types to the GroupVersionKind scheme SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} // AddToScheme adds the types in this group-version to the given scheme. AddToScheme = SchemeBuilder.AddToScheme ) ================================================ FILE: pkg/crds/apis/batch/v1alpha1/zz_generated.deepcopy.go ================================================ //go:build !ignore_autogenerated // +build !ignore_autogenerated /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ // Code generated by controller-gen. DO NOT EDIT. package v1alpha1 import ( "github.com/cortexlabs/cortex/pkg/types/status" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BatchJob) DeepCopyInto(out *BatchJob) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) in.Spec.DeepCopyInto(&out.Spec) in.Status.DeepCopyInto(&out.Status) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BatchJob. func (in *BatchJob) DeepCopy() *BatchJob { if in == nil { return nil } out := new(BatchJob) in.DeepCopyInto(out) return out } // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. func (in *BatchJob) DeepCopyObject() runtime.Object { if c := in.DeepCopy(); c != nil { return c } return nil } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BatchJobList) DeepCopyInto(out *BatchJobList) { *out = *in out.TypeMeta = in.TypeMeta in.ListMeta.DeepCopyInto(&out.ListMeta) if in.Items != nil { in, out := &in.Items, &out.Items *out = make([]BatchJob, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BatchJobList. func (in *BatchJobList) DeepCopy() *BatchJobList { if in == nil { return nil } out := new(BatchJobList) in.DeepCopyInto(out) return out } // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. func (in *BatchJobList) DeepCopyObject() runtime.Object { if c := in.DeepCopy(); c != nil { return c } return nil } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BatchJobSpec) DeepCopyInto(out *BatchJobSpec) { *out = *in if in.Config != nil { in, out := &in.Config, &out.Config *out = new(string) **out = **in } if in.Timeout != nil { in, out := &in.Timeout, &out.Timeout *out = new(v1.Duration) **out = **in } if in.DeadLetterQueue != nil { in, out := &in.DeadLetterQueue, &out.DeadLetterQueue *out = new(DeadLetterQueueSpec) **out = **in } if in.Resources != nil { in, out := &in.Resources, &out.Resources *out = new(corev1.ResourceRequirements) (*in).DeepCopyInto(*out) } if in.NodeGroups != nil { in, out := &in.NodeGroups, &out.NodeGroups *out = make([]string, len(*in)) copy(*out, *in) } if in.Probes != nil { in, out := &in.Probes, &out.Probes *out = make(map[string]corev1.Probe, len(*in)) for key, val := range *in { (*out)[key] = *val.DeepCopy() } } if in.TTL != nil { in, out := &in.TTL, &out.TTL *out = new(v1.Duration) **out = **in } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BatchJobSpec. func (in *BatchJobSpec) DeepCopy() *BatchJobSpec { if in == nil { return nil } out := new(BatchJobSpec) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BatchJobStatus) DeepCopyInto(out *BatchJobStatus) { *out = *in if in.EndTime != nil { in, out := &in.EndTime, &out.EndTime *out = (*in).DeepCopy() } if in.WorkerCounts != nil { in, out := &in.WorkerCounts, &out.WorkerCounts *out = new(status.WorkerCounts) **out = **in } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BatchJobStatus. func (in *BatchJobStatus) DeepCopy() *BatchJobStatus { if in == nil { return nil } out := new(BatchJobStatus) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DeadLetterQueueSpec) DeepCopyInto(out *DeadLetterQueueSpec) { *out = *in } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DeadLetterQueueSpec. func (in *DeadLetterQueueSpec) DeepCopy() *DeadLetterQueueSpec { if in == nil { return nil } out := new(DeadLetterQueueSpec) in.DeepCopyInto(out) return out } ================================================ FILE: pkg/crds/config/crd/bases/batch.cortex.dev_batchjobs.yaml ================================================ --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.4.1 creationTimestamp: null name: batchjobs.batch.cortex.dev spec: group: batch.cortex.dev names: kind: BatchJob listKind: BatchJobList plural: batchjobs singular: batchjob scope: Namespaced versions: - additionalPrinterColumns: - jsonPath: .status.status name: Status type: string - jsonPath: .status.queue_url name: Queue URL type: string name: v1alpha1 schema: openAPIV3Schema: description: BatchJob is the Schema for the batchjobs API properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object spec: description: BatchJobSpec defines the desired state of BatchJob properties: api_id: description: Reference to a cortex BatchAPI apiID type: string api_name: description: Reference to a cortex BatchAPI name type: string config: description: YAML content of the user config type: string dead_letter_queue: description: Configuration for the dead letter queue properties: arn: description: arn of the dead letter queue e.g. arn:aws:sqs:us-west-2:123456789:failed.fifo type: string max_receive_count: default: 1 description: Number of times a batch is allowed to be handled by a worker before it is considered to be failed and transferred to the dead letter queue (must be >= 1) format: int32 minimum: 1 type: integer type: object node_groups: description: Node groups selector items: type: string nullable: true type: array probes: additionalProperties: description: Probe describes a health check to be performed against a container to determine whether it is alive or ready to receive traffic. properties: exec: description: One and only one of the following should be specified. Exec specifies the action to take. properties: command: description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. items: type: string type: array type: object failureThreshold: description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1. format: int32 type: integer httpGet: description: HTTPGet specifies the http request to perform. properties: host: description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. type: string httpHeaders: description: Custom headers to set in the request. HTTP allows repeated headers. items: description: HTTPHeader describes a custom header to be used in HTTP probes properties: name: description: The header field name type: string value: description: The header field value type: string required: - name - value type: object type: array path: description: Path to access on the HTTP server. type: string port: anyOf: - type: integer - type: string description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: description: Scheme to use for connecting to the host. Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer periodSeconds: description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1. format: int32 type: integer successThreshold: description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. format: int32 type: integer tcpSocket: description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' properties: host: description: 'Optional: Host name to connect to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object timeoutSeconds: description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer type: object description: Readiness probes for the job (container name -> probe) nullable: true type: object resources: description: Compute resource requirements properties: limits: additionalProperties: anyOf: - type: integer - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' type: object requests: additionalProperties: anyOf: - type: integer - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' type: object type: object timeout: description: Duration until a batch job times out type: string ttl: description: Time to live for the resource. The controller will clean-up resources that reached a final state when the TTL time is exceeded. type: string workers: default: 1 description: Number of workers for the batch job format: int32 type: integer type: object status: description: BatchJobStatus defines the observed state of BatchJob properties: end_time: description: Processing ending timestamp format: date-time type: string id: description: Job ID type: string queue_url: description: URL for the used SQS queue type: string status: description: Status of the batch job type: string total_batch_count: description: Total batch count type: integer worker_counts: description: Detailed worker counts with respective status properties: creating: format: int32 type: integer err_image_pull: format: int32 type: integer failed: format: int32 type: integer killed: format: int32 type: integer killed_oom: format: int32 type: integer not_ready: format: int32 type: integer pending: format: int32 type: integer ready: format: int32 type: integer stalled: format: int32 type: integer succeeded: format: int32 type: integer terminating: format: int32 type: integer unknown: format: int32 type: integer type: object type: object type: object served: true storage: true subresources: status: {} status: acceptedNames: kind: "" plural: "" conditions: [] storedVersions: [] ================================================ FILE: pkg/crds/config/crd/kustomization.yaml ================================================ # This kustomization.yaml is not intended to be run by itself, # since it depends on service name and namespace that are out of this kustomize package. # It should be run by config/default resources: - bases/batch.cortex.dev_batchjobs.yaml #+kubebuilder:scaffold:crdkustomizeresource patchesStrategicMerge: # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix. # patches here are for enabling the conversion webhook for each CRD #- patches/webhook_in_batchjobs.yaml #+kubebuilder:scaffold:crdkustomizewebhookpatch # [CERTMANAGER] To enable webhook, uncomment all the sections with [CERTMANAGER] prefix. # patches here are for enabling the CA injection for each CRD #- patches/cainjection_in_batchjobs.yaml #+kubebuilder:scaffold:crdkustomizecainjectionpatch # the following config is for teaching kustomize how to do kustomization for CRDs. configurations: - kustomizeconfig.yaml ================================================ FILE: pkg/crds/config/crd/kustomizeconfig.yaml ================================================ # This file is for teaching kustomize how to substitute name and namespace reference in CRD nameReference: - kind: Service version: v1 fieldSpecs: - kind: CustomResourceDefinition version: v1 group: apiextensions.k8s.io path: spec/conversion/webhook/clientConfig/service/name namespace: - kind: CustomResourceDefinition version: v1 group: apiextensions.k8s.io path: spec/conversion/webhook/clientConfig/service/namespace create: false varReference: - path: metadata/annotations ================================================ FILE: pkg/crds/config/crd/patches/cainjection_in_batchjobs.yaml ================================================ # The following patch adds a directive for certmanager to inject CA into the CRD apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) name: batchjobs.batch.cortex.dev ================================================ FILE: pkg/crds/config/crd/patches/webhook_in_batchjobs.yaml ================================================ # The following patch enables a conversion webhook for the CRD apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: batchjobs.batch.cortex.dev spec: conversion: strategy: Webhook webhook: clientConfig: service: namespace: system name: webhook-service path: /convert ================================================ FILE: pkg/crds/config/default/kustomization.yaml ================================================ # Adds namespace to all resources. namespace: default # Value of this field is prepended to the # names of all resources, e.g. a deployment named # "wordpress" becomes "alices-wordpress". # Note that it should also match with the prefix (text before '-') of the namespace # field above. namePrefix: operator- # Labels to add to all resources and selectors. #commonLabels: # someName: someValue bases: - ../crd - ../rbac - ../manager # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml #- ../webhook # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required. #- ../certmanager # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. #- ../prometheus patchesStrategicMerge: # Protect the /metrics endpoint by putting it behind auth. # If you want your controller-manager to expose the /metrics # endpoint w/o any authn/z, please comment the following line. #- manager_auth_proxy_patch.yaml # Mount the controller config file for loading manager configurations # through a ComponentConfig type #- manager_config_patch.yaml # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml #- manager_webhook_patch.yaml # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. # Uncomment 'CERTMANAGER' sections in crd/kustomization.yaml to enable the CA injection in the admission webhooks. # 'CERTMANAGER' needs to be enabled to use ca injection #- webhookcainjection_patch.yaml # the following config is for teaching kustomize how to do var substitution vars: # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. #- name: CERTIFICATE_NAMESPACE # namespace of the certificate CR # objref: # kind: Certificate # group: cert-manager.io # version: v1 # name: serving-cert # this name should match the one in certificate.yaml # fieldref: # fieldpath: metadata.namespace #- name: CERTIFICATE_NAME # objref: # kind: Certificate # group: cert-manager.io # version: v1 # name: serving-cert # this name should match the one in certificate.yaml #- name: SERVICE_NAMESPACE # namespace of the service # objref: # kind: Service # version: v1 # name: webhook-service # fieldref: # fieldpath: metadata.namespace #- name: SERVICE_NAME # objref: # kind: Service # version: v1 # name: webhook-service ================================================ FILE: pkg/crds/config/default/manager_auth_proxy_patch.yaml ================================================ # This patch inject a sidecar container which is a HTTP proxy for the # controller manager, it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews. apiVersion: apps/v1 kind: Deployment metadata: name: controller-manager namespace: system spec: template: spec: containers: - name: kube-rbac-proxy image: gcr.io/kubebuilder/kube-rbac-proxy:v0.8.0 args: - "--secure-listen-address=0.0.0.0:8443" - "--upstream=http://127.0.0.1:8080/" - "--logtostderr=true" - "--v=10" ports: - containerPort: 8443 name: https - name: manager args: - "--health-probe-bind-address=:8081" - "--metrics-bind-address=127.0.0.1:8080" - "--leader-elect" ================================================ FILE: pkg/crds/config/default/manager_config_patch.yaml ================================================ apiVersion: apps/v1 kind: Deployment metadata: name: controller-manager namespace: system spec: template: spec: containers: - name: manager args: - "--config=controller_manager_config.yaml" volumeMounts: - name: manager-config mountPath: /controller_manager_config.yaml subPath: controller_manager_config.yaml volumes: - name: manager-config configMap: name: manager-config ================================================ FILE: pkg/crds/config/manager/controller_manager_config.yaml ================================================ apiVersion: controller-runtime.sigs.k8s.io/v1alpha1 kind: ControllerManagerConfig health: healthProbeBindAddress: :8081 metrics: bindAddress: 127.0.0.1:8080 webhook: port: 9443 leaderElection: leaderElect: true resourceName: 7cc92962.cortex.dev ================================================ FILE: pkg/crds/config/manager/kustomization.yaml ================================================ resources: - manager.yaml generatorOptions: disableNameSuffixHash: true configMapGenerator: - files: - controller_manager_config.yaml name: manager-config apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization ================================================ FILE: pkg/crds/config/manager/manager.yaml ================================================ --- apiVersion: apps/v1 kind: Deployment metadata: name: controller-manager namespace: system labels: control-plane: controller-manager spec: selector: matchLabels: control-plane: controller-manager replicas: 1 strategy: rollingUpdate: maxSurge: 0 template: metadata: labels: control-plane: controller-manager spec: securityContext: runAsNonRoot: true containers: - name: manager command: - /manager args: - "--config=/mnt/cluster.yaml" - "--leader-elect" image: controller:latest imagePullPolicy: Always securityContext: allowPrivilegeEscalation: false env: - name: CORTEX_OPERATOR_IN_CLUSTER value: "true" livenessProbe: httpGet: path: /healthz port: 8081 initialDelaySeconds: 15 periodSeconds: 20 readinessProbe: httpGet: path: /readyz port: 8081 initialDelaySeconds: 5 periodSeconds: 10 resources: limits: cpu: 300m memory: 100Mi requests: cpu: 100m memory: 80Mi envFrom: - configMapRef: name: env-vars volumeMounts: - mountPath: /mnt/cluster.yaml name: cluster-config subPath: cluster.yaml volumes: - name: cluster-config configMap: name: cluster-config serviceAccountName: controller-manager terminationGracePeriodSeconds: 10 ================================================ FILE: pkg/crds/config/prometheus/kustomization.yaml ================================================ resources: - monitor.yaml ================================================ FILE: pkg/crds/config/prometheus/monitor.yaml ================================================ # Prometheus Monitor Service (Metrics) apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor metadata: labels: control-plane: controller-manager name: controller-manager-metrics-monitor namespace: system spec: endpoints: - path: /metrics port: https scheme: https bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token tlsConfig: insecureSkipVerify: true selector: matchLabels: control-plane: controller-manager ================================================ FILE: pkg/crds/config/rbac/auth_proxy_client_clusterrole.yaml ================================================ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: metrics-reader rules: - nonResourceURLs: - "/metrics" verbs: - get ================================================ FILE: pkg/crds/config/rbac/auth_proxy_role.yaml ================================================ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: proxy-role rules: - apiGroups: - authentication.k8s.io resources: - tokenreviews verbs: - create - apiGroups: - authorization.k8s.io resources: - subjectaccessreviews verbs: - create ================================================ FILE: pkg/crds/config/rbac/auth_proxy_role_binding.yaml ================================================ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: proxy-rolebinding roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: proxy-role subjects: - kind: ServiceAccount name: controller-manager namespace: system ================================================ FILE: pkg/crds/config/rbac/auth_proxy_service.yaml ================================================ apiVersion: v1 kind: Service metadata: labels: control-plane: controller-manager name: controller-manager-metrics-service namespace: system spec: ports: - name: https port: 8443 targetPort: https selector: control-plane: controller-manager ================================================ FILE: pkg/crds/config/rbac/batchjob_editor_role.yaml ================================================ # permissions for end users to edit batchjobs. apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: batchjob-editor-role rules: - apiGroups: - batch.cortex.dev resources: - batchjobs verbs: - create - delete - get - list - patch - update - watch - apiGroups: - batch.cortex.dev resources: - batchjobs/status verbs: - get ================================================ FILE: pkg/crds/config/rbac/batchjob_viewer_role.yaml ================================================ # permissions for end users to view batchjobs. apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: batchjob-viewer-role rules: - apiGroups: - batch.cortex.dev resources: - batchjobs verbs: - get - list - watch - apiGroups: - batch.cortex.dev resources: - batchjobs/status verbs: - get ================================================ FILE: pkg/crds/config/rbac/kustomization.yaml ================================================ resources: # All RBAC will be applied under this service account in # the deployment namespace. You may comment out this resource # if your manager will use a service account that exists at # runtime. Be sure to update RoleBinding and ClusterRoleBinding # subjects if changing service account names. - service_account.yaml - role.yaml - role_binding.yaml - leader_election_role.yaml - leader_election_role_binding.yaml # Comment the following 4 lines if you want to disable # the auth proxy (https://github.com/brancz/kube-rbac-proxy) # which protects your /metrics endpoint. - auth_proxy_service.yaml - auth_proxy_role.yaml - auth_proxy_role_binding.yaml - auth_proxy_client_clusterrole.yaml ================================================ FILE: pkg/crds/config/rbac/leader_election_role.yaml ================================================ # permissions to do leader election. apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: leader-election-role rules: - apiGroups: - "" resources: - configmaps verbs: - get - list - watch - create - update - patch - delete - apiGroups: - coordination.k8s.io resources: - leases verbs: - get - list - watch - create - update - patch - delete - apiGroups: - "" resources: - events verbs: - create - patch ================================================ FILE: pkg/crds/config/rbac/leader_election_role_binding.yaml ================================================ apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: leader-election-rolebinding roleRef: apiGroup: rbac.authorization.k8s.io kind: Role name: leader-election-role subjects: - kind: ServiceAccount name: controller-manager namespace: system ================================================ FILE: pkg/crds/config/rbac/role.yaml ================================================ --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: creationTimestamp: null name: manager-role rules: - apiGroups: - "" resources: - configmaps verbs: - create - get - list - watch - apiGroups: - "" resources: - pods verbs: - get - list - watch - apiGroups: - batch resources: - jobs verbs: - create - get - list - patch - update - watch - apiGroups: - batch.cortex.dev resources: - batchjobs verbs: - create - delete - get - list - patch - update - watch - apiGroups: - batch.cortex.dev resources: - batchjobs/finalizers verbs: - update - apiGroups: - batch.cortex.dev resources: - batchjobs/status verbs: - get - patch - update ================================================ FILE: pkg/crds/config/rbac/role_binding.yaml ================================================ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: manager-rolebinding roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: manager-role subjects: - kind: ServiceAccount name: controller-manager namespace: system ================================================ FILE: pkg/crds/config/rbac/service_account.yaml ================================================ apiVersion: v1 kind: ServiceAccount metadata: name: controller-manager namespace: system ================================================ FILE: pkg/crds/config/samples/batch_v1alpha1_batchjob.yaml ================================================ apiVersion: batch.cortex.dev/v1alpha1 kind: BatchJob metadata: name: "123456" spec: api_name: "image-classifier" api_id: "123456" workers: 1 config: | dest_s3_dir: s3://abc/123 ttl: "10s" ================================================ FILE: pkg/crds/controllers/batch/batchjob_controller.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package batchcontrollers import ( "context" "time" batch "github.com/cortexlabs/cortex/pkg/crds/apis/batch/v1alpha1" "github.com/cortexlabs/cortex/pkg/crds/controllers" awslib "github.com/cortexlabs/cortex/pkg/lib/aws" "github.com/cortexlabs/cortex/pkg/lib/slices" "github.com/cortexlabs/cortex/pkg/types/clusterconfig" "github.com/cortexlabs/cortex/pkg/types/userconfig" "github.com/go-logr/logr" promv1 "github.com/prometheus/client_golang/api/prometheus/v1" kbatch "k8s.io/api/batch/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" ) const ( _sqsFinalizer = "sqs.finalizers.batch.cortex.dev" _s3Finalizer = "s3.finalizers.batch.cortex.dev" _completedTimestampAnnotation = "batch.cortex.dev/completed_timestamp" ) // BatchJobReconciler reconciles a BatchJob object type BatchJobReconciler struct { client.Client Config BatchJobReconcilerConfig Log logr.Logger AWS *awslib.Client ClusterConfig *clusterconfig.Config Prometheus promv1.API Scheme *runtime.Scheme } // +kubebuilder:rbac:groups=batch.cortex.dev,resources=batchjobs,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=batch.cortex.dev,resources=batchjobs/status,verbs=get;update;patch // +kubebuilder:rbac:groups=batch.cortex.dev,resources=batchjobs/finalizers,verbs=update // +kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;watch;create;update;patch // +kubebuilder:rbac:groups="",resources=pods,verbs=get;list;watch // +kubebuilder:rbac:groups="",resources=configmaps,verbs=create;get;list;watch // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. func (r *BatchJobReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { log := r.Log.WithValues("cortex.labels", map[string]string{ "batchjob": req.NamespacedName.String(), "apiKind": userconfig.BatchAPIKind.String(), }, ) // Step 1: get resource from request batchJob := batch.BatchJob{} log.V(1).Info("retrieving resource") if err := r.Get(ctx, req.NamespacedName, &batchJob); err != nil { if !kerrors.IsNotFound(err) { log.Error(err, "failed to retrieve resource") } return ctrl.Result{}, client.IgnoreNotFound(err) } log = r.Log.WithValues("cortex.labels", map[string]string{ "batchjob": req.NamespacedName.String(), "apiKind": userconfig.BatchAPIKind.String(), "apiName": batchJob.Spec.APIName, "apiID": batchJob.Spec.APIID, "jobID": batchJob.Name, }, ) // Step 2: create finalizer or handle deletion if batchJob.ObjectMeta.DeletionTimestamp.IsZero() { // The object is not being deleted, so we add our finalizer if it does not exist yet, if !slices.HasString(batchJob.ObjectMeta.Finalizers, _sqsFinalizer) && !slices.HasString(batchJob.ObjectMeta.Finalizers, _s3Finalizer) { log.V(1).Info("adding finalizers") batchJob.ObjectMeta.Finalizers = append(batchJob.ObjectMeta.Finalizers, _sqsFinalizer, _s3Finalizer) if err := r.Update(ctx, &batchJob); err != nil { log.Error(err, "failed to add finalizers to resource") return ctrl.Result{}, err } } } else { // The object is being deleted if slices.HasString(batchJob.ObjectMeta.Finalizers, _sqsFinalizer) { // our finalizer is present, so lets handle any external dependency log.V(1).Info("deleting SQS queue") if err := r.deleteSQSQueue(batchJob); err != nil { log.Error(err, "failed to delete SQS queue") return ctrl.Result{}, err } log.V(1).Info("removing sqs finalizer") // remove our finalizer from the list and update it. batchJob.ObjectMeta.Finalizers = slices.RemoveString(batchJob.ObjectMeta.Finalizers, _sqsFinalizer) if err := r.Update(ctx, &batchJob); err != nil { log.Error(err, "failed to remove sqs finalizer from resource") return ctrl.Result{}, err } return ctrl.Result{}, nil // return here because the status update will trigger another reconcile } if slices.HasString(batchJob.ObjectMeta.Finalizers, _s3Finalizer) { log.V(1).Info("persisting job to S3") if err := r.persistJobToS3(batchJob); err != nil { log.Error(err, "failed to persist job to S3") return ctrl.Result{}, err } log.V(1).Info("removing S3 finalizer") batchJob.ObjectMeta.Finalizers = slices.RemoveString(batchJob.ObjectMeta.Finalizers, _s3Finalizer) if err := r.Update(ctx, &batchJob); err != nil { log.Error(err, "failed to remove S3 finalizer from resource") return ctrl.Result{}, err } return ctrl.Result{}, nil // return here because the status update will trigger another reconcile } return ctrl.Result{}, nil } // Step 3: Update Status log.V(1).Info("checking if queue exists") queueExists, err := r.checkIfQueueExists(batchJob) if err != nil { log.Error(err, "failed to check if queue exists") return ctrl.Result{}, err } log.V(1).Info("getting configmap") configMap, err := r.getConfigMap(ctx, batchJob) if err != nil && !kerrors.IsNotFound(err) { log.Error(err, "failed to get configmap") return ctrl.Result{}, err } log.V(1).Info("getting worker job") workerJob, err := r.getWorkerJob(ctx, batchJob) if err != nil && !kerrors.IsNotFound(err) { log.Error(err, "failed to get worker job") return ctrl.Result{}, err } log.V(1).Info("checking enqueuing status") enqueuerJob, enqueuingStatus, err := r.checkEnqueuingStatus(ctx, batchJob) if err != nil { log.Error(err, "failed to check enqueuing status") return ctrl.Result{}, err } var totalBatchCount int if enqueuingStatus == batch.EnqueuingDone { totalBatchCount, err = r.Config.GetTotalBatchCount(r, batchJob) } workerJobExists := workerJob != nil configMapExists := configMap != nil statusInfo := batchJobStatusInfo{ QueueExists: queueExists, EnqueuingStatus: enqueuingStatus, EnqueuerJob: enqueuerJob, WorkerJob: workerJob, TotalBatchCount: totalBatchCount, } log.V(1).Info("status data successfully acquired", "queueExists", queueExists, "configMapExists", configMapExists, "enqueuingStatus", enqueuingStatus, "workerJobExists", workerJobExists, ) log.V(1).Info("updating status") if err = r.updateStatus(ctx, &batchJob, statusInfo); err != nil { if controllers.IsOptimisticLockError(err) { log.Info("conflict during status update, retrying") return ctrl.Result{Requeue: true}, nil } log.Error(err, "failed to update status") return ctrl.Result{}, err } log.V(1).Info("current job status", "jobStatus", batchJob.Status.Status) // Step 4: Add a completion timestamp annotation if job is in a completed state var completedTimestamp *time.Time if batchJob.Status.Status.IsCompleted() { completedTimestampStr, completedTimestampExists := batchJob.Annotations[_completedTimestampAnnotation] if !completedTimestampExists { if err = r.updateCompletedTimestamp(ctx, &batchJob); err != nil { log.Error(err, "failed to update completed timestamp annotation") return ctrl.Result{}, err } return ctrl.Result{}, nil } ts, err := time.Parse(time.RFC3339, completedTimestampStr) if err != nil { log.Error(err, "failed to parse completed timestamp string") return ctrl.Result{}, err } completedTimestamp = &ts } // Step 5: Create resources var queueURL string if !queueExists { log.Info("creating queue") queueURL, err = r.createQueue(batchJob) if err != nil { log.Error(err, "failed to create queue") return ctrl.Result{}, err } } else { queueURL = r.getQueueURL(batchJob) } switch enqueuingStatus { case batch.EnqueuingNotStarted: log.Info("enqueuing payload") if err = r.enqueuePayload(ctx, batchJob, queueURL); err != nil { log.Error(err, "failed to start enqueuing the payload") return ctrl.Result{}, err } return ctrl.Result{}, nil case batch.EnqueuingInProgress: // wait for enqueuing process to be reach a final state (done|failed) return ctrl.Result{}, nil case batch.EnqueuingFailed: log.Info("failed to enqueue payload") case batch.EnqueuingDone: if !configMapExists { log.V(1).Info("creating worker configmap") if err = r.createWorkerConfigMap(ctx, batchJob, queueURL); err != nil { log.Error(err, "failed to create worker configmap") return ctrl.Result{}, err } } if !workerJobExists { log.Info("creating worker job") if err = r.createWorkerJob(ctx, batchJob, queueURL); err != nil { log.Error(err, "failed to create worker job") return ctrl.Result{}, err } } } // Step 6: Delete self if TTL is enabled and reached a final state if batchJob.Spec.TTL != nil && completedTimestamp != nil { afterFinishedDuration := time.Since(*completedTimestamp) if afterFinishedDuration >= batchJob.Spec.TTL.Duration { if err = r.Delete(ctx, &batchJob); err != nil { return ctrl.Result{}, client.IgnoreNotFound(err) } log.Info("TTL exceeded, deleting resource") return ctrl.Result{}, nil } log.V(1).Info("scheduling reconciliation requeue", "time", batchJob.Spec.TTL.Duration) return ctrl.Result{RequeueAfter: batchJob.Spec.TTL.Duration}, nil } return ctrl.Result{}, nil } // SetupWithManager sets up the controller with the Manager. func (r *BatchJobReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&batch.BatchJob{}). Owns(&kbatch.Job{}). Complete(r) } ================================================ FILE: pkg/crds/controllers/batch/batchjob_controller_config.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package batchcontrollers import ( batch "github.com/cortexlabs/cortex/pkg/crds/apis/batch/v1alpha1" "github.com/cortexlabs/cortex/pkg/types/metrics" ) // BatchJobReconcilerConfig reconciler config for the BatchJob kind. Allows for mocking specific methods type BatchJobReconcilerConfig struct { GetTotalBatchCount func(r *BatchJobReconciler, batchJob batch.BatchJob) (int, error) GetMetrics func(r *BatchJobReconciler, batchJob batch.BatchJob) (metrics.BatchMetrics, error) SaveJobMetrics func(r *BatchJobReconciler, batchJob batch.BatchJob) error SaveJobStatus func(r *BatchJobReconciler, batchJob batch.BatchJob) error } // ApplyDefaults sets the defaults for BatchJobReconcilerConfig func (c BatchJobReconcilerConfig) ApplyDefaults() BatchJobReconcilerConfig { if c.GetTotalBatchCount == nil { c.GetTotalBatchCount = getTotalBatchCount } if c.GetMetrics == nil { c.GetMetrics = getMetrics } if c.SaveJobMetrics == nil { c.SaveJobMetrics = saveJobMetrics } if c.SaveJobStatus == nil { c.SaveJobStatus = saveJobStatus } return c } ================================================ FILE: pkg/crds/controllers/batch/batchjob_controller_helpers.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package batchcontrollers import ( "context" "encoding/json" "fmt" "path/filepath" "strconv" "time" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/service/sqs" "github.com/cortexlabs/cortex/pkg/consts" batch "github.com/cortexlabs/cortex/pkg/crds/apis/batch/v1alpha1" "github.com/cortexlabs/cortex/pkg/lib/errors" libjson "github.com/cortexlabs/cortex/pkg/lib/json" "github.com/cortexlabs/cortex/pkg/lib/k8s" "github.com/cortexlabs/cortex/pkg/lib/parallel" "github.com/cortexlabs/cortex/pkg/lib/pointer" s "github.com/cortexlabs/cortex/pkg/lib/strings" "github.com/cortexlabs/cortex/pkg/types/clusterconfig" "github.com/cortexlabs/cortex/pkg/types/metrics" "github.com/cortexlabs/cortex/pkg/types/spec" "github.com/cortexlabs/cortex/pkg/types/status" "github.com/cortexlabs/cortex/pkg/types/userconfig" "github.com/cortexlabs/cortex/pkg/workloads" "github.com/cortexlabs/yaml" cache "github.com/patrickmn/go-cache" kbatch "k8s.io/api/batch/v1" kcore "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" ) const ( _enqueuerContainerName = "enqueuer" _deadlineExceededReason = "DeadlineExceeded" _cacheDuration = 60 * time.Second ) var totalBatchCountCache, apiSpecCache *cache.Cache func init() { totalBatchCountCache = cache.New(_cacheDuration, _cacheDuration) apiSpecCache = cache.New(_cacheDuration, _cacheDuration) } type batchJobStatusInfo struct { QueueExists bool EnqueuingStatus batch.EnqueuingStatus EnqueuerJob *kbatch.Job WorkerJob *kbatch.Job TotalBatchCount int } func (r *BatchJobReconciler) getConfigMap(ctx context.Context, batchJob batch.BatchJob) (*kcore.ConfigMap, error) { var configMap kcore.ConfigMap err := r.Get(ctx, client.ObjectKey{ Namespace: batchJob.Namespace, Name: batchJob.Spec.APIName + "-" + batchJob.Name, }, &configMap) if err != nil { if kerrors.IsNotFound(err) { return nil, nil } return nil, err } return &configMap, nil } func (r *BatchJobReconciler) checkIfQueueExists(batchJob batch.BatchJob) (bool, error) { queueName := r.getQueueName(batchJob) input := &sqs.GetQueueUrlInput{ QueueName: aws.String(queueName), } _, err := r.AWS.SQS().GetQueueUrl(input) if err != nil { if aerr, ok := err.(awserr.Error); ok { if aerr.Code() == sqs.ErrCodeQueueDoesNotExist { return false, nil } } return false, err } return true, nil } func (r *BatchJobReconciler) createQueue(batchJob batch.BatchJob) (string, error) { queueName := r.getQueueName(batchJob) tags := map[string]string{ clusterconfig.ClusterNameTag: r.ClusterConfig.ClusterName, "apiName": batchJob.Spec.APIName, "apiID": batchJob.Spec.APIID, "jobID": batchJob.Name, } attributes := map[string]string{ sqs.QueueAttributeNameFifoQueue: "true", sqs.QueueAttributeNameVisibilityTimeout: "60", } if batchJob.Spec.DeadLetterQueue != nil { redrivePolicy := map[string]string{ "deadLetterTargetArn": batchJob.Spec.DeadLetterQueue.ARN, "maxReceiveCount": s.Int32(batchJob.Spec.DeadLetterQueue.MaxReceiveCount), } redrivePolicyJSONBytes, err := libjson.Marshal(redrivePolicy) if err != nil { return "", err } attributes[sqs.QueueAttributeNameRedrivePolicy] = string(redrivePolicyJSONBytes) } input := &sqs.CreateQueueInput{ Attributes: aws.StringMap(attributes), QueueName: aws.String(queueName), Tags: aws.StringMap(tags), } output, err := r.AWS.SQS().CreateQueue(input) if err != nil { return "", err } return *output.QueueUrl, nil } func (r *BatchJobReconciler) getQueueURL(batchJob batch.BatchJob) string { // e.g. https://sqs..amazonaws.com// return fmt.Sprintf( "https://sqs.%s.amazonaws.com/%s/%s", r.ClusterConfig.Region, r.ClusterConfig.AccountID, r.getQueueName(batchJob), ) } func (r *BatchJobReconciler) getQueueName(batchJob batch.BatchJob) string { // cx__b__.fifo return clusterconfig.SQSNamePrefix(r.ClusterConfig.ClusterName) + "b" + clusterconfig.SQSQueueDelimiter + batchJob.Spec.APIName + clusterconfig.SQSQueueDelimiter + batchJob.Name + ".fifo" } func (r *BatchJobReconciler) checkEnqueuingStatus(ctx context.Context, batchJob batch.BatchJob) (*kbatch.Job, batch.EnqueuingStatus, error) { var enqueuerJob kbatch.Job if err := r.Get(ctx, client.ObjectKey{ Namespace: batchJob.Namespace, Name: batchJob.Spec.APIName + "-" + batchJob.Name + "-enqueuer", }, &enqueuerJob, ); err != nil { if kerrors.IsNotFound(err) { return nil, batch.EnqueuingNotStarted, nil } return nil, "", err } enqueuerStatus := enqueuerJob.Status switch { case enqueuerStatus.Failed > 0: return &enqueuerJob, batch.EnqueuingFailed, nil case enqueuerStatus.Succeeded > 0: return &enqueuerJob, batch.EnqueuingDone, nil case enqueuerStatus.Active > 0: return &enqueuerJob, batch.EnqueuingInProgress, nil } return &enqueuerJob, batch.EnqueuingInProgress, nil } func (r *BatchJobReconciler) enqueuePayload(ctx context.Context, batchJob batch.BatchJob, queueURL string) error { enqueuerJob, err := r.desiredEnqueuerJob(batchJob, queueURL) if err != nil { return err } if err = r.Create(ctx, enqueuerJob); err != nil { return err } return nil } func (r *BatchJobReconciler) createWorkerConfigMap(ctx context.Context, batchJob batch.BatchJob, queueURL string) error { apiSpec, err := r.getAPISpec(batchJob) if err != nil { return errors.Wrap(err, "failed to get API spec") } jobSpec, err := r.ConvertControllerBatchToJobSpec(batchJob, *apiSpec, queueURL) if err != nil { return errors.Wrap(err, "failed to convert controller batch job to operator batch job") } configMapConfig := workloads.ConfigMapConfig{ BatchJob: &jobSpec, Probes: batchJob.Spec.Probes, } configMapData, err := configMapConfig.GenerateConfigMapData() if err != nil { return errors.Wrap(err, "failed to generate config map data") } configMap, err := r.desiredConfigMap(batchJob, configMapData) if err != nil { return errors.Wrap(err, "failed to get desired configmap spec") } if err := r.Create(ctx, configMap); err != nil { return err } return nil } func (r *BatchJobReconciler) createWorkerJob(ctx context.Context, batchJob batch.BatchJob, queueURL string) error { apiSpec, err := r.getAPISpec(batchJob) if err != nil { return errors.Wrap(err, "failed to get API spec") } jobSpec, err := r.uploadJobSpec(batchJob, *apiSpec, queueURL) if err != nil { return errors.Wrap(err, "failed to upload job spec") } workerJob, err := r.desiredWorkerJob(batchJob, *apiSpec, *jobSpec) if err != nil { return errors.Wrap(err, "failed to get desired worker job") } if err = r.Create(ctx, workerJob); err != nil { return err } return nil } func (r *BatchJobReconciler) desiredEnqueuerJob(batchJob batch.BatchJob, queueURL string) (*kbatch.Job, error) { job := k8s.Job( &k8s.JobSpec{ Name: batchJob.Spec.APIName + "-" + batchJob.Name + "-enqueuer", Namespace: batchJob.Namespace, Parallelism: 1, Labels: map[string]string{ "apiKind": userconfig.BatchAPIKind.String(), "apiName": batchJob.Spec.APIName, "apiID": batchJob.Spec.APIID, "jobID": batchJob.Name, "cortex.dev/api": "true", "cortex.dev/batch": "enqueuer", }, PodSpec: k8s.PodSpec{ Labels: map[string]string{ "apiKind": userconfig.BatchAPIKind.String(), "apiName": batchJob.Spec.APIName, "apiID": batchJob.Spec.APIID, "jobID": batchJob.Name, "cortex.dev/api": "true", "cortex.dev/batch": "enqueuer", }, Annotations: map[string]string{ "traffic.sidecar.istio.io/excludeOutboundIPRanges": "0.0.0.0/0", "cluster-autoscaler.kubernetes.io/safe-to-evict": "false", }, K8sPodSpec: kcore.PodSpec{ RestartPolicy: kcore.RestartPolicyNever, Containers: []kcore.Container{ { Name: _enqueuerContainerName, Image: r.ClusterConfig.ImageEnqueuer, Args: []string{ "-cluster-uid", r.ClusterConfig.ClusterUID, "-region", r.ClusterConfig.Region, "-bucket", r.ClusterConfig.Bucket, "-queue", queueURL, "-apiName", batchJob.Spec.APIName, "-jobID", batchJob.Name, }, Env: workloads.BaseEnvVars, EnvFrom: workloads.BaseClusterEnvVars(), ImagePullPolicy: kcore.PullAlways, }, }, NodeSelector: workloads.NodeSelectors(), Tolerations: workloads.GenerateResourceTolerations(), Affinity: workloads.GenerateNodeAffinities(batchJob.Spec.NodeGroups), ServiceAccountName: workloads.ServiceAccountName, }, }, }, ) if err := ctrl.SetControllerReference(&batchJob, job, r.Scheme); err != nil { return nil, err } return job, nil } func (r *BatchJobReconciler) desiredWorkerJob(batchJob batch.BatchJob, apiSpec spec.API, jobSpec spec.BatchJob) (*kbatch.Job, error) { containers, volumes := workloads.BatchContainers(apiSpec, &jobSpec) job := k8s.Job( &k8s.JobSpec{ Name: batchJob.Spec.APIName + "-" + batchJob.Name, Namespace: batchJob.Namespace, Parallelism: batchJob.Spec.Workers, Labels: map[string]string{ "apiKind": userconfig.BatchAPIKind.String(), "apiName": batchJob.Spec.APIName, "apiID": batchJob.Spec.APIID, "specID": apiSpec.SpecID, "podID": apiSpec.PodID, "jobID": batchJob.Name, "cortex.dev/api": "true", "cortex.dev/batch": "worker", }, PodSpec: k8s.PodSpec{ Labels: map[string]string{ "apiKind": userconfig.BatchAPIKind.String(), "apiName": batchJob.Spec.APIName, "apiID": batchJob.Spec.APIID, "specID": apiSpec.SpecID, "podID": apiSpec.PodID, "jobID": batchJob.Name, "cortex.dev/api": "true", "cortex.dev/batch": "worker", }, Annotations: map[string]string{ "traffic.sidecar.istio.io/excludeOutboundIPRanges": "0.0.0.0/0", "cluster-autoscaler.kubernetes.io/safe-to-evict": "false", }, K8sPodSpec: kcore.PodSpec{ InitContainers: []kcore.Container{ workloads.KubexitInitContainer(), }, Containers: containers, Volumes: volumes, RestartPolicy: kcore.RestartPolicyNever, NodeSelector: workloads.NodeSelectors(), Affinity: workloads.GenerateNodeAffinities(batchJob.Spec.NodeGroups), Tolerations: workloads.GenerateResourceTolerations(), ServiceAccountName: workloads.ServiceAccountName, }, }, }, ) if batchJob.Spec.Timeout != nil { job.Spec.ActiveDeadlineSeconds = pointer.Int64(int64(batchJob.Spec.Timeout.Seconds())) } if err := ctrl.SetControllerReference(&batchJob, job, r.Scheme); err != nil { return nil, err } return job, nil } func (r *BatchJobReconciler) desiredConfigMap(batchJob batch.BatchJob, data map[string]string) (*kcore.ConfigMap, error) { configMap := k8s.ConfigMap(&k8s.ConfigMapSpec{ Name: batchJob.Spec.APIName + "-" + batchJob.Name, Data: data, Labels: map[string]string{ "apiKind": userconfig.BatchAPIKind.String(), "apiName": batchJob.Spec.APIName, "apiID": batchJob.Spec.APIID, "jobID": batchJob.Name, "cortex.dev/api": "true", "cortex.dev/batch": "config", }, }) configMap.Namespace = batchJob.Namespace if err := ctrl.SetControllerReference(&batchJob, configMap, r.Scheme); err != nil { return nil, err } return configMap, nil } func (r *BatchJobReconciler) getAPISpec(batchJob batch.BatchJob) (*spec.API, error) { apiSpecKey := spec.Key(batchJob.Spec.APIName, batchJob.Spec.APIID, r.ClusterConfig.ClusterUID) cachedAPISpec, found := apiSpecCache.Get(apiSpecKey) var apiSpec spec.API if found { apiSpec = cachedAPISpec.(spec.API) return &apiSpec, nil } apiSpecBytes, err := r.AWS.ReadBytesFromS3(r.ClusterConfig.Bucket, apiSpecKey) if err != nil { return nil, err } if err := json.Unmarshal(apiSpecBytes, &apiSpec); err != nil { return nil, err } apiSpecCache.Set(apiSpecKey, apiSpec, _cacheDuration) return &apiSpec, nil } func (r *BatchJobReconciler) getWorkerJob(ctx context.Context, batchJob batch.BatchJob) (*kbatch.Job, error) { workerName := batchJob.Spec.APIName + "-" + batchJob.Name var job kbatch.Job err := r.Get(ctx, client.ObjectKey{Namespace: batchJob.Namespace, Name: workerName}, &job) if err != nil { if kerrors.IsNotFound(err) { return nil, nil } return nil, err } return &job, nil } func (r *BatchJobReconciler) getWorkerJobPods(ctx context.Context, batchJob batch.BatchJob) ([]kcore.Pod, error) { workerJobPods := kcore.PodList{} if err := r.List(ctx, &workerJobPods, client.InNamespace(consts.DefaultNamespace), client.MatchingLabels{ "jobID": batchJob.Name, "apiName": batchJob.Spec.APIName, "apiID": batchJob.Spec.APIID, "cortex.dev/batch": "worker", }, ); err != nil { return nil, err } return workerJobPods.Items, nil } func (r *BatchJobReconciler) updateStatus(ctx context.Context, batchJob *batch.BatchJob, statusInfo batchJobStatusInfo) error { batchJob.Status.ID = batchJob.Name if statusInfo.QueueExists { batchJob.Status.QueueURL = r.getQueueURL(*batchJob) } switch statusInfo.EnqueuingStatus { case batch.EnqueuingNotStarted: batchJob.Status.Status = status.JobPending case batch.EnqueuingInProgress: batchJob.Status.Status = status.JobEnqueuing case batch.EnqueuingFailed: batchJob.Status.Status = status.JobEnqueueFailed batchJob.Status.EndTime = statusInfo.EnqueuerJob.Status.CompletionTime case batch.EnqueuingDone: batchJob.Status.TotalBatchCount = statusInfo.TotalBatchCount } workerJobPods, err := r.getWorkerJobPods(ctx, *batchJob) if err != nil { return errors.Wrap(err, "failed to retrieve worker pods") } worker := statusInfo.WorkerJob if worker != nil { batchJob.Status.EndTime = worker.Status.CompletionTime // assign right away, because it's a pointer if batchJob.Status.EndTime == nil { completedTimestampStr, completedTimestampExists := batchJob.Annotations[_completedTimestampAnnotation] if completedTimestampExists { ts, err := time.Parse(time.RFC3339, completedTimestampStr) if err != nil { return errors.Wrap(err, "failed to parse completed timestamp string") } completedTime := v1.NewTime(ts) batchJob.Status.EndTime = &completedTime } } if worker.Status.Failed == batchJob.Spec.Workers { batchJobStatus := status.JobWorkerError for _, condition := range worker.Status.Conditions { if condition.Reason == _deadlineExceededReason { batchJobStatus = status.JobTimedOut break } } for i := range workerJobPods { if k8s.WasPodOOMKilled(&workerJobPods[i]) { batchJobStatus = status.JobWorkerOOM break } } batchJob.Status.Status = batchJobStatus } else if worker.Status.Succeeded == batchJob.Spec.Workers { batchJob.Status.Status = status.JobSucceeded jobMetrics, err := r.Config.GetMetrics(r, *batchJob) if err != nil { return err } if jobMetrics.Failed > 0 { batchJob.Status.Status = status.JobCompletedWithFailures } } else if worker.Status.Active > 0 { batchJob.Status.Status = status.JobRunning } workerCounts := getReplicaCounts(workerJobPods) batchJob.Status.WorkerCounts = &workerCounts } if err := r.Status().Update(ctx, batchJob); err != nil { return err } return nil } func (r *BatchJobReconciler) deleteSQSQueue(batchJob batch.BatchJob) error { queueURL := r.getQueueURL(batchJob) input := sqs.DeleteQueueInput{QueueUrl: aws.String(queueURL)} if _, err := r.AWS.SQS().DeleteQueue(&input); err != nil { if awsErr, ok := err.(awserr.Error); ok { if awsErr.Code() == sqs.ErrCodeQueueDoesNotExist { return nil } } return err } return nil } func (r *BatchJobReconciler) uploadJobSpec(batchJob batch.BatchJob, api spec.API, queueURL string) (*spec.BatchJob, error) { jobSpec, err := r.ConvertControllerBatchToJobSpec(batchJob, api, queueURL) if err != nil { return nil, err } if err = r.AWS.UploadJSONToS3(&jobSpec, r.ClusterConfig.Bucket, r.jobSpecKey(batchJob)); err != nil { return nil, err } return &jobSpec, nil } func (r *BatchJobReconciler) ConvertControllerBatchToJobSpec(batchJob batch.BatchJob, api spec.API, queueURL string) (spec.BatchJob, error) { var deadLetterQueue *spec.SQSDeadLetterQueue if batchJob.Spec.DeadLetterQueue != nil { deadLetterQueue = &spec.SQSDeadLetterQueue{ ARN: batchJob.Spec.DeadLetterQueue.ARN, MaxReceiveCount: int(batchJob.Spec.DeadLetterQueue.MaxReceiveCount), } } var config map[string]interface{} if batchJob.Spec.Config != nil { err := yaml.Unmarshal([]byte(*batchJob.Spec.Config), &config) if err != nil { return spec.BatchJob{}, errors.Wrap(err, "failed to unmarshal job spec config") } } var timeout *int if batchJob.Spec.Timeout != nil { timeout = pointer.Int(int(batchJob.Spec.Timeout.Seconds())) } totalBatchCount, err := r.Config.GetTotalBatchCount(r, batchJob) if err != nil { return spec.BatchJob{}, errors.Wrap(err, "failed to get total batch count") } return spec.BatchJob{ JobKey: spec.JobKey{ ID: batchJob.Status.ID, APIName: batchJob.Spec.APIName, Kind: userconfig.BatchAPIKind, }, RuntimeBatchJobConfig: spec.RuntimeBatchJobConfig{ Workers: int(batchJob.Spec.Workers), SQSDeadLetterQueue: deadLetterQueue, Config: config, Timeout: timeout, }, APIID: api.ID, SQSUrl: queueURL, StartTime: batchJob.CreationTimestamp.Time, TotalBatchCount: totalBatchCount, }, nil } func (r *BatchJobReconciler) jobSpecKey(batchJob batch.BatchJob) string { // e.g. /jobs/////spec.json return filepath.Join( r.ClusterConfig.ClusterUID, "jobs", userconfig.BatchAPIKind.String(), consts.CortexVersion, batchJob.Spec.APIName, batchJob.Name, "spec.json", ) } func (r *BatchJobReconciler) updateCompletedTimestamp(ctx context.Context, batchJob *batch.BatchJob) error { ts := time.Now().Format(time.RFC3339) if batchJob.Annotations != nil { batchJob.Annotations[_completedTimestampAnnotation] = ts } else { batchJob.Annotations = map[string]string{ _completedTimestampAnnotation: ts, } } if err := r.Update(ctx, batchJob); err != nil { return err } return nil } func (r *BatchJobReconciler) persistJobToS3(batchJob batch.BatchJob) error { return parallel.RunFirstErr( func() error { return r.Config.SaveJobMetrics(r, batchJob) }, func() error { return r.Config.SaveJobStatus(r, batchJob) }, ) } func getTotalBatchCount(r *BatchJobReconciler, batchJob batch.BatchJob) (int, error) { key := spec.JobBatchCountKey(r.ClusterConfig.ClusterUID, userconfig.BatchAPIKind, batchJob.Spec.APIName, batchJob.Name) cachedTotalBatchCount, found := totalBatchCountCache.Get(key) var totalBatchCount int if !found { totalBatchCountBytes, err := r.AWS.ReadBytesFromS3(r.ClusterConfig.Bucket, key) if err != nil { return 0, err } totalBatchCount, err = strconv.Atoi(string(totalBatchCountBytes)) if err != nil { return 0, err } } else { totalBatchCount = cachedTotalBatchCount.(int) } totalBatchCountCache.Set(key, totalBatchCount, _cacheDuration) return totalBatchCount, nil } func getMetrics(r *BatchJobReconciler, batchJob batch.BatchJob) (metrics.BatchMetrics, error) { endTime := time.Now() if batchJob.Status.EndTime != nil { endTime = batchJob.Status.EndTime.Time } jobMetrics, err := batch.GetMetrics(r.Prometheus, spec.JobKey{ ID: batchJob.Name, APIName: batchJob.Spec.APIName, Kind: userconfig.BatchAPIKind, }, endTime) if err != nil { return metrics.BatchMetrics{}, err } return *jobMetrics, nil } func saveJobMetrics(r *BatchJobReconciler, batchJob batch.BatchJob) error { jobMetrics, err := r.Config.GetMetrics(r, batchJob) if err != nil { return err } key := spec.JobMetricsKey(r.ClusterConfig.ClusterUID, userconfig.BatchAPIKind, batchJob.Spec.APIName, batchJob.Name) if err = r.AWS.UploadJSONToS3(&jobMetrics, r.ClusterConfig.Bucket, key); err != nil { return err } return nil } func saveJobStatus(r *BatchJobReconciler, batchJob batch.BatchJob) error { return parallel.RunFirstErr( func() error { stoppedStatusKey := filepath.Join( spec.JobAPIPrefix(r.ClusterConfig.ClusterUID, userconfig.BatchAPIKind, batchJob.Spec.APIName), batchJob.Name, status.JobStopped.String(), ) return r.AWS.UploadStringToS3("", r.ClusterConfig.Bucket, stoppedStatusKey) }, func() error { jobStatus := batchJob.Status.Status.String() key := filepath.Join( spec.JobAPIPrefix(r.ClusterConfig.ClusterUID, userconfig.BatchAPIKind, batchJob.Spec.APIName), batchJob.Name, jobStatus, ) return r.AWS.UploadStringToS3("", r.ClusterConfig.Bucket, key) }, ) } func getReplicaCounts(workerJobPods []kcore.Pod) status.WorkerCounts { workerCounts := status.WorkerCounts{} for i := range workerJobPods { switch k8s.GetPodStatus(&workerJobPods[i]) { case k8s.PodStatusPending: workerCounts.Pending++ case k8s.PodStatusStalled: workerCounts.Stalled++ case k8s.PodStatusCreating: workerCounts.Creating++ case k8s.PodStatusNotReady: workerCounts.NotReady++ case k8s.PodStatusErrImagePull: workerCounts.ErrImagePull++ case k8s.PodStatusTerminating: workerCounts.Terminating++ case k8s.PodStatusFailed: workerCounts.Failed++ case k8s.PodStatusKilled: workerCounts.Killed++ case k8s.PodStatusKilledOOM: workerCounts.KilledOOM++ case k8s.PodStatusSucceeded: workerCounts.Succeeded++ case k8s.PodStatusUnknown: workerCounts.Unknown++ } } return workerCounts } ================================================ FILE: pkg/crds/controllers/batch/batchjob_controller_test.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package batchcontrollers_test import ( "context" "strings" "time" batch "github.com/cortexlabs/cortex/pkg/crds/apis/batch/v1alpha1" "github.com/cortexlabs/cortex/pkg/lib/pointer" "github.com/cortexlabs/cortex/pkg/lib/random" "github.com/cortexlabs/cortex/pkg/types/spec" "github.com/cortexlabs/cortex/pkg/types/status" "github.com/cortexlabs/cortex/pkg/types/userconfig" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" kbatch "k8s.io/api/batch/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" kmeta "k8s.io/apimachinery/pkg/apis/meta/v1" ktypes "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" ) func uploadTestAPISpec(apiName string, apiID string) error { apiSpec := spec.API{ API: &userconfig.API{ Resource: userconfig.Resource{ Name: apiName, Kind: userconfig.BatchAPIKind, }, Pod: &userconfig.Pod{ Port: pointer.Int32(8080), Containers: []*userconfig.Container{ { Name: "api", Image: "quay.io/cortexlabs/batch-container-test:master", Command: []string{"/bin/run"}, Compute: &userconfig.Compute{}, }, }, }, }, ID: apiID, SpecID: random.String(5), PodID: random.String(5), DeploymentID: random.String(5), } apiSpecKey := spec.Key(apiName, apiID, clusterConfig.ClusterUID) if err := awsClient.UploadJSONToS3(apiSpec, clusterConfig.Bucket, apiSpecKey); err != nil { return err } return nil } func deleteTestAPISpec(apiName string, apiID string) error { apiSpecKey := spec.Key(apiName, apiID, clusterConfig.ClusterUID) if err := awsClient.DeleteS3File(clusterConfig.Bucket, apiSpecKey); err != nil { return err } return nil } var _ = Describe("BatchJob controller", func() { // Define utility constants for object names and testing timeouts/durations and intervals. const ( APIName = "test-api" BatchJobNamespace = "default" timeout = time.Second * 10 interval = time.Millisecond * 250 ) var ( randomJobID string randomAPIID string ) Context("Reconciliation", func() { BeforeEach(func() { // ensures the tests can be ran in rapid succession by avoiding the time limits of SQS queue creation randomJobID = strings.ToLower(random.String(5)) randomAPIID = random.Digits(5) Expect(uploadTestAPISpec(APIName, randomAPIID)).To(Succeed()) }) AfterEach(func(done Done) { Expect(deleteTestAPISpec(APIName, randomAPIID)).To(Succeed()) Expect(k8sClient.Delete( context.Background(), &batch.BatchJob{ ObjectMeta: kmeta.ObjectMeta{Name: randomJobID, Namespace: BatchJobNamespace}, }, )).To(Succeed()) close(done) }) It("Should reach a batch job completed status", func() { When("Every child resource is created or finishes successfully", func() { By("Creating a new BatchJob") ctx := context.Background() batchJob := &batch.BatchJob{ ObjectMeta: kmeta.ObjectMeta{ Name: randomJobID, Namespace: BatchJobNamespace, }, Spec: batch.BatchJobSpec{ APIName: APIName, APIID: randomAPIID, Workers: 1, }, } Expect(k8sClient.Create(ctx, batchJob)).To(Succeed()) batchJobLookupKey := ktypes.NamespacedName{Name: batchJob.Name, Namespace: batchJob.Namespace} createdBatchJob := &batch.BatchJob{} // Check that the resource was created correctly (i.e. if the spec matches) Eventually(func() error { return k8sClient.Get(ctx, batchJobLookupKey, createdBatchJob) }, timeout, interval).Should(Succeed()) Expect(createdBatchJob.Spec).Should(Equal(batchJob.Spec)) By("Creating an SQS queue successfully") Eventually(func() bool { err := k8sClient.Get(ctx, batchJobLookupKey, createdBatchJob) if err != nil { return false } return createdBatchJob.Status.QueueURL != "" }, timeout, interval).Should(BeTrue()) By("Reaching a completed enqueuer status") enqueuerJobLookupKey := ktypes.NamespacedName{ Name: batchJob.Spec.APIName + "-" + batchJob.Name + "-enqueuer", Namespace: batchJob.Namespace, } createdEnqueuerJob := &kbatch.Job{} // wait for the enqueuer job to be created Eventually(func() error { return k8sClient.Get(ctx, enqueuerJobLookupKey, createdEnqueuerJob) }, timeout, interval).Should(Succeed()) // Mock the enqueuer status to match the success condition createdEnqueuerJob.Status.Succeeded = 1 Expect(k8sClient.Status().Update(ctx, createdEnqueuerJob)).To(Succeed()) Eventually(func() bool { err := k8sClient.Get(ctx, batchJobLookupKey, createdBatchJob) if err != nil { return false } return createdBatchJob.Status.Status == status.JobEnqueuing }, timeout, interval) By("Reaching a successful worker job status") workerJobLookupKey := ktypes.NamespacedName{ Name: batchJob.Spec.APIName + "-" + batchJob.Name, Namespace: BatchJobNamespace, } createdWorkerJob := &kbatch.Job{} // Wait for worker job to be created Eventually(func() error { return k8sClient.Get(ctx, workerJobLookupKey, createdWorkerJob) }, timeout, interval).Should(Succeed()) // Mock the worker job status to match the success condition createdWorkerJob.Status.Succeeded = batchJob.Spec.Workers Expect(k8sClient.Status().Update(ctx, createdWorkerJob)).To(Succeed()) Eventually(func() bool { err := k8sClient.Get(ctx, batchJobLookupKey, createdBatchJob) if err != nil { return false } return createdBatchJob.Status.Status == status.JobSucceeded }).Should(BeTrue()) }) }) }) Context("Reconcialiation TTL", func() { BeforeEach(func() { // ensures the tests can be ran in rapid succession by avoiding the time limits of SQS queue creation randomJobID = strings.ToLower(random.String(5)) randomAPIID = random.Digits(5) Expect(uploadTestAPISpec(APIName, randomAPIID)).To(Succeed()) }) AfterEach(func(done Done) { Expect(deleteTestAPISpec(APIName, randomAPIID)).To(Succeed()) close(done) }) It("Should self clean-up when a completed status is reached and the TTL is exceeded", func() { By("Creating a new BatchJob") ttl := kmeta.Duration{Duration: time.Second * 10} ctx := context.Background() batchJob := &batch.BatchJob{ ObjectMeta: kmeta.ObjectMeta{ Name: randomJobID, Namespace: BatchJobNamespace, }, Spec: batch.BatchJobSpec{ APIName: APIName, APIID: randomAPIID, Workers: 1, TTL: &ttl, }, } Expect(k8sClient.Create(ctx, batchJob)).To(Succeed()) By("Reaching a completed enqueuer status") enqueuerJobLookupKey := ktypes.NamespacedName{ Name: batchJob.Spec.APIName + "-" + batchJob.Name + "-enqueuer", Namespace: batchJob.Namespace, } createdEnqueuerJob := &kbatch.Job{} // wait for the enqueuer job to be created Eventually(func() error { return k8sClient.Get(ctx, enqueuerJobLookupKey, createdEnqueuerJob) }, timeout, interval).Should(Succeed()) // Mock the enqueuer status to match the success condition createdEnqueuerJob.Status.Succeeded = 1 Expect(k8sClient.Status().Update(ctx, createdEnqueuerJob)).To(Succeed()) By("Reaching a successful worker job status") workerJobLookupKey := ktypes.NamespacedName{ Name: batchJob.Spec.APIName + "-" + batchJob.Name, Namespace: BatchJobNamespace, } createdWorkerJob := &kbatch.Job{} // Wait for worker job to be created Eventually(func() error { return k8sClient.Get(ctx, workerJobLookupKey, createdWorkerJob) }, timeout, interval).Should(Succeed()) // Mock the worker job status to match the success condition completionTime := time.Now() createdWorkerJob.Status.Succeeded = batchJob.Spec.Workers createdWorkerJob.Status.CompletionTime = &kmeta.Time{Time: completionTime} Expect(k8sClient.Status().Update(ctx, createdWorkerJob)).To(Succeed()) By("Waiting for the TTL to kick in") var deletionTime time.Time Eventually(func() bool { if err := k8sClient.Get(ctx, client.ObjectKey{ Name: randomJobID, Namespace: BatchJobNamespace, }, &batch.BatchJob{}); err != nil { if kerrors.IsNotFound(err) { deletionTime = time.Now() return true } } return false }, ttl.Duration.Seconds()*2).Should(BeTrue()) duration := deletionTime.Sub(completionTime) Expect(duration > ttl.Duration).Should(BeTrue()) }) It("Should self clean-up when enqueing fails and the TTL is exceeded", func() { By("Creating a new BatchJob") ttl := kmeta.Duration{Duration: time.Second * 10} ctx := context.Background() batchJob := &batch.BatchJob{ ObjectMeta: kmeta.ObjectMeta{ Name: randomJobID, Namespace: BatchJobNamespace, }, Spec: batch.BatchJobSpec{ APIName: APIName, APIID: randomAPIID, Workers: 1, TTL: &ttl, }, } Expect(k8sClient.Create(ctx, batchJob)).To(Succeed()) By("Reaching a failed enqueuer status") enqueuerJobLookupKey := ktypes.NamespacedName{ Name: batchJob.Spec.APIName + "-" + batchJob.Name + "-enqueuer", Namespace: batchJob.Namespace, } createdEnqueuerJob := &kbatch.Job{} // wait for the enqueuer job to be created Eventually(func() error { return k8sClient.Get(ctx, enqueuerJobLookupKey, createdEnqueuerJob) }, timeout, interval).Should(Succeed()) // Mock the enqueuer status to match the success condition createdEnqueuerJob.Status.Failed = 1 Expect(k8sClient.Status().Update(ctx, createdEnqueuerJob)).To(Succeed()) By("Waiting for the TTL to kick in") Eventually(func() bool { err := k8sClient.Get(ctx, client.ObjectKey{ Name: randomJobID, Namespace: BatchJobNamespace, }, &batch.BatchJob{}) return kerrors.IsNotFound(err) }, ttl.Duration.Seconds()*2, interval).Should(BeTrue()) }) }) }) ================================================ FILE: pkg/crds/controllers/batch/suite_test.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package batchcontrollers_test import ( // "os" "path/filepath" // "testing" // "github.com/cortexlabs/cortex/pkg/config" // "github.com/cortexlabs/cortex/pkg/consts" // batch "github.com/cortexlabs/cortex/pkg/crds/apis/batch/v1alpha1" // batchcontrollers "github.com/cortexlabs/cortex/pkg/crds/controllers/batch" awslib "github.com/cortexlabs/cortex/pkg/lib/aws" // "github.com/cortexlabs/cortex/pkg/lib/hash" "github.com/cortexlabs/cortex/pkg/types/clusterconfig" // "github.com/cortexlabs/cortex/pkg/types/metrics" // . "github.com/onsi/ginkgo" // . "github.com/onsi/gomega" // "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" // ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/envtest" // "sigs.k8s.io/controller-runtime/pkg/envtest/printer" // logf "sigs.k8s.io/controller-runtime/pkg/log" // "sigs.k8s.io/controller-runtime/pkg/log/zap" // +kubebuilder:scaffold:imports ) // These tests use Ginkgo (BDD-style Go testing framework). Refer to // http://onsi.github.io/ginkgo/ to learn more about Ginkgo. var projectRoot = filepath.Join("..", "..", "..", "..") var devClusterConfigPath = filepath.Join(projectRoot, "dev", "config", "cluster.yaml") var cfg *rest.Config var k8sClient client.Client var awsClient *awslib.Client var clusterConfig *clusterconfig.Config var testEnv *envtest.Environment // func TestAPIs(t *testing.T) { // RegisterFailHandler(Fail) // RunSpecsWithDefaultAndCustomReporters(t, // "Controller Suite", // []Reporter{printer.NewlineReporter{}}) // } // var _ = BeforeSuite(func(done Done) { // logf.SetLogger(zap.New(zap.UseDevMode(true), zap.WriteTo(GinkgoWriter))) // crdDirectoryPath := filepath.Join("..", "..", "config", "crd", "bases") // Expect(crdDirectoryPath).To(BeADirectory()) // By("bootstrapping test environment") // testEnv = &envtest.Environment{ // CRDDirectoryPaths: []string{crdDirectoryPath}, // } // var err error // cfg, err = testEnv.Start() // Expect(err).ToNot(HaveOccurred()) // Expect(cfg).ToNot(BeNil()) // err = batch.AddToScheme(scheme.Scheme) // Expect(err).NotTo(HaveOccurred()) // // +kubebuilder:scaffold:scheme // k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) // Expect(err).ToNot(HaveOccurred()) // Expect(k8sClient).ToNot(BeNil()) // k8sManager, err := ctrl.NewManager(cfg, ctrl.Options{ // Scheme: scheme.Scheme, // }) // Expect(err).ToNot(HaveOccurred()) // clusterConfigPath := os.Getenv("CORTEX_TEST_CLUSTER_CONFIG") // if clusterConfigPath == "" { // clusterConfigPath = devClusterConfigPath // } // clusterConfig, err = clusterconfig.NewForFile(clusterConfigPath) // Expect(err).ToNot(HaveOccurred(), // "error during cluster config creation (custom cluster "+ // "config paths can be set with the CORTEX_TEST_CLUSTER_CONFIG env variable)", // ) // awsClient, err = awslib.NewForRegion(clusterConfig.Region) // Expect(err).ToNot(HaveOccurred()) // accountID, hashedAccountID, err := awsClient.CheckCredentials() // Expect(err).ToNot(HaveOccurred()) // clusterConfig.AccountID = accountID // clusterConfig.Bucket = clusterconfig.BucketName(accountID, clusterConfig.ClusterName, clusterConfig.Region) // operatorMetadata := &clusterconfig.OperatorMetadata{ // APIVersion: consts.CortexVersion, // OperatorID: hashedAccountID, // ClusterID: hash.String(clusterConfig.ClusterName + clusterConfig.Region + hashedAccountID), // IsOperatorInCluster: false, // } // // initialize some of the global values for the k8s helpers // config.InitConfigs(clusterConfig, operatorMetadata) // // mock certain methods of the reconciler // reconcilerConfig := batchcontrollers.BatchJobReconcilerConfig{ // GetTotalBatchCount: func(r *batchcontrollers.BatchJobReconciler, batchJob batch.BatchJob) (int, error) { // return 1, nil // }, // GetMetrics: func(r *batchcontrollers.BatchJobReconciler, batchJob batch.BatchJob) (metrics.BatchMetrics, error) { // return metrics.BatchMetrics{Succeeded: 1}, nil // }, // SaveJobMetrics: func(r *batchcontrollers.BatchJobReconciler, batchJob batch.BatchJob) error { // return nil // }, // SaveJobStatus: func(r *batchcontrollers.BatchJobReconciler, batchJob batch.BatchJob) error { // return nil // }, // } // err = (&batchcontrollers.BatchJobReconciler{ // Client: k8sManager.GetClient(), // Config: reconcilerConfig, // Log: ctrl.Log.WithName("controllers").WithName("BatchJob"), // ClusterConfig: clusterConfig, // AWS: awsClient, // Scheme: k8sManager.GetScheme(), // }).SetupWithManager(k8sManager) // Expect(err).ToNot(HaveOccurred()) // go func() { // defer GinkgoRecover() // err = k8sManager.Start(ctrl.SetupSignalHandler()) // Expect(err).ToNot(HaveOccurred()) // }() // k8sClient = k8sManager.GetClient() // Expect(k8sClient).ToNot(BeNil()) // close(done) // }, 60) // var _ = AfterSuite(func() { // By("tearing down the test environment") // err := testEnv.Stop() // Expect(err).ToNot(HaveOccurred()) // }) ================================================ FILE: pkg/crds/controllers/errors.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package controllers import ( "strings" kerrors "k8s.io/apimachinery/pkg/api/errors" ) const ( _optimisticLockErrorMessage = "the object has been modified; please apply your changes to the latest version and try again" ) // IsOptimisticLockError checks if an error is an optimistic lock error func IsOptimisticLockError(err error) bool { return kerrors.IsConflict(err) && strings.Contains(err.Error(), _optimisticLockErrorMessage) } ================================================ FILE: pkg/crds/hack/boilerplate.go.txt ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ ================================================ FILE: pkg/crds/hack/run_manager.sh ================================================ #!/usr/bin/env bash # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # This is a subset of lint.sh, and is only meant to be run on master CLUSTER_CONFIG=$1 port_forward_cmd="kubectl port-forward -n prometheus prometheus-prometheus-0 9090" kill $(pgrep -f "${port_forward_cmd}") >/dev/null 2>&1 || true echo "Port-forwarding Prometheus to localhost:9090" eval "${port_forward_cmd}" >/dev/null 2>&1 & CORTEX_DISABLE_JSON_LOGGING="true" \ CORTEX_PROMETHEUS_URL="http://localhost:9090" \ go run ./main.go -config "${CLUSTER_CONFIG}" ================================================ FILE: pkg/crds/main.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package main import ( "flag" "fmt" "os" "strings" "github.com/cortexlabs/cortex/pkg/config" "github.com/cortexlabs/cortex/pkg/consts" awslib "github.com/cortexlabs/cortex/pkg/lib/aws" "github.com/cortexlabs/cortex/pkg/lib/hash" "github.com/cortexlabs/cortex/pkg/types/clusterconfig" promapi "github.com/prometheus/client_golang/api" promv1 "github.com/prometheus/client_golang/api/prometheus/v1" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. _ "k8s.io/client-go/plugin/pkg/client/auth" "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" batch "github.com/cortexlabs/cortex/pkg/crds/apis/batch/v1alpha1" batchcontrollers "github.com/cortexlabs/cortex/pkg/crds/controllers/batch" //+kubebuilder:scaffold:imports ) var ( scheme = runtime.NewScheme() setupLog = ctrl.Log.WithName("setup") ) func init() { utilruntime.Must(clientgoscheme.AddToScheme(scheme)) utilruntime.Must(batch.AddToScheme(scheme)) //+kubebuilder:scaffold:scheme } func main() { var ( metricsAddr string enableLeaderElection bool probeAddr string clusterConfigPath string prometheusURL string // defaults to http://prometheus.:9090 inCluster = strings.ToLower(os.Getenv("CORTEX_OPERATOR_IN_CLUSTER")) == "true" useDevMode = strings.ToLower(os.Getenv("CORTEX_DISABLE_JSON_LOGGING")) == "true" ) flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") flag.BoolVar(&enableLeaderElection, "leader-elect", false, "Enable leader election for controller manager. "+ "Enabling this will ensure there is only one active controller manager.") flag.StringVar(&clusterConfigPath, "config", os.Getenv("CORTEX_CLUSTER_CONFIG_PATH"), "The path to the cluster config yaml file. "+ "Can be set with the CORTEX_CLUSTER_CONFIG_PATH env variable. [Required]", ) flag.StringVar(&prometheusURL, "prometheus-url", os.Getenv("CORTEX_PROMETHEUS_URL"), "Prometheus server URL", ) opts := zap.Options{ Development: useDevMode, } opts.BindFlags(flag.CommandLine) flag.Parse() ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) switch { case clusterConfigPath == "": setupLog.Error(nil, "-config is a required flag") os.Exit(1) } clusterConfig, err := clusterconfig.NewForFile(clusterConfigPath) if err != nil { setupLog.Error(err, "failed to initialize cluster config") os.Exit(1) } if prometheusURL == "" { prometheusURL = fmt.Sprintf("http://prometheus.%s:9090", consts.PrometheusNamespace) } mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ Scheme: scheme, MetricsBindAddress: metricsAddr, Port: 9443, HealthProbeBindAddress: probeAddr, LeaderElection: enableLeaderElection, LeaderElectionID: "7cc92962.cortex.dev", }) if err != nil { setupLog.Error(err, "unable to start manager") os.Exit(1) } awsClient, err := awslib.NewForRegion(clusterConfig.Region) if err != nil { setupLog.Error(err, "failed to create AWS client") os.Exit(1) } accountID, hashedAccountID, err := awsClient.CheckCredentials() if err != nil { setupLog.Error(err, "failed to check AWS credentials") os.Exit(1) } clusterConfig.AccountID = accountID operatorMetadata := &clusterconfig.OperatorMetadata{ APIVersion: consts.CortexVersion, OperatorID: hashedAccountID, ClusterID: hash.String(clusterConfig.ClusterName + clusterConfig.Region + hashedAccountID), IsOperatorInCluster: inCluster, } promClient, err := promapi.NewClient(promapi.Config{Address: prometheusURL}) if err != nil { setupLog.Error(err, "failed to initialize prometheus client") os.Exit(1) } // initialize some of the global values for the k8s helpers config.InitConfigs(clusterConfig, operatorMetadata) if err = (&batchcontrollers.BatchJobReconciler{ Client: mgr.GetClient(), Config: batchcontrollers.BatchJobReconcilerConfig{}.ApplyDefaults(), Log: ctrl.Log.WithName("controllers").WithName("BatchJob"), ClusterConfig: clusterConfig, AWS: awsClient, Prometheus: promv1.NewAPI(promClient), Scheme: mgr.GetScheme(), }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "BatchJob") os.Exit(1) } //+kubebuilder:scaffold:builder if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { setupLog.Error(err, "unable to set up health check") os.Exit(1) } if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { setupLog.Error(err, "unable to set up ready check") os.Exit(1) } setupLog.Info("starting manager") if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { setupLog.Error(err, "problem running manager") os.Exit(1) } } ================================================ FILE: pkg/dequeuer/async_handler.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package dequeuer import ( "encoding/json" "fmt" "io" "net/http" "strings" "time" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/sqs" awslib "github.com/cortexlabs/cortex/pkg/lib/aws" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/telemetry" "github.com/cortexlabs/cortex/pkg/types/async" "go.uber.org/zap" ) const ( // CortexRequestIDHeader is the header containing the workload request id for the user container CortexRequestIDHeader = "X-Cortex-Request-ID" ) type AsyncMessageHandler struct { config AsyncMessageHandlerConfig aws *awslib.Client log *zap.SugaredLogger storagePath string httpClient *http.Client eventHandler RequestEventHandler } type AsyncMessageHandlerConfig struct { ClusterUID string Bucket string APIName string TargetURL string } func NewAsyncMessageHandler(config AsyncMessageHandlerConfig, awsClient *awslib.Client, eventHandler RequestEventHandler, logger *zap.SugaredLogger) *AsyncMessageHandler { return &AsyncMessageHandler{ config: config, aws: awsClient, log: logger, storagePath: async.StoragePath(config.ClusterUID, config.APIName), httpClient: &http.Client{}, eventHandler: eventHandler, } } func (h *AsyncMessageHandler) Handle(message *sqs.Message) error { if message == nil { return errors.ErrorUnexpected("got unexpected nil SQS message") } if message.Body == nil || *message.Body == "" { return errors.ErrorUnexpected("got unexpected sqs message with empty or nil body") } requestID := *message.Body err := h.handleMessage(requestID) if err != nil { return err } return nil } func (h *AsyncMessageHandler) handleMessage(requestID string) error { h.log.Infow("processing workload", "id", requestID) err := h.updateStatus(requestID, async.StatusInProgress) if err != nil { return errors.Wrap(err, fmt.Sprintf("failed to update status to %s", async.StatusInProgress)) } payload, err := h.getPayload(requestID) if err != nil { updateStatusErr := h.updateStatus(requestID, async.StatusFailed) if updateStatusErr != nil { h.log.Errorw("failed to update status after failure to get payload", "id", requestID, "error", updateStatusErr) } return errors.Wrap(err, "failed to get payload") } defer func() { h.deletePayload(requestID) _ = payload.Close() }() headers, err := h.getHeaders(requestID) if err != nil { updateStatusErr := h.updateStatus(requestID, async.StatusFailed) if updateStatusErr != nil { h.log.Errorw("failed to update status after failure to get headers", "id", requestID, "error", updateStatusErr) } return errors.Wrap(err, "failed to get payload") } result, err := h.submitRequest(payload, headers, requestID) if err != nil { h.log.Errorw("failed to submit request to user container", "id", requestID, "error", err) updateStatusErr := h.updateStatus(requestID, async.StatusFailed) if updateStatusErr != nil { return errors.Wrap(updateStatusErr, fmt.Sprintf("failed to update status to %s", async.StatusFailed)) } return nil } if err = h.uploadResult(requestID, result); err != nil { updateStatusErr := h.updateStatus(requestID, async.StatusFailed) if updateStatusErr != nil { h.log.Errorw("failed to update status after failure to upload result", "id", requestID, "error", updateStatusErr) } return errors.Wrap(err, "failed to upload result to storage") } if err = h.updateStatus(requestID, async.StatusCompleted); err != nil { return errors.Wrap(err, fmt.Sprintf("failed to update status to %s", async.StatusCompleted)) } h.log.Infow("workload processing complete", "id", requestID) return nil } func (h *AsyncMessageHandler) updateStatus(requestID string, status async.Status) error { key := async.StatusPath(h.storagePath, requestID, status) return h.aws.UploadStringToS3("", h.config.Bucket, key) } func (h *AsyncMessageHandler) getPayload(requestID string) (io.ReadCloser, error) { key := async.PayloadPath(h.storagePath, requestID) output, err := h.aws.S3().GetObject( &s3.GetObjectInput{ Key: aws.String(key), Bucket: aws.String(h.config.Bucket), }, ) if err != nil { return nil, errors.WithStack(err) } return output.Body, nil } func (h *AsyncMessageHandler) deletePayload(requestID string) { key := async.PayloadPath(h.storagePath, requestID) err := h.aws.DeleteS3File(h.config.Bucket, key) if err != nil { h.log.Errorw("failed to delete user payload", "error", err) telemetry.Error(errors.Wrap(err, "failed to delete user payload")) } } func (h *AsyncMessageHandler) submitRequest(payload io.Reader, headers http.Header, requestID string) (interface{}, error) { req, err := http.NewRequest(http.MethodPost, h.config.TargetURL, payload) if err != nil { return nil, errors.WithStack(err) } req.Header = headers req.Header.Set(CortexRequestIDHeader, requestID) startTime := time.Now() response, err := h.httpClient.Do(req) if err != nil { return nil, ErrorUserContainerNotReachable(err) } defer func() { _ = response.Body.Close() }() h.eventHandler.HandleEvent( RequestEvent{ StatusCode: response.StatusCode, Duration: time.Since(startTime), }, ) if response.StatusCode != http.StatusOK { return nil, ErrorUserContainerResponseStatusCode(response.StatusCode) } if !strings.HasPrefix(response.Header.Get("Content-Type"), "application/json") { return nil, ErrorUserContainerResponseMissingJSONHeader() } var result interface{} if err = json.NewDecoder(response.Body).Decode(&result); err != nil { return nil, ErrorUserContainerResponseNotJSONDecodable() } return result, nil } func (h *AsyncMessageHandler) uploadResult(requestID string, result interface{}) error { key := async.ResultPath(h.storagePath, requestID) return h.aws.UploadJSONToS3(result, h.config.Bucket, key) } func (h *AsyncMessageHandler) getHeaders(requestID string) (http.Header, error) { key := async.HeadersPath(h.storagePath, requestID) var headers http.Header if err := h.aws.ReadJSONFromS3(&headers, h.config.Bucket, key); err != nil { return nil, err } return headers, nil } ================================================ FILE: pkg/dequeuer/async_handler_test.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package dequeuer import ( "fmt" "net/http" "net/http/httptest" "testing" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/sqs" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/random" "github.com/cortexlabs/cortex/pkg/types/async" "github.com/stretchr/testify/require" ) const ( _testBucket = "test" ) func TestAsyncMessageHandler_Handle(t *testing.T) { t.Parallel() log := newLogger(t) defer func() { _ = log.Sync() }() awsClient := testAWSClient(t) requestID := random.String(8) server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { require.Equal(t, requestID, r.Header.Get(CortexRequestIDHeader)) w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) _, _ = w.Write([]byte("{}")) })) var requestEventsCount int eventHandler := NewRequestEventHandlerFunc(func(event RequestEvent) { requestEventsCount++ }) asyncHandler := NewAsyncMessageHandler(AsyncMessageHandlerConfig{ ClusterUID: "cortex-test", Bucket: _testBucket, APIName: "async-test", TargetURL: server.URL, }, awsClient, eventHandler, log) _, err := awsClient.S3().CreateBucket(&s3.CreateBucketInput{ Bucket: aws.String(_testBucket), }) require.NoError(t, err) err = awsClient.UploadStringToS3("{}", asyncHandler.config.Bucket, async.PayloadPath(asyncHandler.storagePath, requestID)) require.NoError(t, err) err = awsClient.UploadStringToS3("{}", asyncHandler.config.Bucket, async.HeadersPath(asyncHandler.storagePath, requestID)) require.NoError(t, err) err = asyncHandler.Handle(&sqs.Message{ Body: aws.String(requestID), MessageId: aws.String(requestID), }) require.NoError(t, err) _, err = awsClient.ReadStringFromS3( _testBucket, fmt.Sprintf("%s/%s/status/%s", asyncHandler.storagePath, requestID, async.StatusCompleted), ) require.NoError(t, err) require.Equal(t, 1, requestEventsCount) } func TestAsyncMessageHandler_Handle_Errors(t *testing.T) { t.Parallel() cases := []struct { name string message *sqs.Message expectedError error }{ { name: "nil", message: nil, expectedError: errors.ErrorUnexpected("got unexpected nil SQS message"), }, { name: "nil body", message: &sqs.Message{}, expectedError: errors.ErrorUnexpected("got unexpected sqs message with empty or nil body"), }, { name: "empty body", message: &sqs.Message{Body: aws.String("")}, expectedError: errors.ErrorUnexpected("got unexpected sqs message with empty or nil body"), }, } log := newLogger(t) defer func() { _ = log.Sync() }() awsClient := testAWSClient(t) eventHandler := NewRequestEventHandlerFunc(func(event RequestEvent) {}) asyncHandler := NewAsyncMessageHandler(AsyncMessageHandlerConfig{ ClusterUID: "cortex-test", Bucket: _testBucket, APIName: "async-test", TargetURL: "http://fake.cortex.dev", }, awsClient, eventHandler, log) for _, tt := range cases { t.Run(tt.name, func(t *testing.T) { err := asyncHandler.Handle(tt.message) require.EqualError(t, err, tt.expectedError.Error()) }) } } ================================================ FILE: pkg/dequeuer/async_stats.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package dequeuer import ( "net/http" "strconv" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/client_golang/prometheus/promhttp" ) type AsyncStatsReporter struct { handler http.Handler latencies *prometheus.HistogramVec requestCount *prometheus.CounterVec } func NewAsyncPrometheusStatsReporter() *AsyncStatsReporter { latenciesHist := promauto.NewHistogramVec(prometheus.HistogramOpts{ Name: "cortex_async_latency", Help: "Histogram of the latencies for an AsyncAPI kind in seconds", }, []string{"status_code"}) requestCounter := promauto.NewCounterVec(prometheus.CounterOpts{ Name: "cortex_async_request_count", Help: "Request count for an AsyncAPI", }, []string{"status_code"}) handler := promhttp.Handler() return &AsyncStatsReporter{ handler: handler, latencies: latenciesHist, requestCount: requestCounter, } } func (r *AsyncStatsReporter) HandleEvent(event RequestEvent) { labels := map[string]string{ "status_code": strconv.Itoa(event.StatusCode), } r.latencies.With(labels).Observe(event.Duration.Seconds()) r.requestCount.With(labels).Add(1) } func (r *AsyncStatsReporter) ServeHTTP(w http.ResponseWriter, req *http.Request) { r.handler.ServeHTTP(w, req) } ================================================ FILE: pkg/dequeuer/async_stats_test.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package dequeuer import ( "net/http/httptest" "strings" "testing" "time" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/client_golang/prometheus/testutil" "github.com/stretchr/testify/require" ) func TestNewAsyncPrometheusStatsReporter(t *testing.T) { t.Parallel() statsReporter := NewAsyncPrometheusStatsReporter() statsReporter.HandleEvent( RequestEvent{ StatusCode: 200, Duration: 100 * time.Millisecond, }, ) w := httptest.NewRecorder() r := httptest.NewRequest("GET", "/metrics", nil) statsReporter.ServeHTTP(w, r) result := w.Body.String() require.Contains(t, result, "cortex_async_latency") require.Contains(t, result, "cortex_async_request_count") } func TestAsyncStatsReporter_HandleEvent(t *testing.T) { t.Parallel() reg := prometheus.NewRegistry() latenciesHist := promauto.With(reg).NewHistogramVec(prometheus.HistogramOpts{ Name: "cortex_async_latency", Help: "Histogram of the latencies for an AsyncAPI kind in seconds", }, []string{"status_code"}) requestCounter := promauto.With(reg).NewCounterVec(prometheus.CounterOpts{ Name: "cortex_async_request_count", Help: "Request count for an AsyncAPI", }, []string{"status_code"}) statsReporter := AsyncStatsReporter{ latencies: latenciesHist, requestCount: requestCounter, } statsReporter.HandleEvent( RequestEvent{ StatusCode: 200, Duration: 100 * time.Millisecond, }, ) expectedHist := ` # HELP cortex_async_latency Histogram of the latencies for an AsyncAPI kind in seconds # TYPE cortex_async_latency histogram cortex_async_latency_bucket{status_code="200",le="0.005"} 0 cortex_async_latency_bucket{status_code="200",le="0.01"} 0 cortex_async_latency_bucket{status_code="200",le="0.025"} 0 cortex_async_latency_bucket{status_code="200",le="0.05"} 0 cortex_async_latency_bucket{status_code="200",le="0.1"} 1 cortex_async_latency_bucket{status_code="200",le="0.25"} 1 cortex_async_latency_bucket{status_code="200",le="0.5"} 1 cortex_async_latency_bucket{status_code="200",le="1"} 1 cortex_async_latency_bucket{status_code="200",le="2.5"} 1 cortex_async_latency_bucket{status_code="200",le="5"} 1 cortex_async_latency_bucket{status_code="200",le="10"} 1 cortex_async_latency_bucket{status_code="200",le="+Inf"} 1 cortex_async_latency_sum{status_code="200"} 0.1 cortex_async_latency_count{status_code="200"} 1 ` require.Equal(t, float64(1), testutil.ToFloat64(statsReporter.requestCount)) require.NoError(t, testutil.CollectAndCompare(statsReporter.latencies, strings.NewReader(expectedHist))) } ================================================ FILE: pkg/dequeuer/batch_handler.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package dequeuer import ( "bytes" "net/http" "time" "github.com/DataDog/datadog-go/statsd" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/sqs" awslib "github.com/cortexlabs/cortex/pkg/lib/aws" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/urls" "github.com/xtgo/uuid" "go.uber.org/zap" ) const ( // CortexJobIDHeader is the header containing the job id for the user container CortexJobIDHeader = "X-Cortex-Job-ID" _jobCompleteMessageDelay = 10 * time.Second ) type BatchMessageHandler struct { config BatchMessageHandlerConfig jobCompleteMessageDelay time.Duration tags []string aws *awslib.Client metrics statsd.ClientInterface log *zap.SugaredLogger httpClient *http.Client } type BatchMessageHandlerConfig struct { APIName string JobID string QueueURL string Region string TargetURL string } func NewBatchMessageHandler(config BatchMessageHandlerConfig, awsClient *awslib.Client, statsdClient statsd.ClientInterface, log *zap.SugaredLogger) *BatchMessageHandler { tags := []string{ "api_name:" + config.APIName, "job_id:" + config.JobID, } return &BatchMessageHandler{ config: config, jobCompleteMessageDelay: _jobCompleteMessageDelay, tags: tags, aws: awsClient, metrics: statsdClient, log: log, httpClient: &http.Client{}, } } func (h *BatchMessageHandler) Handle(message *sqs.Message) error { if isOnJobCompleteMessage(message) { err := h.onJobComplete(message) if err != nil { return errors.Wrap(err, "failed to handle 'onJobComplete' message") } return nil } err := h.handleBatch(message) if err != nil { return err } return nil } func (h *BatchMessageHandler) recordSuccess() error { err := h.metrics.Incr("cortex_batch_succeeded", h.tags, 1.0) if err != nil { return errors.WithStack(err) } return nil } func (h *BatchMessageHandler) recordFailure() error { err := h.metrics.Incr("cortex_batch_failed", h.tags, 1.0) if err != nil { return errors.WithStack(err) } return nil } func (h *BatchMessageHandler) recordTimePerBatch(elapsedTime time.Duration) error { err := h.metrics.Histogram("cortex_time_per_batch", elapsedTime.Seconds(), h.tags, 1.0) if err != nil { return errors.WithStack(err) } return nil } func (h *BatchMessageHandler) submitRequest(messageBody string, isOnJobComplete bool) error { targetURL := h.config.TargetURL if isOnJobComplete { targetURL = urls.Join(targetURL, "/on-job-complete") } req, err := http.NewRequest(http.MethodPost, targetURL, bytes.NewBuffer([]byte(messageBody))) if err != nil { return errors.WithStack(err) } req.Header.Set("Content-Type", "application/json") req.Header.Set(CortexJobIDHeader, h.config.JobID) response, err := h.httpClient.Do(req) if err != nil { return ErrorUserContainerNotReachable(err) } defer func() { _ = response.Body.Close() }() if response.StatusCode == http.StatusNotFound && isOnJobComplete { return nil } if response.StatusCode != http.StatusOK { return ErrorUserContainerResponseStatusCode(response.StatusCode) } return nil } func (h *BatchMessageHandler) handleBatch(message *sqs.Message) error { h.log.Infow("processing batch", "id", *message.MessageId) startTime := time.Now() err := h.submitRequest(*message.Body, false) if err != nil { h.log.Errorw("failed to process batch", "id", *message.MessageId, "error", err) recordFailureErr := h.recordFailure() if recordFailureErr != nil { return errors.Wrap(recordFailureErr, "failed to record failure metric") } return nil } endTime := time.Since(startTime) err = h.recordSuccess() if err != nil { return errors.Wrap(err, "failed to record success metric") } err = h.recordTimePerBatch(endTime) if err != nil { return errors.Wrap(err, "failed to record time per batch") } return nil } func (h *BatchMessageHandler) onJobComplete(message *sqs.Message) error { shouldRunOnJobComplete := false h.log.Info("received job_complete message") for { queueAttributes, err := GetQueueAttributes(h.aws, h.config.QueueURL) if err != nil { return err } totalMessages := queueAttributes.TotalMessages() if totalMessages > 1 { time.Sleep(h.jobCompleteMessageDelay) h.log.Infow("found other messages in queue, requeuing job_complete message", "id", *message.MessageId) newMessageID := uuid.NewRandom().String() if _, err = h.aws.SQS().SendMessage( &sqs.SendMessageInput{ QueueUrl: &h.config.QueueURL, MessageBody: aws.String("job_complete"), MessageAttributes: map[string]*sqs.MessageAttributeValue{ "job_complete": { DataType: aws.String("String"), StringValue: aws.String("true"), }, "api_name": { DataType: aws.String("String"), StringValue: aws.String(h.config.APIName), }, "job_id": { DataType: aws.String("String"), StringValue: aws.String(h.config.JobID), }, }, MessageDeduplicationId: aws.String(newMessageID), MessageGroupId: aws.String(newMessageID), }, ); err != nil { return errors.WithStack(err) } return nil } if shouldRunOnJobComplete { h.log.Infow("processing job_complete message", "id", *message.MessageId) return h.submitRequest(*message.Body, true) } shouldRunOnJobComplete = true time.Sleep(h.jobCompleteMessageDelay) } } func isOnJobCompleteMessage(message *sqs.Message) bool { _, found := message.MessageAttributes["job_complete"] return found } ================================================ FILE: pkg/dequeuer/batch_handler_test.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package dequeuer import ( "net/http" "net/http/httptest" "testing" "github.com/DataDog/datadog-go/statsd" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/sqs" "github.com/stretchr/testify/require" ) func TestBatchMessageHandler_Handle(t *testing.T) { t.Parallel() awsClient := testAWSClient(t) var callCount int server := httptest.NewServer( http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { callCount++ w.WriteHeader(http.StatusOK) }), ) logger := newLogger(t) defer func() { _ = logger.Sync() }() batchHandler := NewBatchMessageHandler(BatchMessageHandlerConfig{ APIName: "test", JobID: "12345", Region: _localStackDefaultRegion, TargetURL: server.URL, }, awsClient, &statsd.NoOpClient{}, logger) err := batchHandler.Handle(&sqs.Message{ Body: aws.String(""), MessageId: aws.String("1"), }) require.Equal(t, callCount, 1) require.NoError(t, err) } func TestBatchMessageHandler_Handle_OnJobComplete(t *testing.T) { t.Parallel() awsClient := testAWSClient(t) queueURL := createQueue(t, awsClient) var callCount int mux := http.NewServeMux() mux.HandleFunc("/on-job-complete", func(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodPost { w.WriteHeader(http.StatusMethodNotAllowed) return } callCount++ w.WriteHeader(http.StatusOK) }) server := httptest.NewServer(mux) logger := newLogger(t) defer func() { _ = logger.Sync() }() batchHandler := NewBatchMessageHandler(BatchMessageHandlerConfig{ APIName: "test", JobID: "12345", Region: _localStackDefaultRegion, TargetURL: server.URL, QueueURL: queueURL, }, awsClient, &statsd.NoOpClient{}, logger) batchHandler.jobCompleteMessageDelay = 0 err := batchHandler.Handle(&sqs.Message{ Body: aws.String("job_complete"), MessageAttributes: map[string]*sqs.MessageAttributeValue{ "job_complete": { DataType: aws.String("String"), StringValue: aws.String("true"), }, "api_name": { DataType: aws.String("String"), StringValue: aws.String("test"), }, "job_id": { DataType: aws.String("String"), StringValue: aws.String("12345"), }, }, MessageId: aws.String("00000"), }) require.NoError(t, err) require.Equal(t, callCount, 1) } ================================================ FILE: pkg/dequeuer/dequeuer.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package dequeuer import ( "time" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/sqs" awslib "github.com/cortexlabs/cortex/pkg/lib/aws" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/math" "github.com/cortexlabs/cortex/pkg/lib/telemetry" "go.uber.org/zap" ) var ( _messageAttributes = []string{"All"} _waitTime = 10 * time.Second _visibilityTimeout = 30 * time.Second _notFoundSleepTime = 10 * time.Second _renewalPeriod = 10 * time.Second _probeRefreshPeriod = 1 * time.Second ) type SQSDequeuerConfig struct { Region string QueueURL string StopIfNoMessages bool Workers int } type SQSDequeuer struct { aws *awslib.Client config SQSDequeuerConfig hasDeadLetterQueue bool waitTimeSeconds *int64 visibilityTimeout *int64 notFoundSleepTime time.Duration renewalPeriod time.Duration probeRefreshPeriod time.Duration log *zap.SugaredLogger done chan struct{} } func NewSQSDequeuer(config SQSDequeuerConfig, awsClient *awslib.Client, logger *zap.SugaredLogger) (*SQSDequeuer, error) { attr, err := GetQueueAttributes(awsClient, config.QueueURL) if err != nil { return nil, err } return &SQSDequeuer{ aws: awsClient, config: config, hasDeadLetterQueue: attr.HasRedrivePolicy, waitTimeSeconds: aws.Int64(int64(_waitTime.Seconds())), visibilityTimeout: aws.Int64(int64(_visibilityTimeout.Seconds())), notFoundSleepTime: _notFoundSleepTime, renewalPeriod: _renewalPeriod, probeRefreshPeriod: _probeRefreshPeriod, log: logger, done: make(chan struct{}), }, nil } func (d *SQSDequeuer) ReceiveMessage() (*sqs.Message, error) { output, err := d.aws.SQS().ReceiveMessage(&sqs.ReceiveMessageInput{ QueueUrl: aws.String(d.config.QueueURL), MaxNumberOfMessages: aws.Int64(1), MessageAttributeNames: aws.StringSlice(_messageAttributes), VisibilityTimeout: d.visibilityTimeout, WaitTimeSeconds: d.waitTimeSeconds, }) if err != nil { return nil, errors.WithStack(err) } if len(output.Messages) == 0 { return nil, nil } return output.Messages[0], nil } func (d *SQSDequeuer) Start(messageHandler MessageHandler, readinessProbeFunc func() bool) error { numWorkers := math.MaxInt(d.config.Workers, 1) d.log.Infof("Starting %d workers", numWorkers) errCh := make(chan error) doneChs := make([]chan struct{}, d.config.Workers) for i := 0; i < numWorkers; i++ { doneChs[i] = make(chan struct{}) go func(i int) { errCh <- d.worker(messageHandler, readinessProbeFunc, doneChs[i]) }(i) } select { case err := <-errCh: return err case <-d.done: for _, doneCh := range doneChs { doneCh <- struct{}{} } } return nil } func (d SQSDequeuer) worker(messageHandler MessageHandler, readinessProbeFunc func() bool, workerDone chan struct{}) error { noMessagesInPreviousIteration := false loop: for { select { case <-workerDone: break loop default: if !readinessProbeFunc() { time.Sleep(d.probeRefreshPeriod) continue } message, err := d.ReceiveMessage() if err != nil { return err } if message == nil { // no message received queueAttributes, err := GetQueueAttributes(d.aws, d.config.QueueURL) if err != nil { telemetry.Error(err) return err } if queueAttributes.TotalMessages() == 0 { if noMessagesInPreviousIteration && d.config.StopIfNoMessages { d.log.Info("no messages found in queue, exiting ...") return nil } noMessagesInPreviousIteration = true } time.Sleep(d.notFoundSleepTime) continue } noMessagesInPreviousIteration = false receiptHandle := *message.ReceiptHandle renewerDone := d.StartMessageRenewer(receiptHandle) err = d.handleMessage(message, messageHandler, renewerDone) if err != nil { d.log.Error(err) telemetry.Error(err) } } } return nil } func (d *SQSDequeuer) Shutdown() { d.done <- struct{}{} } func (d *SQSDequeuer) handleMessage(message *sqs.Message, messageHandler MessageHandler, done chan struct{}) error { messageErr := messageHandler.Handle(message) // handle error later done <- struct{}{} isOnJobComplete := isOnJobCompleteMessage(message) if messageErr != nil && d.hasDeadLetterQueue && !isOnJobComplete { // expire messages when dead letter queue is configured to facilitate redrive policy. // always delete onJobComplete messages regardless of redrive policy because a new one will // be added if an onJobComplete message has been consumed prematurely _, err := d.aws.SQS().ChangeMessageVisibility( &sqs.ChangeMessageVisibilityInput{ QueueUrl: &d.config.QueueURL, ReceiptHandle: message.ReceiptHandle, VisibilityTimeout: aws.Int64(0), }, ) if err != nil { return errors.Wrap(err, "failed to change sqs message visibility") } return nil } _, err := d.aws.SQS().DeleteMessage( &sqs.DeleteMessageInput{ QueueUrl: &d.config.QueueURL, ReceiptHandle: message.ReceiptHandle, }, ) if err != nil { return errors.Wrap(err, "failed to delete sqs message") } if messageErr != nil { return messageErr } return nil } func (d *SQSDequeuer) StartMessageRenewer(receiptHandle string) chan struct{} { done := make(chan struct{}) ticker := time.NewTicker(d.renewalPeriod) startTime := time.Now() go func() { defer ticker.Stop() for { select { case <-done: return case tickerTime := <-ticker.C: newVisibilityTimeout := tickerTime.Sub(startTime) + d.renewalPeriod _, err := d.aws.SQS().ChangeMessageVisibility( &sqs.ChangeMessageVisibilityInput{ QueueUrl: &d.config.QueueURL, ReceiptHandle: &receiptHandle, VisibilityTimeout: aws.Int64(int64(newVisibilityTimeout.Seconds())), }, ) if err != nil { d.log.Errorw("failed to renew message visibility timeout", "error", err) } } } }() return done } ================================================ FILE: pkg/dequeuer/dequeuer_test.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package dequeuer import ( "errors" "fmt" "log" "net/http" "os" "testing" "time" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/sqs" awslib "github.com/cortexlabs/cortex/pkg/lib/aws" "github.com/cortexlabs/cortex/pkg/lib/random" "github.com/ory/dockertest/v3" dc "github.com/ory/dockertest/v3/docker" "github.com/stretchr/testify/require" "go.uber.org/zap" ) var localStackEndpoint string const ( _localStackDefaultRegion = "us-east-1" ) func TestMain(m *testing.M) { // uses a sensible default on windows (tcp/http) and linux/osx (socket) log.Println("Starting AWS localstack docker...") pool, err := dockertest.NewPool("") if err != nil { log.Fatalf("Could not connect to docker: %s", err) } options := &dockertest.RunOptions{ Repository: "localstack/localstack", Tag: "latest", PortBindings: map[dc.Port][]dc.PortBinding{ "4566/tcp": { {HostPort: "4566"}, }, }, Env: []string{"SERVICES=sqs,s3"}, } resource, err := pool.RunWithOptions(options) if err != nil { log.Fatalf("Could not start resource: %s", err) } err = resource.Expire(90) if err != nil { log.Fatal(err) } localStackEndpoint = fmt.Sprintf("localhost:%s", resource.GetPort("4566/tcp")) // exponential backoff-retry, because the application in the container might not be ready to accept connections yet // the minio client does not do service discovery for you (i.e. it does not check if connection can be established), so we have to use the health check if err := pool.Retry(func() error { url := fmt.Sprintf("http://%s/health", localStackEndpoint) resp, err := http.Get(url) if err != nil { return err } if resp.StatusCode != http.StatusOK { return fmt.Errorf("status code not OK") } return nil }); err != nil { log.Fatalf("Could not connect to docker: %s", err) } code := m.Run() // You can't defer this because os.Exit doesn't care for defer if err := pool.Purge(resource); err != nil { log.Fatalf("Could not purge resource: %s", err) } os.Exit(code) } func testAWSClient(t *testing.T) *awslib.Client { t.Helper() sess, err := session.NewSessionWithOptions(session.Options{ Config: aws.Config{ Credentials: credentials.NewStaticCredentials("test", "test", ""), Endpoint: aws.String(localStackEndpoint), Region: aws.String(_localStackDefaultRegion), // localstack default region DisableSSL: aws.Bool(true), S3ForcePathStyle: aws.Bool(true), }, }) require.NoError(t, err) client, err := awslib.NewForSession(sess) require.NoError(t, err) return client } func newLogger(t *testing.T) *zap.SugaredLogger { t.Helper() config := zap.NewDevelopmentConfig() config.Level = zap.NewAtomicLevelAt(zap.ErrorLevel) logger, err := config.Build() require.NoError(t, err) logr := logger.Sugar() return logr } func createQueue(t *testing.T, awsClient *awslib.Client) string { t.Helper() createQueueOutput, err := awsClient.SQS().CreateQueue( &sqs.CreateQueueInput{ QueueName: aws.String(fmt.Sprintf("test-%s.fifo", random.Digits(5))), Attributes: aws.StringMap( map[string]string{ sqs.QueueAttributeNameFifoQueue: "true", sqs.QueueAttributeNameVisibilityTimeout: "60", }, ), }, ) require.NoError(t, err) require.NotNil(t, createQueueOutput.QueueUrl) require.NotEmpty(t, *createQueueOutput.QueueUrl) queueURL := *createQueueOutput.QueueUrl return queueURL } func TestSQSDequeuer_ReceiveMessage(t *testing.T) { t.Parallel() awsClient := testAWSClient(t) queueURL := createQueue(t, awsClient) messageID := "12345" messageBody := "blah" sentMessage, err := awsClient.SQS().SendMessage(&sqs.SendMessageInput{ MessageBody: aws.String(messageBody), MessageDeduplicationId: aws.String(messageID), MessageGroupId: aws.String(messageID), QueueUrl: aws.String(queueURL), }) require.NoError(t, err) logger := newLogger(t) defer func() { _ = logger.Sync() }() dq, err := NewSQSDequeuer( SQSDequeuerConfig{ Region: _localStackDefaultRegion, QueueURL: queueURL, StopIfNoMessages: true, Workers: 1, }, awsClient, logger, ) require.NoError(t, err) gotMessage, err := dq.ReceiveMessage() require.NoError(t, err) require.NotNil(t, gotMessage) require.Equal(t, messageBody, *gotMessage.Body) require.Equal(t, *sentMessage.MessageId, *gotMessage.MessageId) } func TestSQSDequeuer_StartMessageRenewer(t *testing.T) { t.Parallel() awsClient := testAWSClient(t) queueURL := createQueue(t, awsClient) logger := newLogger(t) defer func() { _ = logger.Sync() }() dq, err := NewSQSDequeuer( SQSDequeuerConfig{ Region: _localStackDefaultRegion, QueueURL: queueURL, StopIfNoMessages: true, Workers: 1, }, awsClient, logger, ) require.NoError(t, err) dq.renewalPeriod = time.Second dq.visibilityTimeout = aws.Int64(2) messageID := "12345" messageBody := "blah" _, err = awsClient.SQS().SendMessage(&sqs.SendMessageInput{ MessageBody: aws.String(messageBody), MessageDeduplicationId: aws.String(messageID), MessageGroupId: aws.String(messageID), QueueUrl: aws.String(queueURL), }) require.NoError(t, err) message, err := dq.ReceiveMessage() require.NoError(t, err) require.NotNil(t, message) done := dq.StartMessageRenewer(*message.ReceiptHandle) defer func() { done <- struct{}{} }() require.Never(t, func() bool { msg, err := dq.ReceiveMessage() require.NoError(t, err) return msg != nil }, time.Second, 10*time.Second) } func TestSQSDequeuerTerminationOnEmptyQueue(t *testing.T) { t.Parallel() awsClient := testAWSClient(t) queueURL := createQueue(t, awsClient) logger := newLogger(t) defer func() { _ = logger.Sync() }() dq, err := NewSQSDequeuer( SQSDequeuerConfig{ Region: _localStackDefaultRegion, QueueURL: queueURL, StopIfNoMessages: true, Workers: 1, }, awsClient, logger, ) require.NoError(t, err) dq.notFoundSleepTime = 0 dq.waitTimeSeconds = aws.Int64(0) messageID := "12345" messageBody := "blah" _, err = awsClient.SQS().SendMessage(&sqs.SendMessageInput{ MessageBody: aws.String(messageBody), MessageDeduplicationId: aws.String(messageID), MessageGroupId: aws.String(messageID), QueueUrl: aws.String(queueURL), }) require.NoError(t, err) msgHandler := &messageHandlerFunc{ HandleFunc: func(msg *sqs.Message) error { return nil }, } errCh := make(chan error, 1) go func() { errCh <- dq.Start(msgHandler, func() bool { return true }) }() time.AfterFunc(10*time.Second, func() { errCh <- errors.New("timeout: dequeuer did not finish") }) err = <-errCh require.NoError(t, err) } func TestSQSDequeuer_Shutdown(t *testing.T) { t.Parallel() awsClient := testAWSClient(t) queueURL := createQueue(t, awsClient) logger := newLogger(t) defer func() { _ = logger.Sync() }() dq, err := NewSQSDequeuer( SQSDequeuerConfig{ Region: _localStackDefaultRegion, QueueURL: queueURL, StopIfNoMessages: true, Workers: 1, }, awsClient, logger, ) require.NoError(t, err) dq.notFoundSleepTime = 0 dq.waitTimeSeconds = aws.Int64(0) msgHandler := NewMessageHandlerFunc( func(message *sqs.Message) error { return nil }, ) errCh := make(chan error, 1) go func() { errCh <- dq.Start(msgHandler, func() bool { return true }) }() time.AfterFunc(5*time.Second, func() { errCh <- errors.New("timeout: dequeuer did not exit") }) dq.Shutdown() err = <-errCh require.NoError(t, err) } func TestSQSDequeuer_Start_HandlerError(t *testing.T) { t.Parallel() awsClient := testAWSClient(t) queueURL := createQueue(t, awsClient) logger := newLogger(t) defer func() { _ = logger.Sync() }() dq, err := NewSQSDequeuer( SQSDequeuerConfig{ Region: _localStackDefaultRegion, QueueURL: queueURL, StopIfNoMessages: true, Workers: 1, }, awsClient, logger, ) require.NoError(t, err) dq.waitTimeSeconds = aws.Int64(0) dq.notFoundSleepTime = 0 dq.renewalPeriod = 500 * time.Millisecond dq.visibilityTimeout = aws.Int64(1) msgHandler := NewMessageHandlerFunc( func(message *sqs.Message) error { return fmt.Errorf("an error occurred") }, ) messageID := "12345" messageBody := "blah" _, err = awsClient.SQS().SendMessage(&sqs.SendMessageInput{ MessageBody: aws.String(messageBody), MessageDeduplicationId: aws.String(messageID), MessageGroupId: aws.String(messageID), QueueUrl: aws.String(queueURL), }) require.NoError(t, err) go func() { err := dq.Start(msgHandler, func() bool { return true }) require.NoError(t, err) }() require.Never(t, func() bool { msg, err := dq.ReceiveMessage() require.NoError(t, err) return msg != nil }, 5*time.Second, time.Second) } // this test seems to be non-deterministically timing out // it seems to be an issue with the test, not the deqeueur // func TestSQSDequeuer_MultipleWorkers(t *testing.T) { // t.Parallel() // awsClient := testAWSClient(t) // queueURL := createQueue(t, awsClient) // numMessages := 3 // expectedMsgs := make([]string, numMessages) // for i := 0; i < numMessages; i++ { // message := fmt.Sprintf("%d", i) // expectedMsgs[i] = message // _, err := awsClient.SQS().SendMessage(&sqs.SendMessageInput{ // MessageBody: aws.String(message), // MessageDeduplicationId: aws.String(message), // MessageGroupId: aws.String(message), // QueueUrl: aws.String(queueURL), // }) // require.NoError(t, err) // } // logger := newLogger(t) // defer func() { _ = logger.Sync() }() // dq, err := NewSQSDequeuer( // SQSDequeuerConfig{ // Region: _localStackDefaultRegion, // QueueURL: queueURL, // StopIfNoMessages: true, // Workers: numMessages, // }, awsClient, logger, // ) // require.NoError(t, err) // dq.waitTimeSeconds = aws.Int64(0) // dq.notFoundSleepTime = 0 // msgCh := make(chan string, numMessages) // handler := NewMessageHandlerFunc( // func(message *sqs.Message) error { // msgCh <- *message.Body // return nil // }, // ) // errCh := make(chan error) // go func() { // errCh <- dq.Start(handler, func() bool { return true }) // }() // receivedMessages := make([]string, numMessages) // for i := 0; i < numMessages; i++ { // receivedMessages[i] = <-msgCh // } // dq.Shutdown() // // timeout test after 30 seconds // time.AfterFunc(30*time.Second, func() { // close(msgCh) // errCh <- errors.New("test timed out") // }) // require.Len(t, receivedMessages, numMessages) // set := strset.FromSlice(receivedMessages) // require.True(t, set.Has(expectedMsgs...)) // require.NoError(t, <-errCh) // } ================================================ FILE: pkg/dequeuer/errors.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package dequeuer import ( "fmt" "github.com/cortexlabs/cortex/pkg/lib/errors" ) const ( ErrUserContainerResponseStatusCode = "dequeuer.user_container_response_status_code" ErrUserContainerResponseMissingJSONHeader = "dequeuer.user_container_response_missing_json_header" ErrUserContainerResponseNotJSONDecodable = "dequeuer.user_container_response_not_json_decodable" ErrUserContainerNotReachable = "dequeuer.user_container_not_reachable" ) func ErrorUserContainerResponseStatusCode(statusCode int) error { return &errors.Error{ Kind: ErrUserContainerResponseStatusCode, Message: fmt.Sprintf("invalid response from user container; got status code %d, expected status code 200", statusCode), NoTelemetry: true, } } func ErrorUserContainerResponseMissingJSONHeader() error { return &errors.Error{ Kind: ErrUserContainerResponseMissingJSONHeader, Message: "invalid response from user container; response content type header is not 'application/json'", NoTelemetry: true, } } func ErrorUserContainerResponseNotJSONDecodable() error { return &errors.Error{ Kind: ErrUserContainerResponseNotJSONDecodable, Message: "invalid response from user container; response is not json decodable", NoTelemetry: true, } } func ErrorUserContainerNotReachable(err error) error { return &errors.Error{ Kind: ErrUserContainerNotReachable, Message: fmt.Sprintf("user container not reachable: %v", err), NoTelemetry: true, } } ================================================ FILE: pkg/dequeuer/http_handler.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package dequeuer import "net/http" func HealthcheckHandler(isHealthy func() bool) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { if !isHealthy() { w.WriteHeader(http.StatusInternalServerError) _, _ = w.Write([]byte("unhealthy")) return } w.WriteHeader(http.StatusOK) _, _ = w.Write([]byte("healthy")) } } ================================================ FILE: pkg/dequeuer/message_handler.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package dequeuer import "github.com/aws/aws-sdk-go/service/sqs" type MessageHandler interface { Handle(*sqs.Message) error } func NewMessageHandlerFunc(handleFunc func(*sqs.Message) error) MessageHandler { return &messageHandlerFunc{HandleFunc: handleFunc} } type messageHandlerFunc struct { HandleFunc func(message *sqs.Message) error } func (h *messageHandlerFunc) Handle(msg *sqs.Message) error { return h.HandleFunc(msg) } ================================================ FILE: pkg/dequeuer/probes.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package dequeuer import ( "github.com/cortexlabs/cortex/pkg/lib/files" libjson "github.com/cortexlabs/cortex/pkg/lib/json" "github.com/cortexlabs/cortex/pkg/probe" "go.uber.org/zap" kcore "k8s.io/api/core/v1" ) func ProbesFromFile(probesPath string, logger *zap.SugaredLogger) ([]*probe.Probe, error) { fileBytes, err := files.ReadFileBytes(probesPath) if err != nil { return nil, err } probesMap := map[string]kcore.Probe{} if err := libjson.Unmarshal(fileBytes, &probesMap); err != nil { return nil, err } probesSlice := make([]*probe.Probe, len(probesMap)) var i int for _, p := range probesMap { auxProbe := p probesSlice[i] = probe.NewProbe(&auxProbe, logger) i++ } return probesSlice, nil } func HasTCPProbeTargetingUserPod(probes []*probe.Probe, userPort int) bool { for _, pb := range probes { if pb == nil { continue } if pb.Handler.TCPSocket != nil && pb.Handler.TCPSocket.Port.IntValue() == userPort { return true } } return false } ================================================ FILE: pkg/dequeuer/probes_test.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package dequeuer import ( "testing" "github.com/cortexlabs/cortex/pkg/probe" "github.com/stretchr/testify/require" kcore "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/intstr" ) func TestDefaultTCPProbeNotPresent(t *testing.T) { t.Parallel() log := newLogger(t) defer func() { _ = log.Sync() }() userPodPort := 8080 probes := []*probe.Probe{ probe.NewProbe(&kcore.Probe{ Handler: kcore.Handler{ Exec: &kcore.ExecAction{ Command: []string{"/bin/bash", "python", "test.py"}, }, }, }, log), probe.NewProbe(&kcore.Probe{ Handler: kcore.Handler{ HTTPGet: &kcore.HTTPGetAction{ Path: "some-path", Port: intstr.FromInt(12345), Host: "localhost", }, }, }, log), probe.NewProbe(&kcore.Probe{ Handler: kcore.Handler{ TCPSocket: &kcore.TCPSocketAction{ Port: intstr.FromInt(8447), Host: "localhost", }, }, }, log), } require.False(t, HasTCPProbeTargetingUserPod(probes, userPodPort)) } func TestDefaultTCPProbePresent(t *testing.T) { t.Parallel() log := newLogger(t) defer func() { _ = log.Sync() }() userPodPort := intstr.FromInt(8080) probes := []*probe.Probe{ probe.NewProbe(&kcore.Probe{ Handler: kcore.Handler{ Exec: &kcore.ExecAction{ Command: []string{"/bin/bash", "python", "test.py"}, }, }, }, log), probe.NewProbe(&kcore.Probe{ Handler: kcore.Handler{ HTTPGet: &kcore.HTTPGetAction{ Path: "some-path", Port: intstr.FromInt(12345), Host: "localhost", }, }, }, log), probe.NewProbe(&kcore.Probe{ Handler: kcore.Handler{ TCPSocket: &kcore.TCPSocketAction{ Port: userPodPort, Host: "localhost", }, }, }, log), } require.True(t, HasTCPProbeTargetingUserPod(probes, userPodPort.IntValue())) } ================================================ FILE: pkg/dequeuer/queue_attributes.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package dequeuer import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/sqs" awslib "github.com/cortexlabs/cortex/pkg/lib/aws" "github.com/cortexlabs/cortex/pkg/lib/errors" s "github.com/cortexlabs/cortex/pkg/lib/strings" ) type QueueAttributes struct { VisibleMessages int InvisibleMessages int HasRedrivePolicy bool } func (attr QueueAttributes) TotalMessages() int { return attr.VisibleMessages + attr.InvisibleMessages } func GetQueueAttributes(client *awslib.Client, queueURL string) (QueueAttributes, error) { result, err := client.SQS().GetQueueAttributes( &sqs.GetQueueAttributesInput{ QueueUrl: aws.String(queueURL), AttributeNames: aws.StringSlice([]string{"All"}), }, ) if err != nil { return QueueAttributes{}, errors.WithStack(err) } attributes := aws.StringValueMap(result.Attributes) var visibleCount int var notVisibleCount int var hasRedrivePolicy bool if val, found := attributes["ApproximateNumberOfMessages"]; found { count, ok := s.ParseInt(val) if ok { visibleCount = count } } if val, found := attributes["ApproximateNumberOfMessagesNotVisible"]; found { count, ok := s.ParseInt(val) if ok { notVisibleCount = count } } _, hasRedrivePolicy = attributes["RedrivePolicy"] return QueueAttributes{ VisibleMessages: visibleCount, InvisibleMessages: notVisibleCount, HasRedrivePolicy: hasRedrivePolicy, }, nil } ================================================ FILE: pkg/dequeuer/request_stats.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package dequeuer import "time" type RequestEvent struct { StatusCode int Duration time.Duration } type RequestEventHandler interface { HandleEvent(event RequestEvent) } type requestEventHandlerFunc struct { HandleFunc func(event RequestEvent) } func (h *requestEventHandlerFunc) HandleEvent(event RequestEvent) { h.HandleFunc(event) } func NewRequestEventHandlerFunc(handleFunc func(event RequestEvent)) RequestEventHandler { return &requestEventHandlerFunc{ HandleFunc: handleFunc, } } ================================================ FILE: pkg/enqueuer/enqueuer.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package enqueuer import ( "bytes" "encoding/json" "fmt" "io" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/sqs" awslib "github.com/cortexlabs/cortex/pkg/lib/aws" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/pointer" "github.com/cortexlabs/cortex/pkg/lib/random" s "github.com/cortexlabs/cortex/pkg/lib/strings" "github.com/cortexlabs/cortex/pkg/types/spec" "github.com/cortexlabs/cortex/pkg/types/userconfig" "go.uber.org/zap" ) const ( _s3DownloadChunkSize = 32 * 1024 * 1024 ) type EnvConfig struct { ClusterUID string Region string Version string Bucket string APIName string JobID string } // FIXME: all these types should be shared with the cortex web server (from where the payload is submitted) type ItemList struct { Items []json.RawMessage `json:"items"` BatchSize int `json:"batch_size"` } type S3Lister struct { S3Paths []string `json:"s3_paths"` // s3:///key Includes []string `json:"includes"` Excludes []string `json:"excludes"` MaxResults *int64 `json:"-"` // this is not currently exposed to the user (it's used for validations) } type FilePathLister struct { S3Lister BatchSize int `json:"batch_size"` } type DelimitedFiles struct { S3Lister BatchSize int `json:"batch_size"` } type JobSubmission struct { ItemList *ItemList `json:"item_list"` FilePathLister *FilePathLister `json:"file_path_lister"` DelimitedFiles *DelimitedFiles `json:"delimited_files"` } type onJobCompleteRequestBody struct { Message string `json:"message"` } func randomMessageID() string { return random.String(40) // maximum is 80 (for sqs.SendMessageBatchRequestEntry.Id) but this ID may show up in a user error message } type Enqueuer struct { aws *awslib.Client envConfig EnvConfig queueURL string logger *zap.Logger } func NewEnqueuer(envConfig EnvConfig, queueURL string, logger *zap.Logger) (*Enqueuer, error) { awsClient, err := awslib.NewForRegion(envConfig.Region) if err != nil { return nil, err } return &Enqueuer{ aws: awsClient, envConfig: envConfig, queueURL: queueURL, logger: logger, }, nil } func (e *Enqueuer) Enqueue() (int, error) { submission, err := e.getJobPayload() if err != nil { return 0, err } totalBatches := 0 if submission.ItemList != nil { totalBatches, err = e.enqueueItems(submission.ItemList) if err != nil { return 0, err } } else if submission.FilePathLister != nil { totalBatches, err = e.enqueueS3Paths(submission.FilePathLister) if err != nil { return 0, err } } else if submission.DelimitedFiles != nil { totalBatches, err = e.enqueueS3FileContents(submission.DelimitedFiles) if err != nil { return 0, err } } onJobCompleteBodyBytes, err := json.Marshal(onJobCompleteRequestBody{ Message: "job_complete", }) if err != nil { return 0, err } randomID := randomMessageID() _, err = e.aws.SQS().SendMessage(&sqs.SendMessageInput{ QueueUrl: aws.String(e.queueURL), MessageBody: aws.String(string(onJobCompleteBodyBytes)), MessageDeduplicationId: aws.String(randomID), // prevent content based deduping MessageGroupId: aws.String(randomID), // aws recommends message group id per message to improve chances of exactly-once MessageAttributes: map[string]*sqs.MessageAttributeValue{ "job_complete": { DataType: aws.String("String"), StringValue: aws.String("true"), }, "api_name": { DataType: aws.String("String"), StringValue: aws.String(e.envConfig.APIName), }, "job_id": { DataType: aws.String("String"), StringValue: aws.String(e.envConfig.JobID), }, }, }) if err != nil { return 0, errors.Wrap(err, "failed to enqueue job_complete placeholder") } if err = e.deleteJobPayload(); err != nil { return 0, err } return totalBatches, nil } func (e *Enqueuer) UploadBatchCount(batchCount int) error { key := spec.JobBatchCountKey(e.envConfig.ClusterUID, userconfig.BatchAPIKind, e.envConfig.APIName, e.envConfig.JobID) return e.aws.UploadStringToS3(s.Int(batchCount), e.envConfig.Bucket, key) } func (e *Enqueuer) getJobPayload() (JobSubmission, error) { // e.g. /jobs//// key := spec.JobPayloadKey(e.envConfig.ClusterUID, userconfig.BatchAPIKind, e.envConfig.APIName, e.envConfig.JobID) submissionBytes, err := e.aws.ReadBytesFromS3(e.envConfig.Bucket, key) if err != nil { return JobSubmission{}, err } var submission JobSubmission if err = json.Unmarshal(submissionBytes, &submission); err != nil { return JobSubmission{}, err } return submission, nil } func (e *Enqueuer) deleteJobPayload() error { key := spec.JobPayloadKey(e.envConfig.ClusterUID, userconfig.BatchAPIKind, e.envConfig.APIName, e.envConfig.JobID) if err := e.aws.DeleteS3File(e.envConfig.Bucket, key); err != nil { return err } return nil } func (e *Enqueuer) enqueueItems(itemList *ItemList) (int, error) { log := e.logger batchCount := len(itemList.Items) / itemList.BatchSize if len(itemList.Items)%itemList.BatchSize != 0 { batchCount++ } log.Info( "partitioning items found in job submission into batches", zap.Int("numItems", len(itemList.Items)), zap.Int("batchCount", batchCount), zap.Int("batchSize", itemList.BatchSize), ) uploader := newSQSBatchUploader(e.envConfig.APIName, e.envConfig.JobID, e.queueURL, e.aws.SQS()) for i := 0; i < batchCount; i++ { min := i * (itemList.BatchSize) max := (i + 1) * (itemList.BatchSize) if max > len(itemList.Items) { max = len(itemList.Items) } jsonBytes, err := json.Marshal(itemList.Items[min:max]) if err != nil { if itemList.BatchSize == 1 { return 0, errors.Wrap(err, fmt.Sprintf("item %d", i)) } return 0, errors.Wrap(err, fmt.Sprintf("items with index between %d to %d", min, max)) } err = uploader.AddToBatch(randomMessageID(), pointer.String(string(jsonBytes))) if err != nil { if itemList.BatchSize == 1 { return 0, errors.Wrap(err, fmt.Sprintf("item %d", i)) } return 0, errors.Wrap(err, fmt.Sprintf("items with index between %d to %d", min, max)) } if uploader.TotalBatches%100 == 0 { log.Info("enqueued batches", zap.Int("batchCount", uploader.TotalBatches)) } } err := uploader.Flush() if err != nil { return 0, err } return uploader.TotalBatches, nil } func (e *Enqueuer) enqueueS3Paths(s3PathsLister *FilePathLister) (int, error) { log := e.logger var s3PathList []string uploader := newSQSBatchUploader(e.envConfig.APIName, e.envConfig.JobID, e.queueURL, e.aws.SQS()) _, err := s3IteratorFromLister(e.aws, s3PathsLister.S3Lister, func(bucket string, s3Obj *s3.Object) (bool, error) { s3Path := awslib.S3Path(bucket, *s3Obj.Key) s3PathList = append(s3PathList, s3Path) if len(s3PathList) == s3PathsLister.BatchSize { err := addS3PathsToQueue(uploader, s3PathList) if err != nil { return false, err } s3PathList = nil if uploader.TotalBatches%100 == 0 { log.Info("enqueued batches", zap.Int("numBatches", uploader.TotalBatches)) } } return true, nil }) if err != nil { return 0, err } if len(s3PathList) > 0 { err := addS3PathsToQueue(uploader, s3PathList) if err != nil { return 0, err } } err = uploader.Flush() if err != nil { return 0, err } return uploader.TotalBatches, nil } func (e *Enqueuer) enqueueS3FileContents(delimitedFiles *DelimitedFiles) (int, error) { log := e.logger jsonMessageList := newJSONBuffer(delimitedFiles.BatchSize) uploader := newSQSBatchUploader(e.envConfig.APIName, e.envConfig.JobID, e.queueURL, e.aws.SQS()) bytesBuffer := bytes.NewBuffer([]byte{}) _, err := s3IteratorFromLister(e.aws, delimitedFiles.S3Lister, func(bucket string, s3Obj *s3.Object) (bool, error) { s3Path := awslib.S3Path(bucket, *s3Obj.Key) log.Info("enqueuing contents from file", zap.String("path", s3Path)) awsClientForBucket, err := awslib.NewFromClientS3Path(s3Path, e.aws) if err != nil { return false, err } itemIndex := 0 err = awsClientForBucket.S3FileIterator(bucket, s3Obj, _s3DownloadChunkSize, func(readCloser io.ReadCloser, isLastChunk bool) (bool, error) { _, err := bytesBuffer.ReadFrom(readCloser) if err != nil { return false, err } err = e.streamJSONToQueue(uploader, bytesBuffer, jsonMessageList, &itemIndex) if err != nil { if errors.CauseOrSelf(err) != io.ErrUnexpectedEOF || (errors.CauseOrSelf(err) == io.ErrUnexpectedEOF && isLastChunk) { return false, err } } return true, nil }) if err != nil { return false, err } return true, nil }) if err != nil { return 0, err } if jsonMessageList.Length() != 0 { err := addJSONObjectsToQueue(uploader, jsonMessageList) if err != nil { return 0, err } } err = uploader.Flush() if err != nil { return 0, err } return uploader.TotalBatches, nil } func (e *Enqueuer) streamJSONToQueue(uploader *sqsBatchUploader, bytesBuffer *bytes.Buffer, jsonMessageList *jsonBuffer, itemIndex *int) error { log := e.logger dec := json.NewDecoder(bytesBuffer) for { var doc json.RawMessage err := dec.Decode(&doc) if err == io.EOF { break } else if err == io.ErrUnexpectedEOF { bytesBuffer.Reset() _, _ = bytesBuffer.ReadFrom(dec.Buffered()) return io.ErrUnexpectedEOF } else if err != nil { return errors.Wrap(err, fmt.Sprintf("item %d", *itemIndex)) } if len(doc) > _messageSizeLimit { return errors.Wrap(ErrorMessageExceedsMaxSize(len(doc), _messageSizeLimit), fmt.Sprintf("item %d", *itemIndex)) } *itemIndex++ jsonMessageList.Add(doc) if jsonMessageList.Length() == jsonMessageList.BatchSize { err := addJSONObjectsToQueue(uploader, jsonMessageList) if err != nil { return err } jsonMessageList.Clear() if uploader.TotalBatches%100 == 0 { log.Info("enqueued batches", zap.Int("numBatches", uploader.TotalBatches)) } } } return nil } func addS3PathsToQueue(uploader *sqsBatchUploader, s3PathList []string) error { jsonBytes, err := json.Marshal(s3PathList) if err != nil { return errors.Wrap(err, fmt.Sprintf("batch %d", uploader.TotalBatches)) } err = uploader.AddToBatch(randomMessageID(), pointer.String(string(jsonBytes))) if err != nil { return err } return nil } ================================================ FILE: pkg/enqueuer/errors.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package enqueuer import ( "fmt" "github.com/cortexlabs/cortex/pkg/lib/errors" ) const ( ErrFailedToEnqueueMessages = "batchapi.failed_to_enqueue_messages" ErrMessageExceedsMaxSize = "batchapi.message_exceeds_max_size" ) func ErrorFailedToEnqueueMessages(message string) error { return errors.WithStack(&errors.Error{ Kind: ErrFailedToEnqueueMessages, Message: message, }) } func ErrorMessageExceedsMaxSize(messageSize int, messageLimit int) error { return errors.WithStack(&errors.Error{ Kind: ErrMessageExceedsMaxSize, Message: fmt.Sprintf("cannot enqueue message because its size of %d bytes exceeds the %d bytes limit; use a smaller batch size or reduce the size of each of item in the batch", messageSize, messageLimit), }) } ================================================ FILE: pkg/enqueuer/helpers.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package enqueuer import ( "encoding/json" "github.com/aws/aws-sdk-go/service/s3" awslib "github.com/cortexlabs/cortex/pkg/lib/aws" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/pointer" "github.com/gobwas/glob" ) type jsonBuffer struct { BatchSize int messageList []json.RawMessage } func newJSONBuffer(batchSize int) *jsonBuffer { return &jsonBuffer{ BatchSize: batchSize, messageList: make([]json.RawMessage, 0, batchSize), } } func (j *jsonBuffer) Add(jsonMessage json.RawMessage) { j.messageList = append(j.messageList, jsonMessage) } func (j *jsonBuffer) Clear() { j.messageList = make([]json.RawMessage, 0, j.BatchSize) } func (j *jsonBuffer) Length() int { return len(j.messageList) } func addJSONObjectsToQueue(uploader *sqsBatchUploader, jsonMessageList *jsonBuffer) error { jsonBytes, err := json.Marshal(jsonMessageList.messageList) if err != nil { return err } err = uploader.AddToBatch(randomMessageID(), pointer.String(string(jsonBytes))) if err != nil { return err } return nil } func s3IteratorFromLister(awsClient *awslib.Client, s3Lister S3Lister, fn func(string, *s3.Object) (bool, error)) (int64, error) { includeGlobPatterns := make([]glob.Glob, 0, len(s3Lister.Includes)) for _, includePattern := range s3Lister.Includes { globExpression, err := glob.Compile(includePattern, '/') if err != nil { return 0, errors.Wrap(err, "failed to interpret glob pattern", includePattern) } includeGlobPatterns = append(includeGlobPatterns, globExpression) } excludeGlobPatterns := make([]glob.Glob, 0, len(s3Lister.Excludes)) for _, excludePattern := range s3Lister.Excludes { globExpression, err := glob.Compile(excludePattern, '/') if err != nil { return 0, errors.Wrap(err, "failed to interpret glob pattern", excludePattern) } excludeGlobPatterns = append(excludeGlobPatterns, globExpression) } var numResults int64 for _, s3Path := range s3Lister.S3Paths { bucket, key, err := awslib.SplitS3Path(s3Path) if err != nil { return 0, err } awsClientForBucket, err := awslib.NewFromClientS3Path(s3Path, awsClient) if err != nil { return 0, err } err = awsClientForBucket.S3Iterator(bucket, key, false, nil, nil, func(s3Obj *s3.Object) (bool, error) { s3FilePath := awslib.S3Path(bucket, *s3Obj.Key) shouldSkip := false if len(includeGlobPatterns) > 0 { shouldSkip = true for _, includeGlobPattern := range includeGlobPatterns { if includeGlobPattern.Match(s3FilePath) { shouldSkip = false break } } } for _, excludeGlobPattern := range excludeGlobPatterns { if excludeGlobPattern.Match(s3FilePath) { shouldSkip = true break } } if !shouldSkip { shouldContinue, err := fn(bucket, s3Obj) numResults++ if s3Lister.MaxResults != nil && numResults >= *s3Lister.MaxResults { shouldContinue = false } return shouldContinue, err } return true, nil }) if err != nil { return 0, err } if s3Lister.MaxResults != nil && numResults >= *s3Lister.MaxResults { return numResults, nil } } return numResults, nil } ================================================ FILE: pkg/enqueuer/uploader.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package enqueuer import ( "fmt" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/sqs" "github.com/cortexlabs/cortex/pkg/lib/errors" ) const ( _messageSizeLimit = 250 * 1024 // normally its 256 * 1024 but reserve 6k for message attributes _maxMessagesPerBatch = 10 ) type sqsBatchUploader struct { client *sqs.SQS messageAttributes map[string]*sqs.MessageAttributeValue queueURL string retries int // default 3 times messageList []*sqs.SendMessageBatchRequestEntry messageIDToListIndex map[string]int totalBytes int TotalBatches int } func newSQSBatchUploader(apiName, jobID, queueURL string, client *sqs.SQS) *sqsBatchUploader { messageAttributes := map[string]*sqs.MessageAttributeValue{ "api_name": { DataType: aws.String("String"), StringValue: aws.String(apiName), }, "job_id": { DataType: aws.String("String"), StringValue: aws.String(jobID), }, } return &sqsBatchUploader{ client: client, messageAttributes: messageAttributes, queueURL: queueURL, retries: 3, messageIDToListIndex: map[string]int{}, } } func (uploader *sqsBatchUploader) AddToBatch(id string, body *string) error { if len(*body) > _messageSizeLimit { return ErrorMessageExceedsMaxSize(len(*body), _messageSizeLimit) } message := &sqs.SendMessageBatchRequestEntry{ MessageAttributes: uploader.messageAttributes, Id: aws.String(id), MessageBody: body, MessageDeduplicationId: aws.String(id), // prevent content based deduping MessageGroupId: aws.String(id), // aws recommends message group id per message to improve chances of exactly-once } if len(*message.MessageBody)+uploader.totalBytes > _messageSizeLimit || len(uploader.messageList) == _maxMessagesPerBatch { err := uploader.Flush() if err != nil { return err } } uploader.messageList = append(uploader.messageList, message) uploader.messageIDToListIndex[id] = uploader.TotalBatches uploader.totalBytes += len(*message.MessageBody) uploader.TotalBatches++ return nil } func (uploader *sqsBatchUploader) Flush() error { if len(uploader.messageList) == 0 { return nil } var err error for attempt := 0; attempt < uploader.retries; attempt++ { err = uploader.enqueueToSQS() if err == nil { uploader.messageList = nil uploader.messageIDToListIndex = map[string]int{} uploader.totalBytes = 0 return nil } } return errors.Wrap(err, fmt.Sprintf("failed after retrying %d times", uploader.retries)) } func (uploader *sqsBatchUploader) enqueueToSQS() error { output, err := uploader.client.SendMessageBatch(&sqs.SendMessageBatchInput{ QueueUrl: aws.String(uploader.queueURL), Entries: uploader.messageList, }) if err != nil { if output == nil { return errors.ErrorUnexpected("got unexpected nil pointer") } if len(output.Failed) == 0 { return errors.WithStack(err) } return errors.Wrap(ErrorFailedToEnqueueMessages(*output.Failed[0].Message), fmt.Sprintf("batch %d", uploader.messageIDToListIndex[*output.Failed[0].Id])) } return nil } ================================================ FILE: pkg/health/health.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package health import ( "context" "fmt" "reflect" "github.com/cortexlabs/cortex/pkg/consts" "github.com/cortexlabs/cortex/pkg/lib/aws" awslib "github.com/cortexlabs/cortex/pkg/lib/aws" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/json" "github.com/cortexlabs/cortex/pkg/lib/k8s" "github.com/cortexlabs/cortex/pkg/lib/parallel" "github.com/cortexlabs/cortex/pkg/types/clusterconfig" kapps "k8s.io/api/apps/v1" kcore "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" kresource "k8s.io/apimachinery/pkg/api/resource" kmeta "k8s.io/apimachinery/pkg/apis/meta/v1" kmetrics "k8s.io/metrics/pkg/client/clientset/versioned" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" ) // ClusterHealth represents the healthiness of each component of a cluster type ClusterHealth struct { Operator bool `json:"operator"` ControllerManager bool `json:"controller_manager"` Prometheus bool `json:"prometheus"` Autoscaler bool `json:"autoscaler"` Activator bool `json:"activator"` AsyncGateway bool `json:"async_gateway"` Grafana bool `json:"grafana"` OperatorGateway bool `json:"operator_gateway"` APIsGateway bool `json:"apis_gateway"` ClusterAutoscaler bool `json:"cluster_autoscaler"` OperatorLoadBalancer bool `json:"operator_load_balancer"` APIsLoadBalancer bool `json:"apis_load_balancer"` FluentBit bool `json:"fluent_bit"` NodeExporter bool `json:"node_exporter"` DCGMExporter bool `json:"dcgm_exporter"` StatsDExporter bool `json:"statsd_exporter"` EventExporter bool `json:"event_exporter"` KubeStateMetrics bool `json:"kube_state_metrics"` } func (c ClusterHealth) String() string { bytes, err := json.Marshal(c) if err != nil { panic(err) } return string(bytes) } type ClusterWarnings struct { Prometheus string } // HasWarnings checks if ClusterWarnings has any warnings in its' fields func (w ClusterWarnings) HasWarnings() bool { v := reflect.ValueOf(w) for i := 0; i < v.NumField(); i++ { if v.Field(i).String() != "" { return true } } return false } // Check checks for the health of the different components of a cluster func Check(awsClient *awslib.Client, k8sClient *k8s.Client, clusterName string) (ClusterHealth, error) { var ( operatorHealth bool controllerManagerHealth bool prometheusHealth bool autoscalerHealth bool activatorHealth bool asyncGatewayHealth bool grafanaHealth bool operatorGatewayHealth bool apisGatewayHealth bool clusterAutoscalerHealth bool operatorLoadBalancerHealth bool apisLoadBalancerHealth bool fluentBitHealth bool nodeExporterHealth bool dcgmExporterHealth bool statsdExporterHealth bool eventExporterHealth bool kubeStateMetricsHealth bool ) if err := parallel.RunFirstErr( func() error { var err error operatorHealth, err = getDeploymentReadiness(k8sClient, "operator", consts.DefaultNamespace) return err }, func() error { var err error controllerManagerHealth, err = getDeploymentReadiness(k8sClient, "operator-controller-manager", consts.DefaultNamespace) return err }, func() error { var err error prometheusHealth, err = getStatefulSetReadiness(k8sClient, "prometheus-prometheus", consts.PrometheusNamespace) return err }, func() error { var err error autoscalerHealth, err = getDeploymentReadiness(k8sClient, "autoscaler", consts.DefaultNamespace) return err }, func() error { var err error activatorHealth, err = getDeploymentReadiness(k8sClient, "activator", consts.DefaultNamespace) return err }, func() error { var err error asyncGatewayHealth, err = getDeploymentReadiness(k8sClient, "async-gateway", consts.DefaultNamespace) return err }, func() error { var err error grafanaHealth, err = getStatefulSetReadiness(k8sClient, "grafana", consts.DefaultNamespace) return err }, func() error { var err error operatorGatewayHealth, err = getDeploymentReadiness(k8sClient, "ingressgateway-operator", consts.IstioNamespace) return err }, func() error { var err error apisGatewayHealth, err = getDeploymentReadiness(k8sClient, "ingressgateway-apis", consts.IstioNamespace) return err }, func() error { var err error clusterAutoscalerHealth, err = getDeploymentReadiness(k8sClient, "cluster-autoscaler", consts.KubeSystemNamespace) return err }, func() error { var err error operatorLoadBalancerHealth, err = getLoadBalancerHealth(awsClient, clusterName, "operator", false) return err }, func() error { var err error apisLoadBalancerHealth, err = getLoadBalancerHealth(awsClient, clusterName, "api", true) return err }, func() error { var err error fluentBitHealth, err = getDaemonSetReadiness(k8sClient, "fluent-bit", consts.LoggingNamespace) return err }, func() error { var err error dcgmExporterHealth, err = getDaemonSetReadiness(k8sClient, "dcgm-exporter", consts.PrometheusNamespace) return err }, func() error { var err error nodeExporterHealth, err = getDaemonSetReadiness(k8sClient, "node-exporter", consts.PrometheusNamespace) return err }, func() error { var err error statsdExporterHealth, err = getDeploymentReadiness(k8sClient, "prometheus-statsd-exporter", consts.PrometheusNamespace) return err }, func() error { var err error eventExporterHealth, err = getDeploymentReadiness(k8sClient, "event-exporter", consts.LoggingNamespace) return err }, func() error { var err error kubeStateMetricsHealth, err = getDeploymentReadiness(k8sClient, "kube-state-metrics", consts.PrometheusNamespace) return err }, ); err != nil { return ClusterHealth{}, err } return ClusterHealth{ Operator: operatorHealth, ControllerManager: controllerManagerHealth, Prometheus: prometheusHealth, Autoscaler: autoscalerHealth, Activator: activatorHealth, AsyncGateway: asyncGatewayHealth, Grafana: grafanaHealth, OperatorGateway: operatorGatewayHealth, APIsGateway: apisGatewayHealth, ClusterAutoscaler: clusterAutoscalerHealth, OperatorLoadBalancer: operatorLoadBalancerHealth, APIsLoadBalancer: apisLoadBalancerHealth, FluentBit: fluentBitHealth, NodeExporter: nodeExporterHealth, DCGMExporter: dcgmExporterHealth, StatsDExporter: statsdExporterHealth, EventExporter: eventExporterHealth, KubeStateMetrics: kubeStateMetricsHealth, }, nil } func GetWarnings(k8sClient *k8s.Client) (ClusterWarnings, error) { var prometheusMemorySaturationWarn string saturation, err := getPodMemorySaturation(k8sClient, "prometheus-prometheus-0", consts.PrometheusNamespace) if err != nil { return ClusterWarnings{}, err } if saturation >= 0.7 { prometheusMemorySaturationWarn = fmt.Sprintf("memory usage is critically high (%.1f%%)", saturation*100) } return ClusterWarnings{ Prometheus: prometheusMemorySaturationWarn, }, nil } func getDeploymentReadiness(k8sClient *k8s.Client, name, namespace string) (bool, error) { ctx := context.Background() var deployment kapps.Deployment if err := k8sClient.Get(ctx, ctrlclient.ObjectKey{ Namespace: namespace, Name: name, }, &deployment); err != nil { if kerrors.IsNotFound(err) { return false, nil } return false, err } return deployment.Status.ReadyReplicas > 0, nil } func getStatefulSetReadiness(k8sClient *k8s.Client, name, namespace string) (bool, error) { ctx := context.Background() var statefulSet kapps.StatefulSet if err := k8sClient.Get(ctx, ctrlclient.ObjectKey{ Namespace: namespace, Name: name, }, &statefulSet); err != nil { if kerrors.IsNotFound(err) { return false, nil } return false, err } return statefulSet.Status.ReadyReplicas > 0, nil } func getDaemonSetReadiness(k8sClient *k8s.Client, name, namespace string) (bool, error) { ctx := context.Background() var daemonSet kapps.DaemonSet if err := k8sClient.Get(ctx, ctrlclient.ObjectKey{ Namespace: namespace, Name: name, }, &daemonSet); err != nil { if kerrors.IsNotFound(err) { return false, nil } return false, err } return daemonSet.Status.NumberReady == daemonSet.Status.CurrentNumberScheduled, nil } func getLoadBalancerHealth(awsClient *awslib.Client, clusterName string, loadBalancerName string, testClassicLB bool) (bool, error) { loadBalancerV2, err := awsClient.FindLoadBalancerV2(map[string]string{ clusterconfig.ClusterNameTag: clusterName, "cortex.dev/load-balancer": loadBalancerName, }) loadBalancerV2Exists := err == nil && loadBalancerV2 != nil if loadBalancerV2Exists { return aws.IsLoadBalancerV2Healthy(*loadBalancerV2), nil } if !testClassicLB { if err == nil { return false, errors.ErrorUnexpected(fmt.Sprintf("unable to locate %s nlb load balancer", loadBalancerName)) } return false, errors.Wrap(err, fmt.Sprintf("unable to locate %s nlb load balancer", loadBalancerName)) } loadBalancer, err := awsClient.FindLoadBalancer(map[string]string{ clusterconfig.ClusterNameTag: clusterName, "cortex.dev/load-balancer": loadBalancerName, }) loadBalancerExists := err == nil && loadBalancer != nil if !loadBalancerExists { if err == nil { return false, errors.ErrorUnexpected(fmt.Sprintf("unable to locate %s elb load balancer", loadBalancerName)) } return false, errors.Wrap(err, fmt.Sprintf("unable to locate %s elb load balancer", loadBalancerName)) } healthy, err := awsClient.IsLoadBalancerHealthy(*loadBalancer.LoadBalancerName) if err != nil { return false, errors.Wrap(err, fmt.Sprintf("unable to check %s elb load balancer", loadBalancerName)) } return healthy, nil } func getPodMemorySaturation(k8sClient *k8s.Client, podName, namespace string) (float64, error) { ctx := context.Background() var pod kcore.Pod if err := k8sClient.Get(ctx, ctrlclient.ObjectKey{ Namespace: namespace, Name: podName, }, &pod); err != nil { return 0, err } metricsClient, err := kmetrics.NewForConfig(k8sClient.RestConfig) if err != nil { return 0, err } podMetrics, err := metricsClient.MetricsV1beta1().PodMetricses(namespace).Get(ctx, podName, kmeta.GetOptions{}) if err != nil { return 0, err } var totalMememoryUsage kresource.Quantity for _, container := range podMetrics.Containers { memory := container.Usage.Memory() if memory != nil { totalMememoryUsage.Add(*container.Usage.Memory()) } } node, err := k8sClient.ClientSet().CoreV1().Nodes().Get(ctx, pod.Spec.NodeName, kmeta.GetOptions{}) if err != nil { return 0, err } nodeMemory := node.Status.Allocatable.Memory() memRatio := totalMememoryUsage.AsApproximateFloat64() / nodeMemory.AsApproximateFloat64() return memRatio, nil } ================================================ FILE: pkg/lib/archive/archive_test.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package archive import ( "io/ioutil" "os" "path/filepath" "testing" "github.com/cortexlabs/cortex/pkg/lib/files" "github.com/cortexlabs/cortex/pkg/lib/maps" "github.com/stretchr/testify/require" ) func TestArchive(t *testing.T) { tmpDir, err := files.TmpDir() defer os.RemoveAll(tmpDir) require.NoError(t, err) filesList := []string{ filepath.Join(tmpDir, "1.txt"), filepath.Join(tmpDir, "2.py"), filepath.Join(tmpDir, "3/1.py"), filepath.Join(tmpDir, "3/2/1.py"), filepath.Join(tmpDir, "3/2/2.txt"), filepath.Join(tmpDir, "3/2/3/.tmp"), filepath.Join(tmpDir, "4/1.yaml"), filepath.Join(tmpDir, "4/2.pyc"), filepath.Join(tmpDir, "5/4/3/2/1.txt"), filepath.Join(tmpDir, "5/4/3/2/1.py"), filepath.Join(tmpDir, "5/4/3/2/2/1.py"), } err = files.MakeEmptyFiles(filesList[0], filesList[1:]...) require.NoError(t, err) var input *Input var expected []string input = &Input{ Bytes: []BytesInput{ { Content: []byte(""), Dest: "text.txt", }, { Content: []byte(""), Dest: "test2/text2.txt", }, { Content: []byte(""), Dest: "/test3/text3.txt", }, }, } expected = []string{ "text.txt", "test2/text2.txt", "test3/text3.txt", } CheckArchive(input, expected, false, t) input = &Input{ Files: []FileInput{ { Source: filepath.Join(tmpDir, "1.txt"), Dest: "1.txt", }, { Source: filepath.Join(tmpDir, "3/1.py"), Dest: "3/1.py", }, { Source: filepath.Join(tmpDir, "3/2/1.py"), Dest: "3/2/1.py", }, { Source: filepath.Join(tmpDir, "3/2/3/.tmp"), Dest: "3/2/3/.tmp", }, { Source: filepath.Join(tmpDir, "4/2.pyc"), Dest: "4/4/2.pyc", }, }, } expected = []string{ "1.txt", "3/1.py", "3/2/1.py", "3/2/3/.tmp", "4/4/2.pyc", } CheckArchive(input, expected, false, t) input = &Input{ Bytes: []BytesInput{ { Content: []byte(""), Dest: "text.txt", }, }, Files: []FileInput{ { Source: filepath.Join(tmpDir, "1.txt"), Dest: "1/2/3.txt", }, }, AddPrefix: "test", } expected = []string{ "test/text.txt", "test/1/2/3.txt", } CheckArchive(input, expected, false, t) input = &Input{ Bytes: []BytesInput{ { Content: []byte(""), Dest: "text.txt", }, }, Files: []FileInput{ { Source: filepath.Join(tmpDir, "1.txt"), Dest: "1/2/3.txt", }, }, AddPrefix: "/test", } expected = []string{ "test/text.txt", "test/1/2/3.txt", } CheckArchive(input, expected, false, t) input = &Input{ EmptyFiles: []string{ "text.txt", "1/2/3.txt", }, } expected = []string{ "text.txt", "1/2/3.txt", } CheckArchive(input, expected, false, t) input = &Input{ Dirs: []DirInput{ { Source: tmpDir, }, }, } expected = []string{ "1.txt", "2.py", "3/1.py", "3/2/1.py", "3/2/2.txt", "3/2/3/.tmp", "4/1.yaml", "4/2.pyc", "5/4/3/2/1.txt", "5/4/3/2/1.py", "5/4/3/2/2/1.py", } CheckArchive(input, expected, false, t) input = &Input{ Dirs: []DirInput{ { Source: filepath.Join(tmpDir, "3"), Dest: ".", }, }, } expected = []string{ "1.py", "2/1.py", "2/2.txt", "2/3/.tmp", } CheckArchive(input, expected, false, t) input = &Input{ Dirs: []DirInput{ { Source: tmpDir, Dest: "/", IgnoreFns: []files.IgnoreFn{files.IgnoreHiddenFiles}, }, }, } expected = []string{ "1.txt", "2.py", "3/1.py", "3/2/1.py", "3/2/2.txt", "4/1.yaml", "4/2.pyc", "5/4/3/2/1.txt", "5/4/3/2/1.py", "5/4/3/2/2/1.py", } CheckArchive(input, expected, false, t) input = &Input{ Dirs: []DirInput{ { Source: filepath.Join(tmpDir, "3"), IgnoreFns: []files.IgnoreFn{files.IgnoreHiddenFiles}, Dest: "test3", Flatten: true, }, { Source: filepath.Join(tmpDir, "4"), Dest: "test4/", }, }, AllowOverwrite: true, } expected = []string{ "test3/1.py", "test3/2.txt", "test4/1.yaml", "test4/2.pyc", } CheckArchive(input, expected, false, t) input = &Input{ Dirs: []DirInput{ { Source: filepath.Join(tmpDir, "5"), RemovePrefix: "4/3", }, }, } expected = []string{ "2/1.txt", "2/1.py", "2/2/1.py", } CheckArchive(input, expected, false, t) input = &Input{ Dirs: []DirInput{ { Source: filepath.Join(tmpDir, "5"), RemovePrefix: "/4/3", }, }, } expected = []string{ "2/1.txt", "2/1.py", "2/2/1.py", } CheckArchive(input, expected, false, t) input = &Input{ Dirs: []DirInput{ { Source: filepath.Join(tmpDir, "5"), Flatten: true, }, }, AllowOverwrite: true, } expected = []string{ "1.txt", "1.py", } CheckArchive(input, expected, false, t) input = &Input{ Dirs: []DirInput{ { Source: filepath.Join(tmpDir, "5"), RemoveCommonPrefix: true, }, }, } expected = []string{ "1.txt", "1.py", "2/1.py", } CheckArchive(input, expected, false, t) input = &Input{ Bytes: []BytesInput{ { Content: []byte(""), Dest: "1/text.txt", }, { Content: []byte(""), Dest: "1/text.txt", }, }, } CheckArchive(input, nil, true, t) input = &Input{ Bytes: []BytesInput{ { Content: []byte(""), Dest: "1/text.txt", }, }, EmptyFiles: []string{"1/text.txt"}, } CheckArchive(input, nil, true, t) input = &Input{ Dirs: []DirInput{ { Source: filepath.Join(tmpDir, "3"), Flatten: true, }, }, } CheckArchive(input, nil, true, t) input = &Input{ Dirs: []DirInput{ { Source: filepath.Join(tmpDir, "5"), Flatten: true, }, }, } CheckArchive(input, nil, true, t) } func CheckArchive(input *Input, expected []string, shouldErr bool, t *testing.T) { CheckZip(input, expected, shouldErr, t) CheckTar(input, expected, shouldErr, t) CheckTgz(input, expected, shouldErr, t) } func CheckZip(input *Input, expected []string, shouldErr bool, t *testing.T) { tmpDir, err := files.TmpDir() defer os.RemoveAll(tmpDir) require.NoError(t, err) _, err = ZipToFile(input, filepath.Join(tmpDir, "archive.zip")) if shouldErr { require.Error(t, err) return } require.NoError(t, err) _, err = UnzipFileToDir(filepath.Join(tmpDir, "archive.zip"), filepath.Join(tmpDir, "archive")) require.NoError(t, err) unzippedFiles, err := files.ListDirRecursive(filepath.Join(tmpDir, "archive"), true) require.NoError(t, err) require.ElementsMatch(t, expected, unzippedFiles) contents, err := UnzipFileToMem(filepath.Join(tmpDir, "archive.zip")) require.NoError(t, err) require.ElementsMatch(t, expected, maps.InterfaceMapKeysUnsafe(contents)) zipBytes, err := ioutil.ReadFile(filepath.Join(tmpDir, "archive.zip")) require.NoError(t, err) contents, err = UnzipMemToMem(zipBytes) require.NoError(t, err) require.ElementsMatch(t, expected, maps.InterfaceMapKeysUnsafe(contents)) } func CheckTar(input *Input, expected []string, shouldErr bool, t *testing.T) { tmpDir, err := files.TmpDir() defer os.RemoveAll(tmpDir) require.NoError(t, err) _, err = TarToFile(input, filepath.Join(tmpDir, "archive.tar")) if shouldErr { require.Error(t, err) return } require.NoError(t, err) _, err = UntarFileToDir(filepath.Join(tmpDir, "archive.tar"), filepath.Join(tmpDir, "archive")) require.NoError(t, err) untaredFiles, err := files.ListDirRecursive(filepath.Join(tmpDir, "archive"), true) require.NoError(t, err) require.ElementsMatch(t, expected, untaredFiles) contents, err := UntarFileToMem(filepath.Join(tmpDir, "archive.tar")) require.NoError(t, err) require.ElementsMatch(t, expected, maps.InterfaceMapKeysUnsafe(contents)) tarBytes, err := ioutil.ReadFile(filepath.Join(tmpDir, "archive.tar")) require.NoError(t, err) contents, err = UntarMemToMem(tarBytes) require.NoError(t, err) require.ElementsMatch(t, expected, maps.InterfaceMapKeysUnsafe(contents)) } func CheckTgz(input *Input, expected []string, shouldErr bool, t *testing.T) { tmpDir, err := files.TmpDir() defer os.RemoveAll(tmpDir) require.NoError(t, err) _, err = TgzToFile(input, filepath.Join(tmpDir, "archive.tgz")) if shouldErr { require.Error(t, err) return } require.NoError(t, err) _, err = UntgzFileToDir(filepath.Join(tmpDir, "archive.tgz"), filepath.Join(tmpDir, "archive")) require.NoError(t, err) untgzedFiles, err := files.ListDirRecursive(filepath.Join(tmpDir, "archive"), true) require.NoError(t, err) require.ElementsMatch(t, expected, untgzedFiles) contents, err := UntgzFileToMem(filepath.Join(tmpDir, "archive.tgz")) require.NoError(t, err) require.ElementsMatch(t, expected, maps.InterfaceMapKeysUnsafe(contents)) tgzBytes, err := ioutil.ReadFile(filepath.Join(tmpDir, "archive.tgz")) require.NoError(t, err) contents, err = UntgzMemToMem(tgzBytes) require.NoError(t, err) require.ElementsMatch(t, expected, maps.InterfaceMapKeysUnsafe(contents)) } ================================================ FILE: pkg/lib/archive/archiver.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package archive import ( "bytes" "io" "path/filepath" "strings" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/files" "github.com/cortexlabs/cortex/pkg/lib/sets/strset" s "github.com/cortexlabs/cortex/pkg/lib/strings" ) type archiveType int const ( unknownArchiveType archiveType = iota zipArchiveType tarArchiveType tgzArchiveType ) type archiver interface { add(reader io.Reader, dest string, size int64) error close() error } func archive(input *Input, arc archiver) (strset.Set, error) { addedPaths := strset.New() var err error for i := range input.Bytes { err = addBytesToArchive(&input.Bytes[i], input, arc, addedPaths) if err != nil { return nil, err } } for i := range input.Files { err = addFileToArchive(&input.Files[i], input, arc, addedPaths) if err != nil { return nil, err } } for i := range input.Dirs { err = addDirToArchive(&input.Dirs[i], input, arc, addedPaths) if err != nil { return nil, err } } for i := range input.FileLists { err = addFileListToArchive(&input.FileLists[i], input, arc, addedPaths) if err != nil { return nil, err } } for _, emptyFilePath := range input.EmptyFiles { err = addEmptyFileToArchive(emptyFilePath, input, arc, addedPaths) if err != nil { return nil, err } } return addedPaths, nil } func addBytesToArchive(byteInput *BytesInput, input *Input, arc archiver, addedPaths strset.Set) error { path := filepath.Join(input.AddPrefix, byteInput.Dest) path = strings.TrimPrefix(path, "/") if !input.AllowOverwrite { if addedPaths.Has(path) { return ErrorDuplicatePath(path) } addedPaths.Add(path) } reader := bytes.NewReader(byteInput.Content) return arc.add(reader, path, reader.Size()) } func addFileToArchive(fileInput *FileInput, input *Input, arc archiver, addedPaths strset.Set) error { content, err := files.ReadFileBytes(fileInput.Source) if err != nil { return err } byteInput := &BytesInput{ Content: content, Dest: fileInput.Dest, } return addBytesToArchive(byteInput, input, arc, addedPaths) } func addDirToArchive(dirInput *DirInput, input *Input, arc archiver, addedPaths strset.Set) error { paths, err := files.ListDirRecursive(dirInput.Source, true, dirInput.IgnoreFns...) if err != nil { return err } commonPrefix := "" if dirInput.RemoveCommonPrefix { commonPrefix = s.LongestCommonPrefix(paths...) } removePrefix := strings.TrimPrefix(dirInput.RemovePrefix, "/") for _, path := range paths { file := filepath.Join(dirInput.Source, path) if dirInput.Flatten { path = filepath.Base(path) } else { path = strings.TrimPrefix(path, removePrefix) path = strings.TrimPrefix(path, commonPrefix) } fileInput := &FileInput{ Source: file, Dest: filepath.Join(dirInput.Dest, path), } err = addFileToArchive(fileInput, input, arc, addedPaths) if err != nil { return err } } return nil } func addFileListToArchive(fileListInput *FileListInput, input *Input, arc archiver, addedPaths strset.Set) error { commonPrefix := "" if fileListInput.RemoveCommonPrefix { commonPrefix = s.LongestCommonPrefix(fileListInput.Sources...) } for _, path := range fileListInput.Sources { fullPath := path if fileListInput.Flatten { path = filepath.Base(path) } else { path = strings.TrimPrefix(path, fileListInput.RemovePrefix) path = strings.TrimPrefix(path, commonPrefix) } fileInput := &FileInput{ Source: fullPath, Dest: filepath.Join(fileListInput.Dest, path), } err := addFileToArchive(fileInput, input, arc, addedPaths) if err != nil { return err } } return nil } func addEmptyFileToArchive(path string, input *Input, arc archiver, addedPaths strset.Set) error { byteInput := &BytesInput{ Content: []byte{}, Dest: path, } return addBytesToArchive(byteInput, input, arc, addedPaths) } func archiveToWriter(input *Input, writer io.Writer, arcType archiveType) (strset.Set, error) { var arc archiver switch arcType { case zipArchiveType: arc = newZipArchiver(writer) case tarArchiveType: arc = newTarArchiver(writer) case tgzArchiveType: arc = newTgzArchiver(writer) default: return nil, errors.ErrorUnexpected("unknown archive type:", arcType) } paths, err := archive(input, arc) if err != nil { arc.close() return nil, err } err = arc.close() if err != nil { return nil, errors.Wrap(err, _errStrCreateArchive) } return paths, nil } func archiveToMem(input *Input, arcType archiveType) ([]byte, strset.Set, error) { buf := new(bytes.Buffer) paths, err := archiveToWriter(input, buf, arcType) if err != nil { return nil, nil, err } return buf.Bytes(), paths, nil } func archiveToFile(input *Input, destDir string, arcType archiveType) (strset.Set, error) { cleanDestDir, err := files.EscapeTilde(destDir) if err != nil { return nil, err } archiveFile, err := files.Create(cleanDestDir) if err != nil { return nil, err } paths, err := archiveToWriter(input, archiveFile, arcType) if err != nil { archiveFile.Close() return nil, err } err = archiveFile.Close() if err != nil { return nil, errors.Wrap(err, destDir, _errStrCreateArchive) } return paths, nil } ================================================ FILE: pkg/lib/archive/errors.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package archive import ( "fmt" "github.com/cortexlabs/cortex/pkg/lib/errors" s "github.com/cortexlabs/cortex/pkg/lib/strings" ) const ( _errStrCreateArchive = "unable to create archive" _errStrCreateZip = "unable to create zip file" _errStrCreateTar = "unable to create tar file" _errStrUnzip = "unable to unzip file" _errStrUntar = "unable to extract tar file" ) const ( ErrDuplicatePath = "archive.duplicate_path" ) func ErrorDuplicatePath(path string) error { return errors.WithStack(&errors.Error{ Kind: ErrDuplicatePath, Message: fmt.Sprintf("duplicate path was provided (%s)", s.UserStr(path)), }) } ================================================ FILE: pkg/lib/archive/input.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package archive import ( "github.com/cortexlabs/cortex/pkg/lib/files" ) type Input struct { Files []FileInput Bytes []BytesInput Dirs []DirInput FileLists []FileListInput AddPrefix string // Gets added to every item EmptyFiles []string // Empty files to be created AllowOverwrite bool // Don't error if a file in the zip is overwritten } type FileInput struct { Source string Dest string } type BytesInput struct { Content []byte Dest string } type DirInput struct { Source string Dest string IgnoreFns []files.IgnoreFn Flatten bool RemovePrefix string RemoveCommonPrefix bool } type FileListInput struct { Sources []string Dest string Flatten bool RemovePrefix string RemoveCommonPrefix bool } ================================================ FILE: pkg/lib/archive/tar.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package archive import ( "archive/tar" "bytes" "io" "io/ioutil" "os" "path/filepath" "strings" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/files" "github.com/cortexlabs/cortex/pkg/lib/sets/strset" ) type tarArchiver struct { writer *tar.Writer } func newTarArchiver(writer io.Writer) *tarArchiver { return &tarArchiver{ writer: tar.NewWriter(writer), } } func (arc *tarArchiver) add(reader io.Reader, dest string, size int64) error { header := &tar.Header{ Name: dest, Size: size, } err := arc.writer.WriteHeader(header) if err != nil { return errors.Wrap(err, _errStrCreateTar) } _, err = io.Copy(arc.writer, reader) if err != nil { return errors.Wrap(err, _errStrCreateTar) } return nil } func (arc *tarArchiver) close() error { return arc.writer.Close() } func TarToWriter(input *Input, writer io.Writer) (strset.Set, error) { return archiveToWriter(input, writer, tarArchiveType) } func TarToFile(input *Input, destDir string) (strset.Set, error) { return archiveToFile(input, destDir, tarArchiveType) } func TarToMem(input *Input) ([]byte, strset.Set, error) { return archiveToMem(input, tarArchiveType) } // Will create destDir if missing func UntarReaderToDir(reader io.Reader, destDir string) (strset.Set, error) { destDir, err := files.Clean(destDir) if err != nil { return nil, err } tarReader := tar.NewReader(reader) filenames := strset.New() for { header, err := tarReader.Next() switch { case err == io.EOF: return filenames, nil case err != nil: return nil, errors.WithStack(err) case header == nil: continue } name := strings.TrimPrefix(header.Name, "/") target := filepath.Join(destDir, name) switch header.Typeflag { case tar.TypeDir: err := files.CreateDir(target) if err != nil { return nil, err } case tar.TypeReg: filenames.Add(target) err := files.CreateDir(filepath.Dir(target)) if err != nil { return nil, err } outFile, err := files.OpenFile(target, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666) if err != nil { return nil, err } _, err = io.Copy(outFile, tarReader) if err != nil { outFile.Close() return nil, errors.Wrap(err, _errStrUntar) } outFile.Close() } } } // Will create destDir if missing func UntarFileToDir(src string, destDir string) (strset.Set, error) { file, err := files.Open(src) if err != nil { return nil, err } defer file.Close() return UntarReaderToDir(file, destDir) } func UntarReaderToMem(reader io.Reader) (map[string][]byte, error) { fileMap := map[string][]byte{} tarReader := tar.NewReader(reader) for { header, err := tarReader.Next() switch { case err == io.EOF: return fileMap, nil case err != nil: return nil, err case header == nil: continue } if header.Typeflag == tar.TypeReg { contents, err := ioutil.ReadAll(tarReader) if err != nil { return nil, errors.Wrap(err, _errStrUntar) } path := strings.TrimPrefix(header.Name, "/") fileMap[path] = contents } } } func UntarMemToMem(tarBytes []byte) (map[string][]byte, error) { return UntarReaderToMem(bytes.NewReader(tarBytes)) } func UntarFileToMem(src string) (map[string][]byte, error) { file, err := files.Open(src) if err != nil { return nil, errors.Wrap(err, _errStrUntar) } defer file.Close() return UntarReaderToMem(file) } ================================================ FILE: pkg/lib/archive/tgz.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package archive import ( "bytes" "compress/gzip" "io" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/files" "github.com/cortexlabs/cortex/pkg/lib/sets/strset" ) type tgzArchiver struct { tarArc *tarArchiver gzipWriter *gzip.Writer } func newTgzArchiver(writer io.Writer) *tgzArchiver { gzipWriter := gzip.NewWriter(writer) return &tgzArchiver{ tarArc: newTarArchiver(gzipWriter), gzipWriter: gzipWriter, } } func (arc *tgzArchiver) add(reader io.Reader, dest string, size int64) error { return arc.tarArc.add(reader, dest, size) } func (arc *tgzArchiver) close() error { err1 := arc.tarArc.close() err2 := arc.gzipWriter.Close() return errors.FirstError(err1, err2) } func TgzToWriter(input *Input, writer io.Writer) (strset.Set, error) { return archiveToWriter(input, writer, tgzArchiveType) } func TgzToFile(input *Input, destDir string) (strset.Set, error) { return archiveToFile(input, destDir, tgzArchiveType) } func TgzToMem(input *Input) ([]byte, strset.Set, error) { return archiveToMem(input, tgzArchiveType) } // Will create destDir if missing func UntgzReaderToDir(reader io.Reader, destDir string) (strset.Set, error) { gzipReader, err := gzip.NewReader(reader) if err != nil { return nil, err } defer gzipReader.Close() return UntarReaderToDir(gzipReader, destDir) } // Will create destDir if missing func UntgzFileToDir(src string, destDir string) (strset.Set, error) { file, err := files.Open(src) if err != nil { return nil, err } defer file.Close() return UntgzReaderToDir(file, destDir) } func UntgzReaderToMem(reader io.Reader) (map[string][]byte, error) { gzipReader, err := gzip.NewReader(reader) if err != nil { return nil, err } defer gzipReader.Close() return UntarReaderToMem(gzipReader) } func UntgzMemToMem(tgzBytes []byte) (map[string][]byte, error) { return UntgzReaderToMem(bytes.NewReader(tgzBytes)) } func UntgzFileToMem(src string) (map[string][]byte, error) { file, err := files.Open(src) if err != nil { return nil, errors.Wrap(err, _errStrUntar) } defer file.Close() return UntgzReaderToMem(file) } ================================================ FILE: pkg/lib/archive/zip.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package archive import ( "archive/zip" "bytes" "io" "io/ioutil" "os" "path/filepath" "strings" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/files" "github.com/cortexlabs/cortex/pkg/lib/sets/strset" ) type zipArchiver struct { writer *zip.Writer } func newZipArchiver(writer io.Writer) *zipArchiver { return &zipArchiver{ writer: zip.NewWriter(writer), } } func (arc *zipArchiver) add(reader io.Reader, dest string, size int64) error { writer, err := arc.writer.Create(dest) if err != nil { return errors.Wrap(err, _errStrCreateZip) } _, err = io.Copy(writer, reader) if err != nil { return errors.Wrap(err, _errStrCreateZip) } return nil } func (arc *zipArchiver) close() error { return arc.writer.Close() } func ZipToWriter(input *Input, writer io.Writer) (strset.Set, error) { return archiveToWriter(input, writer, zipArchiveType) } func ZipToFile(input *Input, destDir string) (strset.Set, error) { return archiveToFile(input, destDir, zipArchiveType) } func ZipToMem(input *Input) ([]byte, strset.Set, error) { return archiveToMem(input, zipArchiveType) } // Will create destDir if missing func UnzipFileToDir(src string, destDir string) (strset.Set, error) { destDir, err := files.Clean(destDir) if err != nil { return nil, err } cleanSrc, err := files.EscapeTilde(src) if err != nil { return nil, err } filenames := strset.New() zipReader, err := zip.OpenReader(cleanSrc) if err != nil { return nil, errors.Wrap(err, _errStrUnzip) } defer zipReader.Close() for _, zipReaderFile := range zipReader.File { zipFileReader, err := zipReaderFile.Open() if err != nil { return nil, errors.Wrap(err, _errStrUnzip) } defer zipFileReader.Close() name := strings.TrimPrefix(zipReaderFile.Name, "/") target := filepath.Join(destDir, name) if zipReaderFile.FileInfo().IsDir() { err := files.CreateDir(target) if err != nil { return nil, err } } else { filenames.Add(target) err := files.CreateDir(filepath.Dir(target)) if err != nil { return nil, err } outFile, err := files.OpenFile(target, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666) if err != nil { return nil, err } _, err = io.Copy(outFile, zipFileReader) if err != nil { outFile.Close() return nil, errors.Wrap(err, _errStrUnzip) } outFile.Close() } } return filenames, nil } func UnzipMemToMem(zipBytes []byte) (map[string][]byte, error) { zipReader, err := zip.NewReader(bytes.NewReader(zipBytes), int64(len(zipBytes))) if err != nil { return nil, errors.Wrap(err, _errStrUnzip) } return unzipZipReaderToMem(zipReader) } func UnzipFileToMem(src string) (map[string][]byte, error) { cleanSrc, err := files.Clean(src) if err != nil { return nil, err } zipReader, err := zip.OpenReader(cleanSrc) if err != nil { return nil, errors.Wrap(err, _errStrUnzip) } defer zipReader.Close() return unzipZipReaderToMem(&zipReader.Reader) } func unzipZipReaderToMem(zipReader *zip.Reader) (map[string][]byte, error) { fileMap := map[string][]byte{} for _, zipReaderFile := range zipReader.File { if !zipReaderFile.FileInfo().IsDir() { zipFileReader, err := zipReaderFile.Open() if err != nil { return nil, errors.Wrap(err, _errStrUnzip) } defer zipFileReader.Close() contents, err := ioutil.ReadAll(zipFileReader) if err != nil { return nil, errors.Wrap(err, _errStrUnzip) } path := strings.TrimPrefix(zipReaderFile.Name, "/") fileMap[path] = contents } } return fileMap, nil } ================================================ FILE: pkg/lib/aws/acm.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package aws import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/acm" "github.com/cortexlabs/cortex/pkg/lib/errors" ) func (c *Client) DoesCertificateExist(sslCertificateARN string) (bool, error) { _, err := c.ACM().DescribeCertificate(&acm.DescribeCertificateInput{ CertificateArn: aws.String(sslCertificateARN), }) if err != nil { if IsErrCode(err, "ResourceNotFoundException") { return false, nil } return false, errors.Wrap(err, sslCertificateARN) } return true, nil } ================================================ FILE: pkg/lib/aws/apigateway.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package aws import ( "fmt" "strings" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/apigatewayv2" "github.com/cortexlabs/cortex/pkg/lib/errors" ) // CreateAPIGateway Creates a new API Gateway with the default stage func (c *Client) CreateAPIGateway(name string, tags map[string]string) (string, error) { createAPIResponse, err := c.APIGatewayV2().CreateApi(&apigatewayv2.CreateApiInput{ Name: aws.String(name), ProtocolType: aws.String(apigatewayv2.ProtocolTypeHttp), Tags: aws.StringMap(tags), }) if err != nil { return "", errors.Wrap(err, "failed to create api gateway") } if createAPIResponse.ApiId == nil { return "", errors.ErrorUnexpected("failed to create api gateway") } _, err = c.APIGatewayV2().CreateStage(&apigatewayv2.CreateStageInput{ ApiId: createAPIResponse.ApiId, AutoDeploy: aws.Bool(true), StageName: aws.String("$default"), Tags: aws.StringMap(tags), }) if err != nil { c.DeleteAPIGateway(*createAPIResponse.ApiId) // best effort cleanup return "", errors.Wrap(err, "failed to create $default api gateway stage") } return *createAPIResponse.ApiId, nil } // GetVPCLinkByTag Gets a VPC Link by tag (returns nil if there are no matches) func (c *Client) GetVPCLinkByTag(tagName string, tagValue string) (*apigatewayv2.VpcLink, error) { var nextToken *string for { vpcLinks, err := c.APIGatewayV2().GetVpcLinks(&apigatewayv2.GetVpcLinksInput{ NextToken: nextToken, }) if err != nil { return nil, errors.Wrap(err, "failed to get vpc links") } for _, vpcLink := range vpcLinks.Items { for tag, value := range vpcLink.Tags { if tag == tagName && *value == tagValue { return vpcLink, nil } } } nextToken = vpcLinks.NextToken if nextToken == nil { break } } return nil, nil } // GetAPIGatewayByTag Gets an API Gateway by tag (returns nil if there are no matches) func (c *Client) GetAPIGatewayByTag(tagName string, tagValue string) (*apigatewayv2.Api, error) { var nextToken *string for { apis, err := c.APIGatewayV2().GetApis(&apigatewayv2.GetApisInput{ NextToken: nextToken, }) if err != nil { return nil, errors.Wrap(err, "failed to get api gateways") } for _, api := range apis.Items { for tag, value := range api.Tags { if tag == tagName && *value == tagValue { return api, nil } } } nextToken = apis.NextToken if nextToken == nil { break } } return nil, nil } // DeleteVPCLinkByTag Deletes a VPC Link by tag (returns the deleted VPC Link, or nil if it was not found) func (c *Client) DeleteVPCLinkByTag(tagName string, tagValue string) (*apigatewayv2.VpcLink, error) { vpcLink, err := c.GetVPCLinkByTag(tagName, tagValue) if err != nil { return nil, err } else if vpcLink == nil { return nil, nil } _, err = c.APIGatewayV2().DeleteVpcLink(&apigatewayv2.DeleteVpcLinkInput{ VpcLinkId: vpcLink.VpcLinkId, }) if err != nil { return nil, errors.Wrap(err, "failed to delete vpc link "+*vpcLink.VpcLinkId) } return vpcLink, nil } // DeleteAPIGatewayByTag Deletes an API Gateway by tag (returns the deleted API Gateway, or nil if it was not found) func (c *Client) DeleteAPIGatewayByTag(tagName string, tagValue string) (*apigatewayv2.Api, error) { apiGateway, err := c.GetAPIGatewayByTag(tagName, tagValue) if err != nil { return nil, err } else if apiGateway == nil { return nil, nil } err = c.DeleteAPIGateway(*apiGateway.ApiId) if err != nil { return nil, err } return apiGateway, nil } // DeleteAPIGateway Deletes an API Gateway by ID (returns an error if the API Gateway does not exist) func (c *Client) DeleteAPIGateway(apiGatewayID string) error { // Delete mappings in case user added a custom domain name (otherwise this will block API Gateway deletion) err := c.DeleteAPIGatewayMappings(apiGatewayID) if err != nil { return err } _, err = c.APIGatewayV2().DeleteApi(&apigatewayv2.DeleteApiInput{ ApiId: aws.String(apiGatewayID), }) if err != nil { return errors.Wrap(err, "failed to delete api gateway "+apiGatewayID) } return nil } // DeleteAPIGatewayMappingsForDomainName deletes all API mappings that point to the provided api gateway from the provided domain name func (c *Client) DeleteAPIGatewayMappingsForDomainName(apiGatewayID string, domainName string) error { var nextToken *string for { apiMappings, err := c.APIGatewayV2().GetApiMappings(&apigatewayv2.GetApiMappingsInput{ DomainName: aws.String(domainName), NextToken: nextToken, }) if err != nil { return errors.Wrap(err, "failed to get api mappings") } for _, apiMapping := range apiMappings.Items { if *apiMapping.ApiId != apiGatewayID { continue } _, err := c.APIGatewayV2().DeleteApiMapping(&apigatewayv2.DeleteApiMappingInput{ DomainName: aws.String(domainName), ApiMappingId: apiMapping.ApiMappingId, }) if err != nil { return errors.Wrap(err, fmt.Sprintf("failed to delete api mapping %s in domain %s", *apiMapping.ApiMappingId, domainName)) } } nextToken = apiMappings.NextToken if nextToken == nil { break } } return nil } // DeleteAPIGatewayMappings deletes all API mappings that point to the provided api gateway func (c *Client) DeleteAPIGatewayMappings(apiGatewayID string) error { var nextToken *string for { domainNames, err := c.APIGatewayV2().GetDomainNames(&apigatewayv2.GetDomainNamesInput{ NextToken: nextToken, }) if err != nil { return errors.Wrap(err, "failed to get domain names") } for _, domainName := range domainNames.Items { err := c.DeleteAPIGatewayMappingsForDomainName(apiGatewayID, *domainName.DomainName) if err != nil { return err } } nextToken = domainNames.NextToken if nextToken == nil { break } } return nil } // GetVPCLinkIntegration gets the VPC Link integration in an API Gateway, or nil if unable to find it func (c *Client) GetVPCLinkIntegration(apiGatewayID string, vpcLinkID string) (*apigatewayv2.Integration, error) { var nextToken *string for { integrations, err := c.APIGatewayV2().GetIntegrations(&apigatewayv2.GetIntegrationsInput{ ApiId: &apiGatewayID, NextToken: nextToken, }) if err != nil { return nil, errors.Wrap(err, "failed to get api gateway integrations for api gateway "+apiGatewayID) } // find integration which is connected to the VPC link for _, integration := range integrations.Items { if *integration.ConnectionId == vpcLinkID { return integration, nil } } nextToken = integrations.NextToken if nextToken == nil { break } } return nil, nil } // GetRouteIntegrationID returns the integration which is attached to a endpoint route, or empty string if unable to find it func (c *Client) GetRouteIntegrationID(apiGatewayID string, endpoint string) (string, error) { route, err := c.GetRoute(apiGatewayID, endpoint) if err != nil { return "", err } if route == nil { return "", nil } return ExtractRouteIntegrationID(route), nil } // ExtractRouteIntegrationID extracts the integration ID which is attached to a route, or "" if no route is attached func ExtractRouteIntegrationID(route *apigatewayv2.Route) string { if route == nil || route.Target == nil { return "" } // trim of prefix of integrationID. // Note: Integrations get attached to routes via a target of the format integrations/ integrationID := strings.TrimPrefix(*route.Target, "integrations/") return integrationID } // GetRoute retrieves the route matching an endpoint, or nil if unable to find it func (c *Client) GetRoute(apiGatewayID string, endpoint string) (*apigatewayv2.Route, error) { var nextToken *string for { routes, err := c.APIGatewayV2().GetRoutes(&apigatewayv2.GetRoutesInput{ ApiId: &apiGatewayID, NextToken: nextToken, }) if err != nil { return nil, errors.Wrap(err, "failed to get api gateway routes for api gateway "+apiGatewayID) } // find route which matches the endpoint for _, route := range routes.Items { if *route.RouteKey == "ANY "+endpoint { return route, nil } } nextToken = routes.NextToken if nextToken == nil { break } } return nil, nil } // CreateRoute creates a new route and attaches the route to the integration func (c *Client) CreateRoute(apiGatewayID string, integrationID string, endpoint string) error { _, err := c.APIGatewayV2().CreateRoute(&apigatewayv2.CreateRouteInput{ ApiId: &apiGatewayID, RouteKey: aws.String("ANY " + endpoint), Target: aws.String("integrations/" + integrationID), }) if err != nil { return errors.Wrap(err, fmt.Sprintf("failed to create %s route for api gateway %s with integration %s", endpoint, apiGatewayID, integrationID)) } return nil } // CreateHTTPIntegration creates new HTTP integration for API Gateway, returns integration ID func (c *Client) CreateHTTPIntegration(apiGatewayID string, targetEndpoint string) (string, error) { integrationResponse, err := c.APIGatewayV2().CreateIntegration(&apigatewayv2.CreateIntegrationInput{ ApiId: &apiGatewayID, IntegrationType: aws.String("HTTP_PROXY"), IntegrationUri: &targetEndpoint, PayloadFormatVersion: aws.String("1.0"), IntegrationMethod: aws.String("ANY"), }) if err != nil { return "", errors.Wrap(err, fmt.Sprintf("failed to create api gateway integration for endpoint %s in api gateway %s", targetEndpoint, apiGatewayID)) } return *integrationResponse.IntegrationId, nil } // DeleteIntegration deletes an integration from API Gateway func (c *Client) DeleteIntegration(apiGatewayID string, integrationID string) error { _, err := c.APIGatewayV2().DeleteIntegration(&apigatewayv2.DeleteIntegrationInput{ ApiId: &apiGatewayID, IntegrationId: &integrationID, }) if err != nil { return errors.Wrap(err, fmt.Sprintf("failed to delete api gateway integration %s in api gateway %s", integrationID, apiGatewayID)) } return nil } // DeleteRoute deletes a route from API Gateway, and returns the deleted route (or nil if it wasn't found) func (c *Client) DeleteRoute(apiGatewayID string, endpoint string) (*apigatewayv2.Route, error) { route, err := c.GetRoute(apiGatewayID, endpoint) if err != nil { return nil, err } else if route == nil { return nil, nil } _, err = c.APIGatewayV2().DeleteRoute(&apigatewayv2.DeleteRouteInput{ ApiId: &apiGatewayID, RouteId: route.RouteId, }) if err != nil { return nil, errors.Wrap(err, fmt.Sprintf("failed to delete api gateway route %s with endpoint %s in api gateway %s", *route.RouteId, endpoint, apiGatewayID)) } return route, nil } ================================================ FILE: pkg/lib/aws/autoscaling.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package aws import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/autoscaling" "github.com/cortexlabs/cortex/pkg/lib/errors" ) // if specified, all tags must be present func (c *Client) AutoscalingGroups(tags map[string]string) ([]*autoscaling.Group, error) { var asgs []*autoscaling.Group params := autoscaling.DescribeAutoScalingGroupsInput{ AutoScalingGroupNames: nil, } err := c.Autoscaling().DescribeAutoScalingGroupsPages(¶ms, func(page *autoscaling.DescribeAutoScalingGroupsOutput, lastPage bool) bool { for _, asg := range page.AutoScalingGroups { asgTags := make(map[string]string, len(asg.Tags)) for _, asgTag := range asg.Tags { if asgTag.Key != nil && asgTag.Value != nil { asgTags[*asgTag.Key] = *asgTag.Value } } missingTag := false for key, value := range tags { if asgTags[key] != value { missingTag = true break } } if missingTag { continue } asgs = append(asgs, asg) } return true }) if err != nil { return nil, errors.WithStack(err) } return asgs, nil } // Returns the most recent activity for the ASG, or nil if there are no activities func (c *Client) MostRecentASGActivity(asgName string) (*autoscaling.Activity, error) { resp, err := c.Autoscaling().DescribeScalingActivities(&autoscaling.DescribeScalingActivitiesInput{ AutoScalingGroupName: aws.String(asgName), MaxRecords: aws.Int64(1), }) if err != nil { return nil, errors.WithStack(err) } if len(resp.Activities) == 0 { return nil, nil } return resp.Activities[0], nil } ================================================ FILE: pkg/lib/aws/aws.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ //go:generate python3 gen_resource_metadata.py //go:generate gofmt -s -w resource_metadata.go package aws import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/session" "github.com/cortexlabs/cortex/pkg/lib/errors" ) type Client struct { Region string sess *session.Session IsAnonymous bool clients clients accountID *string hashedAccountID *string } func NewForSession(sess *session.Session) (*Client, error) { if sess.Config.Region == nil { return nil, errors.ErrorUnexpected("session config is missing the Region field") } return &Client{ Region: *sess.Config.Region, sess: sess, }, nil } func NewFromClientS3Path(s3Path string, awsClient *Client) (*Client, error) { if !awsClient.IsAnonymous { return NewFromS3Path(s3Path) } region, err := GetBucketRegionFromS3Path(s3Path) if err != nil { return nil, err } return NewAnonymousClientWithRegion(region) } func NewFromS3Path(s3Path string) (*Client, error) { bucket, _, err := SplitS3Path(s3Path) if err != nil { return nil, err } return NewFromS3Bucket(bucket) } func NewFromS3Bucket(bucket string) (*Client, error) { region, err := GetBucketRegion(bucket) if err != nil { return nil, err } return NewForRegion(region) } func NewForRegion(region string) (*Client, error) { sess, err := session.NewSessionWithOptions(session.Options{ Config: aws.Config{ Region: aws.String(region), }, SharedConfigState: session.SharedConfigEnable, }) if err != nil { return nil, errors.WithStack(err) } if sess.Config.Credentials == nil { return nil, ErrorUnableToFindCredentials() } creds, err := sess.Config.Credentials.Get() if err != nil { return nil, ErrorUnableToFindCredentials() } if creds.AccessKeyID == "" || creds.SecretAccessKey == "" { return nil, ErrorUnexpectedMissingCredentials(creds.AccessKeyID, creds.SecretAccessKey) } return &Client{ sess: sess, Region: region, }, nil } func New() (*Client, error) { sess := session.Must(session.NewSessionWithOptions(session.Options{ SharedConfigState: session.SharedConfigEnable, })) if sess.Config.Region == nil { return nil, ErrorRegionNotConfigured() } if sess.Config.Credentials == nil { return nil, ErrorUnableToFindCredentials() } creds, err := sess.Config.Credentials.Get() if err != nil { return nil, ErrorUnableToFindCredentials() } // make sure that credential exists if creds.AccessKeyID == "" || creds.SecretAccessKey == "" { return nil, ErrorUnexpectedMissingCredentials(creds.AccessKeyID, creds.SecretAccessKey) } return &Client{ sess: sess, Region: *sess.Config.Region, }, nil } func NewAnonymousClientWithRegion(region string) (*Client, error) { sess, err := session.NewSession(&aws.Config{ Credentials: credentials.AnonymousCredentials, Region: aws.String(region), }) if err != nil { return nil, err } return &Client{ sess: sess, Region: region, IsAnonymous: true, }, nil } func (c Client) Session() *session.Session { return c.sess } ================================================ FILE: pkg/lib/aws/clients.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package aws import ( "github.com/aws/aws-sdk-go/service/acm" "github.com/aws/aws-sdk-go/service/apigatewayv2" "github.com/aws/aws-sdk-go/service/autoscaling" "github.com/aws/aws-sdk-go/service/cloudformation" "github.com/aws/aws-sdk-go/service/cloudwatch" "github.com/aws/aws-sdk-go/service/cloudwatchlogs" "github.com/aws/aws-sdk-go/service/ec2" "github.com/aws/aws-sdk-go/service/ecr" "github.com/aws/aws-sdk-go/service/eks" "github.com/aws/aws-sdk-go/service/elb" "github.com/aws/aws-sdk-go/service/elbv2" "github.com/aws/aws-sdk-go/service/iam" "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3/s3manager" "github.com/aws/aws-sdk-go/service/servicequotas" "github.com/aws/aws-sdk-go/service/sqs" "github.com/aws/aws-sdk-go/service/sts" ) type clients struct { s3 *s3.S3 s3Uploader *s3manager.Uploader s3Downloader *s3manager.Downloader sts *sts.STS sqs *sqs.SQS ec2 *ec2.EC2 elb *elb.ELB elbv2 *elbv2.ELBV2 eks *eks.EKS ecr *ecr.ECR acm *acm.ACM autoscaling *autoscaling.AutoScaling cloudWatchLogs *cloudwatchlogs.CloudWatchLogs cloudWatch *cloudwatch.CloudWatch apiGatewayV2 *apigatewayv2.ApiGatewayV2 serviceQuotas *servicequotas.ServiceQuotas cloudFormation *cloudformation.CloudFormation iam *iam.IAM } func (c *Client) S3() *s3.S3 { if c.clients.s3 == nil { c.clients.s3 = s3.New(c.sess) } return c.clients.s3 } func (c *Client) S3Uploader() *s3manager.Uploader { if c.clients.s3Uploader == nil { c.clients.s3Uploader = s3manager.NewUploader(c.sess) } return c.clients.s3Uploader } func (c *Client) S3Downloader() *s3manager.Downloader { if c.clients.s3Downloader == nil { c.clients.s3Downloader = s3manager.NewDownloader(c.sess) } return c.clients.s3Downloader } func (c *Client) STS() *sts.STS { if c.clients.sts == nil { c.clients.sts = sts.New(c.sess) } return c.clients.sts } func (c *Client) SQS() *sqs.SQS { if c.clients.sqs == nil { c.clients.sqs = sqs.New(c.sess) } return c.clients.sqs } func (c *Client) EC2() *ec2.EC2 { if c.clients.ec2 == nil { c.clients.ec2 = ec2.New(c.sess) } return c.clients.ec2 } func (c *Client) ELB() *elb.ELB { if c.clients.elb == nil { c.clients.elb = elb.New(c.sess) } return c.clients.elb } func (c *Client) ELBV2() *elbv2.ELBV2 { if c.clients.elbv2 == nil { c.clients.elbv2 = elbv2.New(c.sess) } return c.clients.elbv2 } func (c *Client) EKS() *eks.EKS { if c.clients.eks == nil { c.clients.eks = eks.New(c.sess) } return c.clients.eks } func (c *Client) ECR() *ecr.ECR { if c.clients.ecr == nil { c.clients.ecr = ecr.New(c.sess) } return c.clients.ecr } func (c *Client) CloudFormation() *cloudformation.CloudFormation { if c.clients.cloudFormation == nil { c.clients.cloudFormation = cloudformation.New(c.sess) } return c.clients.cloudFormation } func (c *Client) Autoscaling() *autoscaling.AutoScaling { if c.clients.autoscaling == nil { c.clients.autoscaling = autoscaling.New(c.sess) } return c.clients.autoscaling } func (c *Client) ACM() *acm.ACM { if c.clients.acm == nil { c.clients.acm = acm.New(c.sess) } return c.clients.acm } func (c *Client) CloudWatchLogs() *cloudwatchlogs.CloudWatchLogs { if c.clients.cloudWatchLogs == nil { c.clients.cloudWatchLogs = cloudwatchlogs.New(c.sess) } return c.clients.cloudWatchLogs } func (c *Client) CloudWatch() *cloudwatch.CloudWatch { if c.clients.cloudWatch == nil { c.clients.cloudWatch = cloudwatch.New(c.sess) } return c.clients.cloudWatch } func (c *Client) APIGatewayV2() *apigatewayv2.ApiGatewayV2 { if c.clients.apiGatewayV2 == nil { c.clients.apiGatewayV2 = apigatewayv2.New(c.sess) } return c.clients.apiGatewayV2 } func (c *Client) ServiceQuotas() *servicequotas.ServiceQuotas { if c.clients.serviceQuotas == nil { c.clients.serviceQuotas = servicequotas.New(c.sess) } return c.clients.serviceQuotas } func (c *Client) IAM() *iam.IAM { if c.clients.iam == nil { c.clients.iam = iam.New(c.sess) } return c.clients.iam } ================================================ FILE: pkg/lib/aws/cloudformation.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package aws import ( "github.com/aws/aws-sdk-go/service/cloudformation" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/sets/strset" ) func (c *Client) ListEKSStacks(controlPlaneStackName string, nodeGroupStackNamePrefixes strset.Set) ([]*cloudformation.StackSummary, error) { mostRecentStackRecordsByName := map[string]*cloudformation.StackSummary{} stackSet := strset.Union(nodeGroupStackNamePrefixes, strset.New(controlPlaneStackName)) err := c.CloudFormation().ListStacksPages( &cloudformation.ListStacksInput{}, func(listStackOutput *cloudformation.ListStacksOutput, lastPage bool) bool { for _, stackSummary := range listStackOutput.StackSummaries { if stackSummary == nil || stackSummary.StackName == nil || !stackSet.HasWithPrefix(*stackSummary.StackName) { continue } if _, ok := mostRecentStackRecordsByName[*stackSummary.StackName]; !ok { mostRecentStackRecordsByName[*stackSummary.StackName] = stackSummary } else { created := mostRecentStackRecordsByName[*stackSummary.StackName].CreationTime if created != nil && stackSummary.CreationTime != nil && stackSummary.CreationTime.After(*created) { mostRecentStackRecordsByName[*stackSummary.StackName] = stackSummary } } if *stackSummary.StackName == controlPlaneStackName { return false } } return true }, ) if err != nil { return nil, errors.WithStack(err) } return getStackSummariesFromMap(mostRecentStackRecordsByName), nil } func getStackSummariesFromMap(stackSummaries map[string]*cloudformation.StackSummary) []*cloudformation.StackSummary { var stackSummariesSlice []*cloudformation.StackSummary for _, stack := range stackSummaries { stackSummariesSlice = append(stackSummariesSlice, stack) } return stackSummariesSlice } ================================================ FILE: pkg/lib/aws/cloudwatch.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package aws import ( "encoding/json" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/cloudwatch" "github.com/aws/aws-sdk-go/service/cloudwatchlogs" "github.com/cortexlabs/cortex/pkg/lib/errors" ) var ( _dashboardMinWidthUnits = 1 _dashboardMaxWidthUnits = 24 _dashboardMinHeightUnits = 1 _dashboardMaxHeightUnits = 1000 ) type CloudWatchDashboard struct { Start string `json:"start"` PeriodOverride string `json:"periodOverride"` Widgets []CloudWatchWidget `json:"widgets"` } // Example: // CloudWatchWidget{ // "type":"metric", // "x":0, // "y":0, // "width":12, // "height":6, // "properties":{ // "metrics":[ // [ // "AWS/EC2", // "CPUUtilization", // "InstanceId", // "i-012345" // ] // ], // "period":300, // "stat":"Average", // "region":"us-east-1", // "title":"EC2 Instance CPU" // } // } type CloudWatchWidget struct { Type string `json:"type"` X int `json:"x"` Y int `json:"y"` Width int `json:"width"` Height int `json:"height"` Properties map[string]interface{} `json:"properties"` } type CloudWatchWidgetGrid struct { XOrigin int `json:"x_origin"` YOrigin int `json:"y_origin"` NumColumns int `json:"num_columns"` NumRows int `json:"num_rows"` WidgetHeight int `json:"widget_height"` WidgetWidth int `json:"widget_width"` Widgets []CloudWatchWidget `json:"widgets"` } func (c *Client) DoesLogGroupExist(logGroup string) (bool, error) { _, err := c.CloudWatchLogs().ListTagsLogGroup(&cloudwatchlogs.ListTagsLogGroupInput{ LogGroupName: aws.String(logGroup), }) if err != nil { if IsErrCode(err, "ResourceNotFoundException") { return false, nil } return false, errors.Wrap(err, "log group "+logGroup) } return true, nil } func (c *Client) CreateLogGroup(logGroup string, tags map[string]string) error { _, err := c.CloudWatchLogs().CreateLogGroup(&cloudwatchlogs.CreateLogGroupInput{ LogGroupName: aws.String(logGroup), Tags: aws.StringMap(tags), }) if err != nil { return errors.Wrap(err, "creating log group "+logGroup) } return nil } func (c *Client) DeleteLogGroup(logGroup string) error { _, err := c.CloudWatchLogs().DeleteLogGroup(&cloudwatchlogs.DeleteLogGroupInput{ LogGroupName: aws.String(logGroup), }) if err != nil { return errors.Wrap(err, "log group "+logGroup) } return nil } func (c *Client) TagLogGroup(logGroup string, tagMap map[string]string) error { tags := map[string]*string{} for key, value := range tagMap { tags[key] = aws.String(value) } _, err := c.CloudWatchLogs().TagLogGroup(&cloudwatchlogs.TagLogGroupInput{ LogGroupName: aws.String(logGroup), Tags: tags, }) if err != nil { return errors.Wrap(err, "failed to add tags to log group", logGroup) } return nil } // NewDashboard creates a new dashboard object with title func (c *Client) NewDashboard(title string) *CloudWatchDashboard { return &CloudWatchDashboard{ Start: "-PT1H", PeriodOverride: "inherit", Widgets: []CloudWatchWidget{ TextWidget(7, 0, 10, 1, title), }, } } // GetDashboard gets a dashboard from cloudwatch func (c *Client) GetDashboard(dashboardName string) (*CloudWatchDashboard, error) { dashboardOutput, err := c.CloudWatch().GetDashboard(&cloudwatch.GetDashboardInput{ DashboardName: aws.String(dashboardName), }) if err != nil { return nil, errors.Wrap(err, "failed to get dashboard", dashboardName) } dashboardString := *dashboardOutput.DashboardBody var dashboard CloudWatchDashboard err = json.Unmarshal([]byte(dashboardString), &dashboard) if err != nil { return nil, errors.Wrap(err, "failed to decode cloudwatch body json") } return &dashboard, nil } // GetDashboardOrEmpty gets a dashboard if it exists, or initializes an empty one if not func (c *Client) GetDashboardOrEmpty(dashboardName string, title string) (*CloudWatchDashboard, error) { dashboard, err := c.GetDashboard(dashboardName) if err != nil { if IsErrCode(err, "ResourceNotFound") { dashboard = c.NewDashboard(title) } else { return nil, err } } return dashboard, nil } // CreateDashboard creates a new dashboard (or clears an existing one if it already exists) func (c *Client) CreateDashboard(dashboardName string, title string) error { dashboard := c.NewDashboard(title) err := c.PutDashboard(dashboard, dashboardName) if err != nil { return err } return nil } // PutDashboard updates a dashboard (or creates it if it doesn't exit) func (c *Client) PutDashboard(dashboard *CloudWatchDashboard, dashboardName string) error { dashboardJSON, err := json.Marshal(dashboard) if err != nil { return errors.Wrap(err, "failed to encode cloudwatch body into json") } _, err = c.CloudWatch().PutDashboard(&cloudwatch.PutDashboardInput{ DashboardName: aws.String(dashboardName), DashboardBody: aws.String(string(dashboardJSON)), }) if err != nil { return errors.Wrap(err, "failed to put dashboard", dashboardName) } return nil } // DeleteDashboard deletes a dashboard func (c *Client) DeleteDashboard(dashboardName string) error { _, err := c.CloudWatch().DeleteDashboards(&cloudwatch.DeleteDashboardsInput{ DashboardNames: []*string{aws.String(dashboardName)}, }) if err != nil { return errors.Wrap(err, "failed to delete dashboard", dashboardName) } return nil } // DoesDashboardExist checks if a dashboard exists func (c *Client) DoesDashboardExist(dashboardName string) (bool, error) { _, err := c.CloudWatch().GetDashboard(&cloudwatch.GetDashboardInput{ DashboardName: aws.String(dashboardName), }) if err != nil { if IsErrCode(err, "ResourceNotFound") { return false, nil } return false, errors.Wrap(err, "dashboard", dashboardName) } return true, nil } // TextWidget creates new text widget // Example: // title_widget = { // "type": "text", // "x": x, // "y": y, // "width": wewidthi, // "height": height, // "properties": {"markdown": markdown}, // } func TextWidget(x int, y int, width int, height int, markdown string) CloudWatchWidget { return CloudWatchWidget{Type: "text", X: x, Y: y, Width: width, Height: height, Properties: map[string]interface{}{"markdown": markdown}} } // NewHorizontalGrid sets a CloudWatch Dashboard grid to be filled from left to right, row by row func NewHorizontalGrid(xOrigin, yOrigin, widgetHeight, widgetWidth, numColumns int) (*CloudWatchWidgetGrid, error) { if widgetHeight < 1 || widgetHeight > _dashboardMaxHeightUnits { return &CloudWatchWidgetGrid{}, ErrorDashboardHeightOutOfRange(widgetHeight) } if widgetWidth < 1 || widgetWidth > _dashboardMaxWidthUnits { return &CloudWatchWidgetGrid{}, ErrorDashboardWidthOutOfRange(widgetWidth) } if xOrigin+numColumns*widgetWidth > _dashboardMaxWidthUnits { return &CloudWatchWidgetGrid{}, ErrorDashboardWidthOutOfRange(xOrigin + numColumns*widgetWidth) } return &CloudWatchWidgetGrid{ XOrigin: xOrigin, YOrigin: yOrigin, WidgetHeight: widgetHeight, WidgetWidth: widgetWidth, NumColumns: numColumns, Widgets: make([]CloudWatchWidget, 0), }, nil } // NewVerticalGrid sets a CloudWatch Dashboard grid to be filled from top to bottom, column by column func NewVerticalGrid(xOrigin, yOrigin, widgetHeight, widgetWidth, numRows int) (*CloudWatchWidgetGrid, error) { if widgetHeight < 1 || widgetHeight > _dashboardMaxHeightUnits { return &CloudWatchWidgetGrid{}, ErrorDashboardHeightOutOfRange(widgetHeight) } if widgetWidth < 1 || widgetWidth > _dashboardMaxWidthUnits { return &CloudWatchWidgetGrid{}, ErrorDashboardWidthOutOfRange(widgetWidth) } if yOrigin+numRows*widgetHeight > _dashboardMaxHeightUnits { return &CloudWatchWidgetGrid{}, ErrorDashboardHeightOutOfRange(yOrigin + numRows*widgetHeight) } return &CloudWatchWidgetGrid{ XOrigin: xOrigin, YOrigin: yOrigin, WidgetHeight: widgetHeight, WidgetWidth: widgetWidth, NumRows: numRows, Widgets: make([]CloudWatchWidget, 0), }, nil } // AddWidget adds a widget to the configured grid func (grid *CloudWatchWidgetGrid) AddWidget( metric []interface{}, title string, stat string, period int, region string, ) error { var currentColumn, currentRow int if grid.NumColumns > 0 { currentRow = len(grid.Widgets) / grid.NumColumns currentColumn = len(grid.Widgets) - currentRow*grid.NumColumns } if grid.NumRows > 0 { currentColumn = len(grid.Widgets) / grid.NumRows currentRow = len(grid.Widgets) - currentColumn*grid.NumRows } x := grid.XOrigin + currentColumn*grid.WidgetWidth y := grid.YOrigin + currentRow*grid.WidgetHeight if x+grid.WidgetWidth > _dashboardMaxWidthUnits { return ErrorDashboardWidthOutOfRange(x + grid.WidgetWidth) } if y+grid.WidgetHeight > _dashboardMaxHeightUnits { return ErrorDashboardHeightOutOfRange(y + grid.WidgetHeight) } grid.Widgets = append(grid.Widgets, CloudWatchWidget{ Type: "metric", X: x, Y: y, Width: grid.WidgetWidth, Height: grid.WidgetHeight, Properties: map[string]interface{}{ "metrics": metric, "title": title, "stat": stat, "period": period, "region": region, "view": "timeSeries", }, }, ) return nil } // HighestY returns the largest Y coordinate of a widget on the dashboard (i.e. the lowest widget) func HighestY(dashboard *CloudWatchDashboard) (int, error) { highestY := 0 for _, wid := range dashboard.Widgets { if highestY < wid.Y { highestY = wid.Y } } return highestY, nil } ================================================ FILE: pkg/lib/aws/credentials.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package aws // access key ID may be unavailable depending on how the client was instantiated func (c *Client) AccessKeyID() *string { if c.sess.Config.Credentials == nil { return nil } sessCreds, err := c.sess.Config.Credentials.Get() if err != nil { return nil } if sessCreds.AccessKeyID == "" { return nil } return &sessCreds.AccessKeyID } func (c *Client) SecretAccessKey() *string { if c.sess.Config.Credentials == nil { return nil } sessCreds, err := c.sess.Config.Credentials.Get() if err != nil { return nil } if sessCreds.SecretAccessKey == "" { return nil } return &sessCreds.SecretAccessKey } func (c *Client) SessionToken() *string { if c.sess.Config.Credentials == nil { return nil } sessCreds, err := c.sess.Config.Credentials.Get() if err != nil { return nil } if sessCreds.SessionToken == "" { return nil } return &sessCreds.SessionToken } ================================================ FILE: pkg/lib/aws/ec2.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package aws import ( "math" "regexp" "strings" "time" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/parallel" "github.com/cortexlabs/cortex/pkg/lib/sets/strset" s "github.com/cortexlabs/cortex/pkg/lib/strings" ) var ( _digitsRegex = regexp.MustCompile(`[0-9]+`) _gpuInstanceFamilies = strset.New("g", "p") ) type ParsedInstanceType struct { Family string Generation int Capabilities strset.Set Size string } // Checks weather the input is an AWS instance type func IsValidInstanceType(instanceType string) bool { return AllInstanceTypes.Has(instanceType) } // Checks whether the input is an AWS instance type func CheckValidInstanceType(instanceType string) error { if !IsValidInstanceType(instanceType) { return ErrorInvalidInstanceType(instanceType) } return nil } // AWS instance types take the form of: [family][generation][capabilities].[size] // the first group is the instance family, e.g. "m", "t", "g", "inf", ... // the second group is a generation number for that series, e.g. 3, 4, ... // the third group is optional, and is a set of single-character capabilities // "g" represents ARM (graviton), "a" for AMD, "n" for fast networking, "d" for fast storage, etc. // the fourth and final group (after the dot) is the instance size, e.g. "large" func ParseInstanceType(instanceType string) (ParsedInstanceType, error) { if err := CheckValidInstanceType(instanceType); err != nil { return ParsedInstanceType{}, err } parts := strings.Split(instanceType, ".") if len(parts) != 2 { return ParsedInstanceType{}, errors.ErrorUnexpected("unexpected invalid instance type: " + instanceType) } prefix := parts[0] size := parts[1] digitSets := _digitsRegex.FindAllString(prefix, -1) if len(digitSets) == 0 { return ParsedInstanceType{}, errors.ErrorUnexpected("unexpected invalid instance type: " + instanceType) } prefixParts := _digitsRegex.Split(prefix, -1) capabilitiesStr := prefixParts[len(prefixParts)-1] capabilities := strset.FromSlice(strings.Split(capabilitiesStr, "")) generationStr := digitSets[len(digitSets)-1] generation, ok := s.ParseInt(generationStr) if !ok { return ParsedInstanceType{}, errors.ErrorUnexpected("unexpected invalid instance type: " + instanceType) } generationIndex := strings.LastIndex(prefix, generationStr) if generationIndex == -1 { return ParsedInstanceType{}, errors.ErrorUnexpected("unexpected invalid instance type: " + instanceType) } family := prefix[:generationIndex] return ParsedInstanceType{ Family: family, Generation: generation, Capabilities: capabilities, Size: size, }, nil } func IsARMInstance(instanceType string) (bool, error) { parsedType, err := ParseInstanceType(instanceType) if err != nil { return false, err } if parsedType.Family == "a" { return true, nil } if parsedType.Capabilities.Has("g") { return true, nil } return false, nil } func IsAMDGPUInstance(instanceType string) (bool, error) { parsedType, err := ParseInstanceType(instanceType) if err != nil { return false, err } if !_gpuInstanceFamilies.Has(parsedType.Family) { return false, nil } if parsedType.Capabilities.Has("a") { return true, nil } return false, nil } func IsNvidiaGPUInstance(instanceType string) (bool, error) { parsedType, err := ParseInstanceType(instanceType) if err != nil { return false, err } if !_gpuInstanceFamilies.Has(parsedType.Family) { return false, nil } if !parsedType.Capabilities.Has("a") { return true, nil } return false, nil } func IsGPUInstance(instanceType string) (bool, error) { isAMDGPU, err := IsAMDGPUInstance(instanceType) if err != nil { return false, err } isNvidiaGPU, err := IsNvidiaGPUInstance(instanceType) if err != nil { return false, err } return isAMDGPU || isNvidiaGPU, nil } func IsInferentiaInstance(instanceType string) (bool, error) { parsedType, err := ParseInstanceType(instanceType) if err != nil { return false, err } return parsedType.Family == "inf", nil } func IsTrainiumInstance(instanceType string) (bool, error) { parsedType, err := ParseInstanceType(instanceType) if err != nil { return false, err } return parsedType.Family == "trn", nil } func IsMacInstance(instanceType string) (bool, error) { parsedType, err := ParseInstanceType(instanceType) if err != nil { return false, err } if parsedType.Family == "mac" { return true, nil } return false, nil } func IsFPGAInstance(instanceType string) (bool, error) { parsedType, err := ParseInstanceType(instanceType) if err != nil { return false, err } if parsedType.Family == "f" { return true, nil } return false, nil } func IsAlevoInstance(instanceType string) (bool, error) { parsedType, err := ParseInstanceType(instanceType) if err != nil { return false, err } if parsedType.Family == "vt" { return true, nil } return false, nil } func IsGaudiInstance(instanceType string) (bool, error) { parsedType, err := ParseInstanceType(instanceType) if err != nil { return false, err } if parsedType.Family == "dl" { return true, nil } return false, nil } func (c *Client) SpotInstancePrice(instanceType string) (float64, error) { result, err := c.EC2().DescribeSpotPriceHistory(&ec2.DescribeSpotPriceHistoryInput{ InstanceTypes: []*string{aws.String(instanceType)}, ProductDescriptions: []*string{aws.String("Linux/UNIX")}, StartTime: aws.Time(time.Now()), }) if err != nil { return 0, errors.Wrap(err, "checking spot instance price") } min := math.MaxFloat64 for _, spotPrice := range result.SpotPriceHistory { if spotPrice == nil { continue } price, ok := s.ParseFloat64(*spotPrice.SpotPrice) if !ok { continue } if price < min { min = price } } if min == math.MaxFloat64 { return 0, ErrorNoValidSpotPrices(instanceType, c.Region) } if min <= 0 { return 0, ErrorNoValidSpotPrices(instanceType, c.Region) } return min, nil } func (c *Client) ListAllRegions() (strset.Set, error) { result, err := c.EC2().DescribeRegions(&ec2.DescribeRegionsInput{ AllRegions: aws.Bool(true), }) if err != nil { return nil, errors.WithStack(err) } regions := strset.New() for _, region := range result.Regions { if region.RegionName != nil { regions.Add(*region.RegionName) } } return regions, nil } // Returns only regions that are enabled for your account func (c *Client) ListEnabledRegions() (strset.Set, error) { result, err := c.EC2().DescribeRegions(&ec2.DescribeRegionsInput{ AllRegions: aws.Bool(false), }) if err != nil { return nil, errors.WithStack(err) } regions := strset.New() for _, region := range result.Regions { if region.RegionName != nil { regions.Add(*region.RegionName) } } return regions, nil } // Returns all regions and enabled regions func (c *Client) ListRegions() (strset.Set, strset.Set, error) { var allRegions strset.Set var enabledRegions strset.Set err := parallel.RunFirstErr( func() error { var err error allRegions, err = c.ListAllRegions() return err }, func() error { var err error enabledRegions, err = c.ListEnabledRegions() return err }, ) if err != nil { return nil, nil, err } return allRegions, enabledRegions, nil } func (c *Client) ListAvailabilityZonesInRegion() (strset.Set, error) { input := &ec2.DescribeAvailabilityZonesInput{ Filters: []*ec2.Filter{ { Name: aws.String("region-name"), Values: []*string{aws.String(c.Region)}, }, { Name: aws.String("state"), Values: []*string{aws.String(ec2.AvailabilityZoneStateAvailable)}, }, }, } result, err := c.EC2().DescribeAvailabilityZones(input) if err != nil { return nil, errors.WithStack(err) } zones := strset.New() for _, az := range result.AvailabilityZones { if az.ZoneName != nil { zones.Add(*az.ZoneName) } } return zones, nil } func (c *Client) listSupportedAvailabilityZonesSingle(instanceType string) (strset.Set, error) { input := &ec2.DescribeReservedInstancesOfferingsInput{ InstanceType: &instanceType, IncludeMarketplace: aws.Bool(false), Filters: []*ec2.Filter{ { Name: aws.String("scope"), Values: []*string{aws.String(ec2.ScopeAvailabilityZone)}, }, }, } zones := strset.New() err := c.EC2().DescribeReservedInstancesOfferingsPages(input, func(output *ec2.DescribeReservedInstancesOfferingsOutput, lastPage bool) bool { for _, offering := range output.ReservedInstancesOfferings { if offering.AvailabilityZone != nil { zones.Add(*offering.AvailabilityZone) } } return true }) if err != nil { return nil, errors.WithStack(err) } return zones, nil } func (c *Client) ListSupportedAvailabilityZones(instanceType string, instanceTypes ...string) (strset.Set, error) { allInstanceTypes := append(instanceTypes, instanceType) zoneSets := make([]strset.Set, len(allInstanceTypes)) fns := make([]func() error, len(allInstanceTypes)) for i := range allInstanceTypes { localIdx := i fns[i] = func() error { zones, err := c.listSupportedAvailabilityZonesSingle(allInstanceTypes[localIdx]) if err != nil { return err } zoneSets[localIdx] = zones return nil } } err := parallel.RunFirstErr(fns[0], fns[1:]...) if err != nil { return nil, err } return strset.Intersection(zoneSets...), nil } func (c *Client) ListElasticIPs() ([]string, error) { addresses, err := c.EC2().DescribeAddresses(&ec2.DescribeAddressesInput{}) if err != nil { return nil, errors.WithStack(err) } addressesList := []string{} if addresses != nil { for _, address := range addresses.Addresses { if address != nil && address.PublicIp != nil { addressesList = append(addressesList, *address.PublicIp) } } } return addressesList, nil } func (c *Client) ListInternetGateways() ([]string, error) { gatewaysList := []string{} err := c.EC2().DescribeInternetGatewaysPages(&ec2.DescribeInternetGatewaysInput{}, func(output *ec2.DescribeInternetGatewaysOutput, lastPage bool) bool { if output == nil { return false } for _, gateway := range output.InternetGateways { if gateway != nil && gateway.InternetGatewayId != nil { gatewaysList = append(gatewaysList, *gateway.InternetGatewayId) } } return true }) if err != nil { return nil, errors.WithStack(err) } return gatewaysList, nil } func (c *Client) DescribeNATGateways() ([]ec2.NatGateway, error) { var gateways []ec2.NatGateway err := c.EC2().DescribeNatGatewaysPages(&ec2.DescribeNatGatewaysInput{}, func(output *ec2.DescribeNatGatewaysOutput, lastPage bool) bool { if output == nil { return false } for _, gateway := range output.NatGateways { if gateway == nil { continue } gateways = append(gateways, *gateway) } return true }) if err != nil { return nil, errors.WithStack(err) } return gateways, nil } func (c *Client) DescribeSubnets() ([]ec2.Subnet, error) { var subnets []ec2.Subnet err := c.EC2().DescribeSubnetsPages(&ec2.DescribeSubnetsInput{}, func(output *ec2.DescribeSubnetsOutput, lastPage bool) bool { if output == nil { return false } for _, subnet := range output.Subnets { if subnet == nil { continue } subnets = append(subnets, *subnet) } return true }) if err != nil { return nil, errors.WithStack(err) } return subnets, nil } func (c *Client) DescribeVpcs() ([]ec2.Vpc, error) { var vpcs []ec2.Vpc err := c.EC2().DescribeVpcsPages(&ec2.DescribeVpcsInput{}, func(output *ec2.DescribeVpcsOutput, lastPage bool) bool { if output == nil { return false } for _, vpc := range output.Vpcs { if vpc == nil { continue } vpcs = append(vpcs, *vpc) } return true }) if err != nil { return nil, errors.WithStack(err) } return vpcs, nil } func (c *Client) DescribeSecurityGroups() ([]ec2.SecurityGroup, error) { var sgs []ec2.SecurityGroup err := c.EC2().DescribeSecurityGroupsPages(&ec2.DescribeSecurityGroupsInput{}, func(output *ec2.DescribeSecurityGroupsOutput, lastPage bool) bool { if output == nil { return false } for _, sg := range output.SecurityGroups { if sg == nil { continue } sgs = append(sgs, *sg) } return true }) if err != nil { return nil, errors.WithStack(err) } return sgs, nil } func (c *Client) ListVolumes(tags ...ec2.Tag) ([]ec2.Volume, error) { var volumes []ec2.Volume err := c.EC2().DescribeVolumesPages(&ec2.DescribeVolumesInput{}, func(output *ec2.DescribeVolumesOutput, lastPage bool) bool { if output == nil { return false } for _, volume := range output.Volumes { if volume == nil { continue } if hasAllEC2Tags(tags, volume.Tags) { volumes = append(volumes, *volume) } } return true }) if err != nil { return nil, errors.WithStack(err) } return volumes, nil } func (c *Client) DeleteVolume(volumeID string) error { _, err := c.EC2().DeleteVolume(&ec2.DeleteVolumeInput{ VolumeId: aws.String(volumeID), }) if err != nil { return errors.Wrap(err) } return nil } func hasAllEC2Tags(queryTags []ec2.Tag, allResourceTags []*ec2.Tag) bool { for _, queryTag := range queryTags { if !hasEC2Tag(queryTag, allResourceTags) { return false } } return true } // if queryTag's value is nil, the tag will match as long as the key is present in the resource's tags func hasEC2Tag(queryTag ec2.Tag, allResourceTags []*ec2.Tag) bool { for _, resourceTag := range allResourceTags { if queryTag.Key != nil && resourceTag.Key != nil && *queryTag.Key == *resourceTag.Key { if queryTag.Value == nil { return true } if queryTag.Value != nil && resourceTag.Value != nil && *queryTag.Value == *resourceTag.Value { return true } } } return false } ================================================ FILE: pkg/lib/aws/ec2_test.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package aws import ( "fmt" "testing" "github.com/cortexlabs/cortex/pkg/lib/sets/strset" "github.com/stretchr/testify/require" ) func TestParseInstanceType(t *testing.T) { var testcases = []struct { instanceType string expected ParsedInstanceType }{ {"t3.small", ParsedInstanceType{"t", 3, strset.New(), "small"}}, {"g4dn.xlarge", ParsedInstanceType{"g", 4, strset.New("d", "n"), "xlarge"}}, {"inf1.24xlarge", ParsedInstanceType{"inf", 1, strset.New(), "24xlarge"}}, {"u-9tb1.metal", ParsedInstanceType{"u-9tb", 1, strset.New(), "metal"}}, } invalidTypes := []string{ "badtype", "badtype.large", "badtype1.large", "badtype2ad.large", } for _, testcase := range testcases { parsed, err := ParseInstanceType(testcase.instanceType) require.NoError(t, err) require.Equal(t, testcase.expected.Family, parsed.Family, fmt.Sprintf("unexpected family for input: %s", testcase.instanceType)) require.Equal(t, testcase.expected.Generation, parsed.Generation, fmt.Sprintf("unexpected generation for input: %s", testcase.instanceType)) require.ElementsMatch(t, testcase.expected.Capabilities.Slice(), parsed.Capabilities.Slice(), fmt.Sprintf("unexpected capabilities for input: %s", testcase.instanceType)) require.Equal(t, testcase.expected.Size, parsed.Size, fmt.Sprintf("unexpected size for input: %s", testcase.instanceType)) } for _, instanceType := range invalidTypes { _, err := ParseInstanceType(instanceType) require.Error(t, err) } for instanceType := range AllInstanceTypes { _, err := ParseInstanceType(instanceType) require.NoError(t, err) } } ================================================ FILE: pkg/lib/aws/ecr.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package aws import ( "encoding/base64" "regexp" "strings" "github.com/aws/aws-sdk-go/service/ecr" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/regex" ) type ECRAuthConfig struct { Username string AccessToken string ProxyEndpoint string } var _ecrRegionRegex = regexp.MustCompile(`ecr\.(\S+)\.amazon`) func (c *Client) GetECRAuthToken() (*ecr.GetAuthorizationTokenOutput, error) { result, err := c.ECR().GetAuthorizationToken(&ecr.GetAuthorizationTokenInput{}) if err != nil { return result, errors.Wrap(err, "failed to retrieve ECR auth token") } return result, nil } func (c *Client) GetECRAuthConfig() (ECRAuthConfig, error) { tokenOutput, err := c.GetECRAuthToken() if err != nil { return ECRAuthConfig{}, err } if len(tokenOutput.AuthorizationData) == 0 { return ECRAuthConfig{}, ErrorECRExtractingCredentials() } authData := tokenOutput.AuthorizationData[0] credentials, err := base64.URLEncoding.DecodeString(*authData.AuthorizationToken) if err != nil { return ECRAuthConfig{}, errors.Wrap(err, ErrorECRExtractingCredentials().Error()) } credentialsString := string(credentials) splitCredentials := strings.Split(credentialsString, ":") if len(splitCredentials) != 2 { return ECRAuthConfig{}, ErrorECRExtractingCredentials() } return ECRAuthConfig{ Username: splitCredentials[0], AccessToken: splitCredentials[1], ProxyEndpoint: *authData.ProxyEndpoint, }, nil } func GetAccountIDFromECRURL(path string) string { if regex.IsValidECRURL(path) { return strings.Split(path, ".")[0] } return "" } func GetRegionFromECRURL(path string) string { res := _ecrRegionRegex.FindStringSubmatch(path) if len(res) != 2 { return "" } return res[1] } ================================================ FILE: pkg/lib/aws/eks.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package aws import ( "github.com/aws/aws-sdk-go/service/eks" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/sets/strset" ) var EKSSupportedRegions strset.Set func init() { EKSSupportedRegions = strset.New() for region := range InstanceMetadatas { EKSSupportedRegions.Add(region) } } // Returns info for the cluster, or nil of no cluster exists with the provided name func (c *Client) EKSClusterOrNil(clusterName string) (*eks.Cluster, error) { clusterInfo, err := c.EKS().DescribeCluster(&eks.DescribeClusterInput{Name: &clusterName}) if err != nil { if IsErrCode(err, eks.ErrCodeResourceNotFoundException) { return nil, nil } return nil, errors.WithStack(err) } return clusterInfo.Cluster, nil } ================================================ FILE: pkg/lib/aws/elb.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package aws import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/elb" "github.com/cortexlabs/cortex/pkg/lib/errors" ) type ClassicLoadBalancerState string const ( LoadBalancerStateInService ClassicLoadBalancerState = "InService" LoadBalancerStateOutOfService ClassicLoadBalancerState = "OutOfService" LoadBalancerStateUnknown ClassicLoadBalancerState = "Unknown" ) func (state ClassicLoadBalancerState) String() string { return string(state) } // returns the first classic load balancer which has all of the specified tags, or nil if no load balancers match func (c *Client) FindLoadBalancer(tags map[string]string) (*elb.LoadBalancerDescription, error) { var loadBalancer *elb.LoadBalancerDescription var fnErr error params := elb.DescribeLoadBalancersInput{ PageSize: aws.Int64(20), // 20 is the limit for DescribeTags() } err := c.ELB().DescribeLoadBalancersPages(¶ms, func(page *elb.DescribeLoadBalancersOutput, lastPage bool) bool { loadBalancerNames := make([]string, len(page.LoadBalancerDescriptions)) loadBalancers := make(map[string]*elb.LoadBalancerDescription) for i := range page.LoadBalancerDescriptions { loadBalancerName := *page.LoadBalancerDescriptions[i].LoadBalancerName loadBalancerNames[i] = loadBalancerName loadBalancers[loadBalancerName] = page.LoadBalancerDescriptions[i] } tagsOutput, err := c.ELB().DescribeTags(&elb.DescribeTagsInput{ LoadBalancerNames: aws.StringSlice(loadBalancerNames), }) if err != nil { fnErr = errors.WithStack(err) return false } for _, tagDescription := range tagsOutput.TagDescriptions { lbTags := make(map[string]string, len(tagDescription.Tags)) for _, lbTag := range tagDescription.Tags { if lbTag.Key != nil && lbTag.Value != nil { lbTags[*lbTag.Key] = *lbTag.Value } } missingTag := false for key, value := range tags { if lbTags[key] != value { missingTag = true break } } if !missingTag { loadBalancer = loadBalancers[*tagDescription.LoadBalancerName] return false } } return true }) if err != nil { return nil, errors.WithStack(err) } if fnErr != nil { return nil, fnErr } return loadBalancer, nil } func (c *Client) IsLoadBalancerHealthy(loadBalancerName string) (bool, error) { instanceHealthOutput, err := c.ELB().DescribeInstanceHealth(&elb.DescribeInstanceHealthInput{ LoadBalancerName: &loadBalancerName, }) if err != nil { return false, errors.WithStack(err) } for _, instance := range instanceHealthOutput.InstanceStates { if instance == nil { continue } if instance.State != nil && *instance.State != LoadBalancerStateInService.String() { return false, nil } } return true, nil } ================================================ FILE: pkg/lib/aws/elbv2.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package aws import ( "strings" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/elbv2" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/sets/strset" ) // https://docs.aws.amazon.com/elasticloadbalancing/latest/network/load-balancer-target-groups.html var _nlbUnsupportedInstancePrefixes = strset.New("c1", "cc1", "cc2", "cg1", "cg2", "cr1", "g1", "g2", "hi1", "hs1", "m1", "m2", "m3", "t1") func IsInstanceSupportedByNLB(instanceType string) (bool, error) { if err := CheckValidInstanceType(instanceType); err != nil { return false, err } for prefix := range _nlbUnsupportedInstancePrefixes { if strings.HasPrefix(instanceType, prefix) { return false, nil } } return true, nil } // returns the first network/application load balancer which has all of the specified tags, or nil if no load balancers match func (c *Client) FindLoadBalancerV2(tags map[string]string) (*elbv2.LoadBalancer, error) { var loadBalancer *elbv2.LoadBalancer var fnErr error params := elbv2.DescribeLoadBalancersInput{ PageSize: aws.Int64(20), // 20 is the limit for DescribeTags() } err := c.ELBV2().DescribeLoadBalancersPages(¶ms, func(page *elbv2.DescribeLoadBalancersOutput, lastPage bool) bool { arns := make([]string, len(page.LoadBalancers)) loadBalancers := make(map[string]*elbv2.LoadBalancer) for i := range page.LoadBalancers { arn := *page.LoadBalancers[i].LoadBalancerArn arns[i] = arn loadBalancers[arn] = page.LoadBalancers[i] } tagsOutput, err := c.ELBV2().DescribeTags(&elbv2.DescribeTagsInput{ ResourceArns: aws.StringSlice(arns), }) if err != nil { fnErr = errors.WithStack(err) return false } for _, tagDescription := range tagsOutput.TagDescriptions { lbTags := make(map[string]string, len(tagDescription.Tags)) for _, lbTag := range tagDescription.Tags { if lbTag.Key != nil && lbTag.Value != nil { lbTags[*lbTag.Key] = *lbTag.Value } } missingTag := false for key, value := range tags { if lbTags[key] != value { missingTag = true break } } if !missingTag { loadBalancer = loadBalancers[*tagDescription.ResourceArn] return false } } return true }) if err != nil { return nil, errors.WithStack(err) } if fnErr != nil { return nil, fnErr } return loadBalancer, nil } func IsLoadBalancerV2Healthy(loadBalancer elbv2.LoadBalancer) bool { if loadBalancer.State == nil || loadBalancer.State.Code == nil { return false } return *loadBalancer.State.Code == elbv2.LoadBalancerStateEnumActive } ================================================ FILE: pkg/lib/aws/errors.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package aws import ( "fmt" "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/service/iam" "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/sqs" "github.com/cortexlabs/cortex/pkg/lib/errors" s "github.com/cortexlabs/cortex/pkg/lib/strings" ) const ( ErrInvalidInstanceType = "aws.invalid_instance_type" ErrInvalidAWSCredentials = "aws.invalid_aws_credentials" ErrInvalidS3aPath = "aws.invalid_s3a_path" ErrInvalidS3Path = "aws.invalid_s3_path" ErrUnexpectedMissingCredentials = "aws.unexpected_missing_credentials" ErrAuth = "aws.auth" ErrBucketInaccessible = "aws.bucket_inaccessible" ErrBucketNotFound = "aws.bucket_not_found" ErrInsufficientInstanceQuota = "aws.insufficient_instance_quota" ErrNoValidSpotPrices = "aws.no_valid_spot_prices" ErrECRExtractingCredentials = "aws.ecr_failed_credentials" ErrDashboardWidthOutOfRange = "aws.dashboard_width_ouf_of_range" ErrDashboardHeightOutOfRange = "aws.dashboard_height_out_of_range" ErrRegionNotConfigured = "aws.region_not_configured" ErrUnableToFindCredentials = "aws.unable_to_find_credentials" ErrNATGatewayLimitExceeded = "aws.nat_gateway_limit_exceeded" ErrEIPLimitExceeded = "aws.eip_limit_exceeded" ErrInternetGatewayLimitExceeded = "aws.internet_gateway_limit_exceeded" ErrVPCLimitExceeded = "aws.vpc_limit_exceeded" ErrSecurityGroupRulesExceeded = "aws.security_group_rules_exceeded" ErrSecurityGroupLimitExceeded = "aws.security_group_limit_exceeded" ) func IsAWSError(err error) bool { if _, ok := errors.CauseOrSelf(err).(awserr.Error); ok { return true } return false } func IsNotFoundErr(err error) bool { return IsErrCode(err, "NotFound") } func IsNoSuchKeyErr(err error) bool { return IsErrCode(err, s3.ErrCodeNoSuchKey) } func IsNoSuchEntityErr(err error) bool { return IsErrCode(err, iam.ErrCodeNoSuchEntityException) } func IsNoSuchBucketErr(err error) bool { return IsErrCode(err, s3.ErrCodeNoSuchBucket) } func IsNonExistentQueueErr(err error) bool { return IsErrCode(err, sqs.ErrCodeQueueDoesNotExist) } func IsGenericNotFoundErr(err error) bool { return IsNotFoundErr(err) || IsNoSuchKeyErr(err) || IsNoSuchBucketErr(err) } func IsErrCode(err error, errorCode string) bool { awsErr, ok := errors.CauseOrSelf(err).(awserr.Error) if !ok { return false } if awsErr.Code() == errorCode { return true } return false } func ErrorInvalidInstanceType(instanceType string) error { return errors.WithStack(&errors.Error{ Kind: ErrInvalidInstanceType, Message: fmt.Sprintf("%s is not an AWS instance type (e.g. m5.large is a valid instance type)", s.UserStr(instanceType)), }) } func ErrorInvalidAWSCredentials(awsErr error) error { awsErrMsg := errors.Message(awsErr) return errors.WithStack(&errors.Error{ Kind: ErrInvalidAWSCredentials, Message: "invalid AWS credentials\n" + awsErrMsg, Cause: awsErr, }) } func ErrorInvalidS3aPath(provided string) error { return errors.WithStack(&errors.Error{ Kind: ErrInvalidS3aPath, Message: fmt.Sprintf("%s is not a valid s3a path (e.g. s3a://cortex-examples/pytorch/iris-classifier/weights.pth is a valid s3a path)", s.UserStr(provided)), }) } func ErrorInvalidS3Path(provided string) error { return errors.WithStack(&errors.Error{ Kind: ErrInvalidS3Path, Message: fmt.Sprintf("%s is not a valid s3 path (e.g. s3://cortex-examples/pytorch/iris-classifier/weights.pth is a valid s3 path)", s.UserStr(provided)), }) } func ErrorUnexpectedMissingCredentials(awsAccessKeyID string, awsSecretAccessKey string) error { var msg string if awsAccessKeyID == "" && awsSecretAccessKey == "" { msg = "aws access key id and aws secret access key are missing" } else if awsAccessKeyID == "" { msg = "aws access key id is missing" } else if awsSecretAccessKey == "" { msg = "aws secret access key is missing" } return errors.WithStack(&errors.Error{ Kind: ErrUnexpectedMissingCredentials, Message: msg, }) } func ErrorAuth() error { return errors.WithStack(&errors.Error{ Kind: ErrAuth, Message: "unable to authenticate with AWS", }) } func ErrorBucketInaccessible(bucket string) error { return errors.WithStack(&errors.Error{ Kind: ErrBucketInaccessible, Message: fmt.Sprintf("bucket \"%s\" is not accessible with the specified AWS credentials", bucket), }) } func ErrorBucketNotFound(bucket string) error { return errors.WithStack(&errors.Error{ Kind: ErrBucketNotFound, Message: fmt.Sprintf("bucket \"%s\" not found", bucket), }) } func ErrorInsufficientInstanceQuota(instanceTypes []string, lifecycle string, region string, requiredVCPUs int64, vCPUQuota int64, quotaCode string) error { url := fmt.Sprintf("https://%s.console.aws.amazon.com/servicequotas/home?region=%s#!/services/ec2/quotas/%s", region, region, quotaCode) andInstanceTypes := s.StrsAnd(instanceTypes) return errors.WithStack(&errors.Error{ Kind: ErrInsufficientInstanceQuota, Message: fmt.Sprintf("your cluster may require up to %d vCPU %s %s instances, but your AWS quota for %s %s instances in %s is only %d vCPU; please reduce the maximum number of %s %s instances your cluster may use (e.g. by changing max_instances and/or spot_config if applicable), or request a quota increase to at least %d vCPU here: %s (if your request was recently approved, please allow ~30 minutes for AWS to reflect this change)", requiredVCPUs, lifecycle, andInstanceTypes, lifecycle, andInstanceTypes, region, vCPUQuota, lifecycle, andInstanceTypes, requiredVCPUs, url), }) } func ErrorNoValidSpotPrices(instanceType string, region string) error { return errors.WithStack(&errors.Error{ Kind: ErrNoValidSpotPrices, Message: fmt.Sprintf("no spot prices were found for %s instances in %s", instanceType, region), }) } func ErrorECRExtractingCredentials() error { return errors.WithStack(&errors.Error{ Kind: ErrECRExtractingCredentials, Message: "unable to extract ECR credentials", }) } func ErrorDashboardWidthOutOfRange(width int) error { return errors.WithStack(&errors.Error{ Kind: ErrDashboardWidthOutOfRange, Message: fmt.Sprintf("dashboard width %d out of range; width must be between %d and %d", width, _dashboardMinWidthUnits, _dashboardMaxWidthUnits), }) } func ErrorDashboardHeightOutOfRange(height int) error { return errors.WithStack(&errors.Error{ Kind: ErrDashboardHeightOutOfRange, Message: fmt.Sprintf("dashboard height %d out of range; height must be between %d and %d", height, _dashboardMinHeightUnits, _dashboardMaxHeightUnits), }) } func ErrorRegionNotConfigured() error { return errors.WithStack(&errors.Error{ Kind: ErrRegionNotConfigured, Message: "aws region has not been configured; please set a default region (e.g. `export AWS_DEFAULT_REGION=us-west-2`)", }) } func ErrorUnableToFindCredentials() error { return errors.WithStack(&errors.Error{ Kind: ErrUnableToFindCredentials, Message: "unable to find aws credentials; instructions about configuring aws credentials can be found at https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-quickstart.html", }) } func ErrorNATGatewayLimitExceeded(currentLimit, additionalQuotaRequired int, availabilityZones []string, region string) error { url := "https://console.aws.amazon.com/servicequotas/home?#!/services/vpc/quotas" return errors.WithStack(&errors.Error{ Kind: ErrNATGatewayLimitExceeded, Message: fmt.Sprintf("NAT gateway limit of %d exceeded in availability zones %s of region %s; remove some of the existing NAT gateways or increase your quota for NAT gateways by at least %d here: %s (if your request was recently approved, please allow ~30 minutes for AWS to reflect this change)", currentLimit, s.StrsAnd(availabilityZones), region, additionalQuotaRequired, url), }) } func ErrorEIPLimitExceeded(currentLimit, additionalQuotaRequired int, region string) error { url := "https://console.aws.amazon.com/servicequotas/home?#!/services/ec2/quotas" return errors.WithStack(&errors.Error{ Kind: ErrEIPLimitExceeded, Message: fmt.Sprintf("elastic IPs limit of %d exceeded in region %s; remove some of the existing elastic IPs or increase your quota for elastic IPs by at least %d here: %s (if your request was recently approved, please allow ~30 minutes for AWS to reflect this change)", currentLimit, region, additionalQuotaRequired, url), }) } func ErrorInternetGatewayLimitExceeded(currentLimit, additionalQuotaRequired int, region string) error { url := "https://console.aws.amazon.com/servicequotas/home?#!/services/vpc/quotas" return errors.WithStack(&errors.Error{ Kind: ErrInternetGatewayLimitExceeded, Message: fmt.Sprintf("internet gateway limit of %d exceeded in region %s; remove some of the existing internet gateways or increase your quota for internet gateways by at least %d here: %s (if your request was recently approved, please allow ~30 minutes for AWS to reflect this change)", currentLimit, region, additionalQuotaRequired, url), }) } func ErrorVPCLimitExceeded(currentLimit, additionalQuotaRequired int, region string) error { url := "https://console.aws.amazon.com/servicequotas/home?#!/services/vpc/quotas" return errors.WithStack(&errors.Error{ Kind: ErrVPCLimitExceeded, Message: fmt.Sprintf("VPC limit of %d exceeded in region %s; remove some of the existing VPCs or increase your quota for VPCs by at least %d here: %s (if your request was recently approved, please allow ~30 minutes for AWS to reflect this change)", currentLimit, region, additionalQuotaRequired, url), }) } func ErrorSecurityGroupRulesExceeded(currentLimit, additionalQuotaRequired int, region string) error { url := "https://console.aws.amazon.com/servicequotas/home?#!/services/vpc/quotas" return errors.WithStack(&errors.Error{ Kind: ErrSecurityGroupRulesExceeded, Message: fmt.Sprintf("security group rules limit of %d exceeded in region %s; remove some node groups, use fewer availability zones, reduce the number of CIDR white lists, or increase your quota for inbound/outbound rules per security group by at least %d here: %s (if your request was recently approved, please allow ~30 minutes for AWS to reflect this change)", currentLimit, region, additionalQuotaRequired, url), }) } func ErrorSecurityGroupLimitExceeded(currentLimit, additionalQuotaRequired int, region string) error { url := "https://console.aws.amazon.com/servicequotas/home?#!/services/vpc/quotas" return errors.WithStack(&errors.Error{ Kind: ErrSecurityGroupLimitExceeded, Message: fmt.Sprintf("security group limit of %d exceeded in region %s; remove some node groups or increase your quota for security groups by at least %d here: %s (if your request was recently approved, please allow ~30 minutes for AWS to reflect this change)", currentLimit, region, additionalQuotaRequired, url), }) } ================================================ FILE: pkg/lib/aws/gen_resource_metadata.py ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import requests import re from string import Template # https://docs.aws.amazon.com/general/latest/gr/eks.html # China regions don't seem to support these endpoints (yet?) REGIONS = [ "us-east-2", # Ohio "us-east-1", # N. Virginia "us-west-1", # California "us-west-2", # Oregon "af-south-1", # Cape town "ap-east-1", # Hong Kong "ap-south-1", # Mumbai "ap-northeast-3", # Osaka "ap-northeast-2", # Seoul "ap-southeast-1", # Singapore "ap-southeast-2", # Sydney "ap-northeast-1", # Tokyo "ca-central-1", # Montreal "eu-central-1", # Frankfurt "eu-west-1", # Ireland "eu-west-2", # London "eu-south-1", # Milan "eu-west-3", # Paris "eu-north-1", # Stockholm "me-south-1", # Bahrain "sa-east-1", # Sao Paulo "us-gov-east-1", # GovCloud US-East "us-gov-west-1", # GovCloud US-West ] OUTPUT_FILE_NAME = "resource_metadata.go" EC2_PRICING_ENDPOINT_TEMPLATE = ( "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonEC2/current/{}/index.json" ) EKS_PRICING_ENDPOINT_TEMPLATE = ( "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonEKS/current/{}/index.json" ) inf_per_instance_type = { "inf1.xlarge": 1, "inf1.2xlarge": 1, "inf1.6xlarge": 4, "inf1.24xlarge": 16, } def get_instance_metadatas(pricing): instance_types = set() instance_mapping = {} for _, product in pricing["products"].items(): if product.get("attributes") is None: continue if product["attributes"].get("instanceType") is None: continue if product["attributes"].get("servicecode") != "AmazonEC2": continue if product["attributes"].get("capacitystatus") != "Used": continue instance_type = product["attributes"]["instanceType"] instance_types.add(instance_type) if product["attributes"].get("operation") != "RunInstances": continue if product["attributes"].get("tenancy") != "Shared": continue if product["attributes"].get("operatingSystem") != "Linux": continue price_dimensions = list(pricing["terms"]["OnDemand"][product["sku"]].values())[0][ "priceDimensions" ] price = list(price_dimensions.values())[0]["pricePerUnit"]["USD"] metadata = { "cpu": int(product["attributes"]["vcpu"]), "mem": int( float(re.sub("[^0-9\\.]", "", product["attributes"]["memory"].split(" ")[0])) * 1024 ), "price": float(price), "gpu": 0, } if product["attributes"].get("gpu") is not None: metadata["gpu"] = product["attributes"]["gpu"] instance_mapping[instance_type] = metadata return instance_types, instance_mapping def get_nlb_metadata(pricing): for _, product in pricing["products"].items(): if product.get("attributes") is None: continue if product.get("productFamily") != "Load Balancer-Network": continue if product["attributes"].get("group") != "ELB:Balancer": continue if product["attributes"].get("operation") != "LoadBalancing:Network": continue if "LoadBalancerUsage" not in product["attributes"].get("usagetype"): continue price_dimensions = list(pricing["terms"]["OnDemand"][product["sku"]].values())[0][ "priceDimensions" ] price = list(price_dimensions.values())[0]["pricePerUnit"]["USD"] return {"price": float(price)} def get_elb_metadata(pricing): for _, product in pricing["products"].items(): if product.get("attributes") is None: continue if product.get("productFamily") != "Load Balancer": continue if product["attributes"].get("group") != "ELB:Balancer": continue if product["attributes"].get("operation") != "LoadBalancing": continue if "LoadBalancerUsage" not in product["attributes"].get("usagetype"): continue price_dimensions = list(pricing["terms"]["OnDemand"][product["sku"]].values())[0][ "priceDimensions" ] price = list(price_dimensions.values())[0]["pricePerUnit"]["USD"] return {"price": float(price)} def get_nat_metadata(pricing): for _, product in pricing["products"].items(): if product.get("attributes") is None: continue if product.get("productFamily") != "NAT Gateway": continue if product["attributes"].get("group") != "NGW:NatGateway": continue if product["attributes"].get("operation") != "NatGateway": continue if not product["attributes"].get("usagetype", "").endswith("-Hours"): continue price_dimensions = list(pricing["terms"]["OnDemand"][product["sku"]].values())[0][ "priceDimensions" ] price = list(price_dimensions.values())[0]["pricePerUnit"]["USD"] return {"price": float(price)} def get_ebs_metadata(pricing): storage_mapping = {} for _, product in pricing["products"].items(): if product.get("attributes") is None: continue if product.get("productFamily") != "Storage": continue # ignore legacy standard storage if product["attributes"].get("volumeApiName") == "standard": continue price_dimensions = list(pricing["terms"]["OnDemand"][product["sku"]].values())[0][ "priceDimensions" ] price = list(price_dimensions.values())[0]["pricePerUnit"]["USD"] metadata = {"type": product["attributes"].get("volumeApiName"), "price_gb": float(price)} # io1 has per IOPS pricing --> add pricing to metadata # if storagedevice does not price per IOPS will set value to 0 if product["attributes"].get("volumeApiName") == "io1": # go through pricing data until found data about IOPS pricing for _, product_iops in pricing["products"].items(): if product_iops.get("attributes") is None: continue if product_iops.get("productFamily") != "System Operation": continue if product_iops["attributes"].get("volumeApiName") != "io1": continue if product_iops["attributes"].get("group") != "EBS IOPS": continue if product_iops["attributes"].get("provisioned") != "Yes": continue price_dimensions = list(pricing["terms"]["OnDemand"][product_iops["sku"]].values())[ 0 ]["priceDimensions"] price = list(price_dimensions.values())[0]["pricePerUnit"]["USD"] metadata["price_iops"] = price metadata["price_throughput"] = 0 metadata["iops_configurable"] = "true" metadata["throughput_configurable"] = "false" elif product["attributes"].get("volumeApiName") == "gp3": # go through pricing data until found data about IOPS and throughput pricing for _, product_iops in pricing["products"].items(): if product_iops.get("attributes") is None: continue if product_iops.get("productFamily") not in [ "System Operation", "Provisioned Throughput", ]: continue if product_iops["attributes"].get("volumeApiName") != "gp3": continue if product_iops["attributes"].get("group") not in ["EBS IOPS", "EBS Throughput"]: continue if product_iops["attributes"].get("provisioned") != "Yes": continue price_dimensions = list(pricing["terms"]["OnDemand"][product_iops["sku"]].values())[ 0 ]["priceDimensions"] if product_iops["attributes"].get("group") == "EBS IOPS": price_iops = list(price_dimensions.values())[0]["pricePerUnit"]["USD"] else: price_throughput = ( float(list(price_dimensions.values())[0]["pricePerUnit"]["USD"]) / 1000 ) metadata["price_iops"] = price_iops metadata["price_throughput"] = price_throughput metadata["throughput_configurable"] = "true" metadata["iops_configurable"] = "true" # set default values for all other storage types else: metadata["price_iops"] = 0 metadata["price_throughput"] = 0 metadata["iops_configurable"] = "false" metadata["throughput_configurable"] = "false" storage_mapping[product["attributes"]["volumeApiName"]] = metadata return storage_mapping def get_eks_price(region): response = requests.get(EKS_PRICING_ENDPOINT_TEMPLATE.format(region)) pricing = response.json() for _, product in pricing["products"].items(): if product.get("attributes") is None: continue if product.get("productFamily") != "Compute": continue if product["attributes"].get("servicecode") != "AmazonEKS": continue if product["attributes"].get("operation") != "CreateOperation": continue if not product["attributes"].get("usagetype", "").endswith("-AmazonEKS-Hours:perCluster"): continue price_dimensions = list(pricing["terms"]["OnDemand"][product["sku"]].values())[0][ "priceDimensions" ] price = list(price_dimensions.values())[0]["pricePerUnit"]["USD"] return float(price) file_template = Template( """/* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ // This file was generated by go generate; DO NOT EDIT package aws import ( "github.com/cortexlabs/cortex/pkg/lib/sets/strset" kresource "k8s.io/apimachinery/pkg/api/resource" ) type InstanceMetadata struct { Region string `json:"region"` Type string `json:"type"` Memory kresource.Quantity `json:"memory"` CPU kresource.Quantity `json:"cpu"` GPU int64 `json:"gpu"` Inf int64 `json:"inf"` Price float64 `json:"price"` } type NLBMetadata struct { Region string `json:"region"` Price float64 `json:"price"` } type ELBMetadata struct { Region string `json:"region"` Price float64 `json:"price"` } type NATMetadata struct { Region string `json:"region"` Price float64 `json:"price"` } type EBSMetadata struct { Region string `json:"region"` PriceGB float64 `json:"price_gb"` PriceIOPS float64 `json:"price_iops"` PriceThroughput float64 `json:"price_throughput"` IOPSConfigurable bool `json:"iops_configurable"` ThroughputConfigurable bool `json:"throughput_configurable"` Type string `json:"type"` } // This set contains all known instance types. Metadata is not available for all of them. var AllInstanceTypes = strset.New( ${all_instance_types} ) // This contains all instance types available in each region. Metadata is not available for all of them. var InstanceTypes = map[string]strset.Set{ ${instance_types_map} } // region -> instance type -> instance metadata var InstanceMetadatas = map[string]map[string]InstanceMetadata{ ${instance_region_map} } // region -> NLB metadata var NLBMetadatas = map[string]NLBMetadata{ ${nlb_region_map} } // region -> ELB metadata var ELBMetadatas = map[string]ELBMetadata{ ${elb_region_map} } // region -> NAT metadata var NATMetadatas = map[string]NATMetadata{ ${nat_region_map} } // region -> EBS metadata var EBSMetadatas = map[string]map[string]EBSMetadata{ ${ebs_region_map} } // region -> EKS price var EKSPrices = map[string]float64{ ${eks_region_map} } """ ) instance_region_map_template = Template( """"${region}": { ${instance_metadatas} }, """ ) instance_types_map_template = Template( """"${region}": strset.New( ${instance_types} ), """ ) instance_metadata_template = Template( """"${type}": {Region: "${region}", Type: "${type}", Memory: kresource.MustParse("${memory}Mi"), CPU: kresource.MustParse("${cpu}"), GPU: ${gpu}, Inf: ${inf}, Price: ${price}}, """ ) nlb_region_map_template = Template( """"${region}": {Region: "${region}", Price: ${price}}, """ ) elb_region_map_template = Template( """"${region}": {Region: "${region}", Price: ${price}}, """ ) nat_region_map_template = Template( """"${region}": {Region: "${region}", Price: ${price}}, """ ) ebs_region_map_template = Template( """"${region}": { ${ebs_metadata} }, """ ) ebs_type_map_template = Template( """"${type}": {Region: "${region}",Type: "${type}", PriceGB: ${price_gb}, PriceIOPS: ${price_iops}, PriceThroughput: ${price_throughput}, IOPSConfigurable: ${iops_configurable}, ThroughputConfigurable: ${throughput_configurable}}, """ ) eks_region_map_template = Template( """"${region}": ${price}, """ ) def instanceTypeSorter(instanceType): parts = instanceType.split(".") if len(parts) != 2: raise Exception(f"unknown instance type: {instanceType}") prefix = parts[0] size = parts[1] if size == "nano": return prefix + ".a" if size == "micro": return prefix + ".b" if size == "small": return prefix + ".c" if size == "medium": return prefix + ".d" if size == "large": return prefix + ".e" if size == "xlarge": return prefix + ".f" if size == "metal": return prefix + ".z" if not size.endswith("xlarge"): raise Exception(f"unknown instance type: {instanceType}") num_xlarge = re.sub("xlarge$", "", size) return prefix + ".y" + num_xlarge.zfill(5) def main(): all_instance_types = set() instance_types_map_str = "" instance_region_map_str = "" nlb_region_map_str = "" elb_region_map_str = "" nat_region_map_str = "" ebs_region_map_str = "" eks_region_map_str = "" for i, region in enumerate(sorted(REGIONS), start=1): print("generating region {}/{} ({})...".format(i, len(REGIONS), region)) response = requests.get(EC2_PRICING_ENDPOINT_TEMPLATE.format(region)) pricing = response.json() instance_types, instance_metadatas = get_instance_metadatas(pricing) nlb_metadata = get_nlb_metadata(pricing) elb_metadata = get_elb_metadata(pricing) nat_metadata = get_nat_metadata(pricing) ebs_metadata = get_ebs_metadata(pricing) eks_price = get_eks_price(region) all_instance_types.update(instance_types) instance_types_str = "" for instance_type in sorted(instance_types, key=instanceTypeSorter): instance_types_str += f'"{instance_type}",\n' instance_metadatas_str = "" for instance_type in sorted(instance_metadatas.keys(), key=instanceTypeSorter): metadata = instance_metadatas[instance_type] instance_metadatas_str += instance_metadata_template.substitute( { "region": region, "type": instance_type, "memory": metadata["mem"], "cpu": metadata["cpu"], "gpu": metadata["gpu"], "inf": inf_per_instance_type.get(instance_type, 0), "price": metadata["price"], } ) ebs_metadatas_str = "" for ebs_type in sorted(ebs_metadata.keys()): metadata = ebs_metadata[ebs_type] ebs_metadatas_str += ebs_type_map_template.substitute( { "region": region, "type": ebs_type, "price_gb": metadata["price_gb"], "price_iops": metadata["price_iops"], "price_throughput": metadata["price_throughput"], "iops_configurable": metadata["iops_configurable"], "throughput_configurable": metadata["throughput_configurable"], } ) instance_types_map_str += instance_types_map_template.substitute( {"region": region, "instance_types": instance_types_str} ) instance_region_map_str += instance_region_map_template.substitute( {"region": region, "instance_metadatas": instance_metadatas_str} ) nlb_region_map_str += nlb_region_map_template.substitute( {"region": region, "price": nlb_metadata["price"]} ) elb_region_map_str += elb_region_map_template.substitute( {"region": region, "price": elb_metadata["price"]} ) nat_region_map_str += nat_region_map_template.substitute( {"region": region, "price": nat_metadata["price"]} ) ebs_region_map_str += ebs_region_map_template.substitute( {"region": region, "ebs_metadata": ebs_metadatas_str} ) eks_region_map_str += eks_region_map_template.substitute( {"region": region, "price": eks_price} ) all_instance_types_str = "" for instance_type in sorted(all_instance_types, key=instanceTypeSorter): all_instance_types_str += f'"{instance_type}",\n' file_str = file_template.substitute( { "all_instance_types": all_instance_types_str, "instance_types_map": instance_types_map_str, "instance_region_map": instance_region_map_str, "nlb_region_map": nlb_region_map_str, "elb_region_map": elb_region_map_str, "nat_region_map": nat_region_map_str, "ebs_region_map": ebs_region_map_str, "eks_region_map": eks_region_map_str, } ) with open(OUTPUT_FILE_NAME, "w") as f: print("writing {}...".format(OUTPUT_FILE_NAME)) f.write(file_str) print("✓ done") if __name__ == "__main__": main() ================================================ FILE: pkg/lib/aws/iam.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package aws import ( "fmt" "strings" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/service/iam" "github.com/cortexlabs/cortex/pkg/lib/errors" ) func PartitionFromRegion(region string) string { if strings.Contains(region, "us-gov") { return "aws-us-gov" } return "aws" } func administratorAccessARN(region string) string { return fmt.Sprintf("arn:%s:iam::aws:policy/AdministratorAccess", PartitionFromRegion(region)) } func (c *Client) GetUser() (iam.User, error) { getUserOutput, err := c.IAM().GetUser(nil) if err != nil { return iam.User{}, errors.WithStack(err) } return *getUserOutput.User, nil } func (c *Client) GetGroupsForUser(userName string) ([]iam.Group, error) { input := &iam.ListGroupsForUserInput{ UserName: &userName, } var groups []iam.Group err := c.IAM().ListGroupsForUserPages(input, func(page *iam.ListGroupsForUserOutput, lastPage bool) bool { for _, group := range page.Groups { groups = append(groups, *group) } return true }) if err != nil { return nil, errors.WithStack(err) } return groups, nil } // Note: root users don't have attached policies, but do have full access func (c *Client) GetManagedPoliciesForUser(userName string) ([]iam.AttachedPolicy, error) { var policies []iam.AttachedPolicy userManagedPolicies, err := c.IAM().ListAttachedUserPolicies(&iam.ListAttachedUserPoliciesInput{ UserName: &userName, }) if err != nil { return nil, errors.WithStack(err) } for _, policy := range userManagedPolicies.AttachedPolicies { policies = append(policies, *policy) } groups, err := c.GetGroupsForUser(userName) if err != nil { return nil, err } for _, group := range groups { groupManagedPolicies, err := c.IAM().ListAttachedGroupPolicies(&iam.ListAttachedGroupPoliciesInput{ GroupName: group.GroupName, }) if err != nil { return nil, errors.WithStack(err) } for _, policy := range groupManagedPolicies.AttachedPolicies { policies = append(policies, *policy) } } return policies, nil } func (c *Client) isAdminUser(user iam.User) bool { // Root users may not have a user name if user.UserName == nil { return true } // Root users may have a user name if user.Arn == nil || strings.HasSuffix(*user.Arn, ":root") { return true } policies, err := c.GetManagedPoliciesForUser(*user.UserName) if err != nil { return false } for _, policy := range policies { if *policy.PolicyArn == administratorAccessARN(c.Region) { return true } } return false } func (c *Client) isRoleAdmin() bool { identity, err := c.STS().GetCallerIdentity(nil) if err != nil { return false } arn := identity.Arn if arn == nil { return false } if !strings.Contains(*arn, ":assumed-role/") { return false } // expected to be in form arn:aws:sts::account-id:assumed-role/role-name/role-session-name arnSplit := strings.Split(*arn, "/") if len(arnSplit) < 2 { return false } roleName := arnSplit[1] isAdmin := false c.IAM().ListAttachedRolePoliciesPages(&iam.ListAttachedRolePoliciesInput{ RoleName: &roleName, }, func(policies *iam.ListAttachedRolePoliciesOutput, lastPage bool) bool { for _, policy := range policies.AttachedPolicies { if *policy.PolicyArn == administratorAccessARN(c.Region) { isAdmin = true return false } } return !lastPage }) return isAdmin } func (c *Client) IsAdmin() bool { user, err := c.GetUser() if err != nil { awsErr, ok := errors.CauseOrSelf(err).(awserr.Error) if !ok { return false } // this particular error is returned if GetUser() is invoked using credentials that are not for users if awsErr.Code() == "ValidationError" && strings.Contains(strings.ToLower(err.Error()), strings.ToLower("calling with non-User credentials")) { return c.isRoleAdmin() } return false } return c.isAdminUser(user) } // delete non default policy versions and then delete the policy (as required by aws) func (c *Client) DeletePolicy(policyARN string) error { policyVersionList, err := c.IAM().ListPolicyVersions(&iam.ListPolicyVersionsInput{ PolicyArn: aws.String(policyARN), }) if err != nil { return errors.WithStack(err) } for _, policy := range policyVersionList.Versions { if !*policy.IsDefaultVersion { _, err = c.IAM().DeletePolicyVersion(&iam.DeletePolicyVersionInput{ PolicyArn: aws.String(policyARN), VersionId: policy.VersionId, }) if err != nil { return errors.WithStack(err) } } } _, err = c.IAM().DeletePolicy(&iam.DeletePolicyInput{ PolicyArn: aws.String(policyARN), }) if err != nil { return errors.WithStack(err) } return nil } func (c *Client) GetPolicyOrNil(policyARN string) (*iam.Policy, error) { policyOutput, err := c.IAM().GetPolicy(&iam.GetPolicyInput{ PolicyArn: aws.String(policyARN), }) if err != nil { if IsErrCode(err, iam.ErrCodeNoSuchEntityException) { return nil, nil } return nil, errors.WithStack(err) } if policyOutput != nil { return policyOutput.Policy, nil } return nil, nil } ================================================ FILE: pkg/lib/aws/resource_metadata.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ // This file was generated by go generate; DO NOT EDIT package aws import ( "github.com/cortexlabs/cortex/pkg/lib/sets/strset" kresource "k8s.io/apimachinery/pkg/api/resource" ) type InstanceMetadata struct { Region string `json:"region"` Type string `json:"type"` Memory kresource.Quantity `json:"memory"` CPU kresource.Quantity `json:"cpu"` GPU int64 `json:"gpu"` Inf int64 `json:"inf"` Price float64 `json:"price"` } type NLBMetadata struct { Region string `json:"region"` Price float64 `json:"price"` } type ELBMetadata struct { Region string `json:"region"` Price float64 `json:"price"` } type NATMetadata struct { Region string `json:"region"` Price float64 `json:"price"` } type EBSMetadata struct { Region string `json:"region"` PriceGB float64 `json:"price_gb"` PriceIOPS float64 `json:"price_iops"` PriceThroughput float64 `json:"price_throughput"` IOPSConfigurable bool `json:"iops_configurable"` ThroughputConfigurable bool `json:"throughput_configurable"` Type string `json:"type"` } // This set contains all known instance types. Metadata is not available for all of them. var AllInstanceTypes = strset.New( "a1.medium", "a1.large", "a1.xlarge", "a1.2xlarge", "a1.4xlarge", "a1.metal", "c1.medium", "c1.xlarge", "c3.large", "c3.xlarge", "c3.2xlarge", "c3.4xlarge", "c3.8xlarge", "c4.large", "c4.xlarge", "c4.2xlarge", "c4.4xlarge", "c4.8xlarge", "c5.large", "c5.xlarge", "c5.2xlarge", "c5.4xlarge", "c5.9xlarge", "c5.12xlarge", "c5.18xlarge", "c5.24xlarge", "c5.metal", "c5a.large", "c5a.xlarge", "c5a.2xlarge", "c5a.4xlarge", "c5a.8xlarge", "c5a.12xlarge", "c5a.16xlarge", "c5a.24xlarge", "c5ad.large", "c5ad.xlarge", "c5ad.2xlarge", "c5ad.4xlarge", "c5ad.8xlarge", "c5ad.12xlarge", "c5ad.16xlarge", "c5ad.24xlarge", "c5d.large", "c5d.xlarge", "c5d.2xlarge", "c5d.4xlarge", "c5d.9xlarge", "c5d.12xlarge", "c5d.18xlarge", "c5d.24xlarge", "c5d.metal", "c5n.large", "c5n.xlarge", "c5n.2xlarge", "c5n.4xlarge", "c5n.9xlarge", "c5n.18xlarge", "c5n.metal", "c6a.large", "c6a.xlarge", "c6a.2xlarge", "c6a.4xlarge", "c6a.8xlarge", "c6a.12xlarge", "c6a.16xlarge", "c6a.24xlarge", "c6a.32xlarge", "c6a.48xlarge", "c6a.metal", "c6g.medium", "c6g.large", "c6g.xlarge", "c6g.2xlarge", "c6g.4xlarge", "c6g.8xlarge", "c6g.12xlarge", "c6g.16xlarge", "c6g.metal", "c6gd.medium", "c6gd.large", "c6gd.xlarge", "c6gd.2xlarge", "c6gd.4xlarge", "c6gd.8xlarge", "c6gd.12xlarge", "c6gd.16xlarge", "c6gd.metal", "c6gn.medium", "c6gn.large", "c6gn.xlarge", "c6gn.2xlarge", "c6gn.4xlarge", "c6gn.8xlarge", "c6gn.12xlarge", "c6gn.16xlarge", "c6gn.metal", "c6i.large", "c6i.xlarge", "c6i.2xlarge", "c6i.4xlarge", "c6i.8xlarge", "c6i.12xlarge", "c6i.16xlarge", "c6i.24xlarge", "c6i.32xlarge", "c6i.metal", "cc2.8xlarge", "cr1.8xlarge", "d2.xlarge", "d2.2xlarge", "d2.4xlarge", "d2.8xlarge", "d3.xlarge", "d3.2xlarge", "d3.4xlarge", "d3.8xlarge", "d3en.xlarge", "d3en.2xlarge", "d3en.4xlarge", "d3en.6xlarge", "d3en.8xlarge", "d3en.12xlarge", "dl1.24xlarge", "f1.2xlarge", "f1.4xlarge", "f1.16xlarge", "g2.2xlarge", "g2.8xlarge", "g3.4xlarge", "g3.8xlarge", "g3.16xlarge", "g3s.xlarge", "g4ad.xlarge", "g4ad.2xlarge", "g4ad.4xlarge", "g4ad.8xlarge", "g4ad.16xlarge", "g4dn.xlarge", "g4dn.2xlarge", "g4dn.4xlarge", "g4dn.8xlarge", "g4dn.12xlarge", "g4dn.16xlarge", "g4dn.metal", "g5.xlarge", "g5.2xlarge", "g5.4xlarge", "g5.8xlarge", "g5.12xlarge", "g5.16xlarge", "g5.24xlarge", "g5.48xlarge", "g5g.xlarge", "g5g.2xlarge", "g5g.4xlarge", "g5g.8xlarge", "g5g.16xlarge", "g5g.metal", "h1.2xlarge", "h1.4xlarge", "h1.8xlarge", "h1.16xlarge", "hpc6a.48xlarge", "hs1.8xlarge", "i2.xlarge", "i2.2xlarge", "i2.4xlarge", "i2.8xlarge", "i3.large", "i3.xlarge", "i3.2xlarge", "i3.4xlarge", "i3.8xlarge", "i3.16xlarge", "i3.metal", "i3en.large", "i3en.xlarge", "i3en.2xlarge", "i3en.3xlarge", "i3en.6xlarge", "i3en.12xlarge", "i3en.24xlarge", "i3en.metal", "i3p.16xlarge", "im4gn.large", "im4gn.xlarge", "im4gn.2xlarge", "im4gn.4xlarge", "im4gn.8xlarge", "im4gn.16xlarge", "inf1.xlarge", "inf1.2xlarge", "inf1.6xlarge", "inf1.24xlarge", "is4gen.medium", "is4gen.large", "is4gen.xlarge", "is4gen.2xlarge", "is4gen.4xlarge", "is4gen.8xlarge", "m1.small", "m1.medium", "m1.large", "m1.xlarge", "m2.xlarge", "m2.2xlarge", "m2.4xlarge", "m3.medium", "m3.large", "m3.xlarge", "m3.2xlarge", "m4.large", "m4.xlarge", "m4.2xlarge", "m4.4xlarge", "m4.10xlarge", "m4.16xlarge", "m5.large", "m5.xlarge", "m5.2xlarge", "m5.4xlarge", "m5.8xlarge", "m5.12xlarge", "m5.16xlarge", "m5.24xlarge", "m5.metal", "m5a.large", "m5a.xlarge", "m5a.2xlarge", "m5a.4xlarge", "m5a.8xlarge", "m5a.12xlarge", "m5a.16xlarge", "m5a.24xlarge", "m5ad.large", "m5ad.xlarge", "m5ad.2xlarge", "m5ad.4xlarge", "m5ad.8xlarge", "m5ad.12xlarge", "m5ad.16xlarge", "m5ad.24xlarge", "m5d.large", "m5d.xlarge", "m5d.2xlarge", "m5d.4xlarge", "m5d.8xlarge", "m5d.12xlarge", "m5d.16xlarge", "m5d.24xlarge", "m5d.metal", "m5dn.large", "m5dn.xlarge", "m5dn.2xlarge", "m5dn.4xlarge", "m5dn.8xlarge", "m5dn.12xlarge", "m5dn.16xlarge", "m5dn.24xlarge", "m5dn.metal", "m5n.large", "m5n.xlarge", "m5n.2xlarge", "m5n.4xlarge", "m5n.8xlarge", "m5n.12xlarge", "m5n.16xlarge", "m5n.24xlarge", "m5n.metal", "m5zn.large", "m5zn.xlarge", "m5zn.2xlarge", "m5zn.3xlarge", "m5zn.6xlarge", "m5zn.12xlarge", "m5zn.metal", "m6a.large", "m6a.xlarge", "m6a.2xlarge", "m6a.4xlarge", "m6a.8xlarge", "m6a.12xlarge", "m6a.16xlarge", "m6a.24xlarge", "m6a.32xlarge", "m6a.48xlarge", "m6a.metal", "m6g.medium", "m6g.large", "m6g.xlarge", "m6g.2xlarge", "m6g.4xlarge", "m6g.8xlarge", "m6g.12xlarge", "m6g.16xlarge", "m6g.metal", "m6gd.medium", "m6gd.large", "m6gd.xlarge", "m6gd.2xlarge", "m6gd.4xlarge", "m6gd.8xlarge", "m6gd.12xlarge", "m6gd.16xlarge", "m6gd.metal", "m6i.large", "m6i.xlarge", "m6i.2xlarge", "m6i.4xlarge", "m6i.8xlarge", "m6i.12xlarge", "m6i.16xlarge", "m6i.24xlarge", "m6i.32xlarge", "m6i.metal", "mac1.metal", "mac2.metal", "p2.xlarge", "p2.8xlarge", "p2.16xlarge", "p3.2xlarge", "p3.8xlarge", "p3.16xlarge", "p3dn.24xlarge", "p4d.24xlarge", "r3.large", "r3.xlarge", "r3.2xlarge", "r3.4xlarge", "r3.8xlarge", "r4.large", "r4.xlarge", "r4.2xlarge", "r4.4xlarge", "r4.8xlarge", "r4.16xlarge", "r5.large", "r5.xlarge", "r5.2xlarge", "r5.4xlarge", "r5.8xlarge", "r5.12xlarge", "r5.16xlarge", "r5.24xlarge", "r5.metal", "r5a.large", "r5a.xlarge", "r5a.2xlarge", "r5a.4xlarge", "r5a.8xlarge", "r5a.12xlarge", "r5a.16xlarge", "r5a.24xlarge", "r5ad.large", "r5ad.xlarge", "r5ad.2xlarge", "r5ad.4xlarge", "r5ad.8xlarge", "r5ad.12xlarge", "r5ad.16xlarge", "r5ad.24xlarge", "r5b.large", "r5b.xlarge", "r5b.2xlarge", "r5b.4xlarge", "r5b.8xlarge", "r5b.12xlarge", "r5b.16xlarge", "r5b.24xlarge", "r5b.metal", "r5d.large", "r5d.xlarge", "r5d.2xlarge", "r5d.4xlarge", "r5d.8xlarge", "r5d.12xlarge", "r5d.16xlarge", "r5d.24xlarge", "r5d.metal", "r5dn.large", "r5dn.xlarge", "r5dn.2xlarge", "r5dn.4xlarge", "r5dn.8xlarge", "r5dn.12xlarge", "r5dn.16xlarge", "r5dn.24xlarge", "r5dn.metal", "r5n.large", "r5n.xlarge", "r5n.2xlarge", "r5n.4xlarge", "r5n.8xlarge", "r5n.12xlarge", "r5n.16xlarge", "r5n.24xlarge", "r5n.metal", "r6g.medium", "r6g.large", "r6g.xlarge", "r6g.2xlarge", "r6g.4xlarge", "r6g.8xlarge", "r6g.12xlarge", "r6g.16xlarge", "r6g.metal", "r6gd.medium", "r6gd.large", "r6gd.xlarge", "r6gd.2xlarge", "r6gd.4xlarge", "r6gd.8xlarge", "r6gd.12xlarge", "r6gd.16xlarge", "r6gd.metal", "r6i.large", "r6i.xlarge", "r6i.2xlarge", "r6i.4xlarge", "r6i.8xlarge", "r6i.12xlarge", "r6i.16xlarge", "r6i.24xlarge", "r6i.32xlarge", "r6i.metal", "t1.micro", "t2.nano", "t2.micro", "t2.small", "t2.medium", "t2.large", "t2.xlarge", "t2.2xlarge", "t3.nano", "t3.micro", "t3.small", "t3.medium", "t3.large", "t3.xlarge", "t3.2xlarge", "t3a.nano", "t3a.micro", "t3a.small", "t3a.medium", "t3a.large", "t3a.xlarge", "t3a.2xlarge", "t4g.nano", "t4g.micro", "t4g.small", "t4g.medium", "t4g.large", "t4g.xlarge", "t4g.2xlarge", "u-12tb1.112xlarge", "u-12tb1.metal", "u-18tb1.metal", "u-24tb1.metal", "u-3tb1.56xlarge", "u-6tb1.56xlarge", "u-6tb1.112xlarge", "u-6tb1.metal", "u-9tb1.112xlarge", "u-9tb1.metal", "vt1.3xlarge", "vt1.6xlarge", "vt1.24xlarge", "x1.16xlarge", "x1.32xlarge", "x1e.xlarge", "x1e.2xlarge", "x1e.4xlarge", "x1e.8xlarge", "x1e.16xlarge", "x1e.32xlarge", "x2gd.medium", "x2gd.large", "x2gd.xlarge", "x2gd.2xlarge", "x2gd.4xlarge", "x2gd.8xlarge", "x2gd.12xlarge", "x2gd.16xlarge", "x2gd.metal", "x2idn.16xlarge", "x2idn.24xlarge", "x2idn.32xlarge", "x2iedn.xlarge", "x2iedn.2xlarge", "x2iedn.4xlarge", "x2iedn.8xlarge", "x2iedn.16xlarge", "x2iedn.24xlarge", "x2iedn.32xlarge", "x2iezn.2xlarge", "x2iezn.4xlarge", "x2iezn.6xlarge", "x2iezn.8xlarge", "x2iezn.12xlarge", "x2iezn.metal", "z1d.large", "z1d.xlarge", "z1d.2xlarge", "z1d.3xlarge", "z1d.6xlarge", "z1d.12xlarge", "z1d.metal", ) // This contains all instance types available in each region. Metadata is not available for all of them. var InstanceTypes = map[string]strset.Set{ "af-south-1": strset.New( "c5.large", "c5.xlarge", "c5.2xlarge", "c5.4xlarge", "c5.9xlarge", "c5.12xlarge", "c5.18xlarge", "c5.24xlarge", "c5.metal", "c5a.large", "c5a.xlarge", "c5a.2xlarge", "c5a.4xlarge", "c5a.8xlarge", "c5a.12xlarge", "c5a.16xlarge", "c5a.24xlarge", "c5ad.large", "c5ad.xlarge", "c5ad.2xlarge", "c5ad.4xlarge", "c5ad.8xlarge", "c5ad.12xlarge", "c5ad.16xlarge", "c5ad.24xlarge", "c5d.large", "c5d.xlarge", "c5d.2xlarge", "c5d.4xlarge", "c5d.9xlarge", "c5d.12xlarge", "c5d.18xlarge", "c5d.24xlarge", "c5d.metal", "c5n.large", "c5n.xlarge", "c5n.2xlarge", "c5n.4xlarge", "c5n.9xlarge", "c5n.18xlarge", "c5n.metal", "d2.xlarge", "d2.2xlarge", "d2.4xlarge", "d2.8xlarge", "g4dn.xlarge", "g4dn.2xlarge", "g4dn.4xlarge", "g4dn.8xlarge", "g4dn.12xlarge", "g4dn.16xlarge", "g4dn.metal", "i3.large", "i3.xlarge", "i3.2xlarge", "i3.4xlarge", "i3.8xlarge", "i3.16xlarge", "i3.metal", "i3en.large", "i3en.xlarge", "i3en.2xlarge", "i3en.3xlarge", "i3en.6xlarge", "i3en.12xlarge", "i3en.24xlarge", "i3en.metal", "inf1.24xlarge", "m5.large", "m5.xlarge", "m5.2xlarge", "m5.4xlarge", "m5.8xlarge", "m5.12xlarge", "m5.16xlarge", "m5.24xlarge", "m5.metal", "m5d.large", "m5d.xlarge", "m5d.2xlarge", "m5d.4xlarge", "m5d.8xlarge", "m5d.12xlarge", "m5d.16xlarge", "m5d.24xlarge", "m5d.metal", "r5.large", "r5.xlarge", "r5.2xlarge", "r5.4xlarge", "r5.8xlarge", "r5.12xlarge", "r5.16xlarge", "r5.24xlarge", "r5.metal", "r5d.large", "r5d.xlarge", "r5d.2xlarge", "r5d.4xlarge", "r5d.8xlarge", "r5d.12xlarge", "r5d.16xlarge", "r5d.24xlarge", "r5d.metal", "r5dn.large", "r5dn.xlarge", "r5dn.2xlarge", "r5dn.4xlarge", "r5dn.8xlarge", "r5dn.12xlarge", "r5dn.16xlarge", "r5dn.24xlarge", "r5dn.metal", "t3.nano", "t3.micro", "t3.small", "t3.medium", "t3.large", "t3.xlarge", "t3.2xlarge", "x1.16xlarge", "x1.32xlarge", "x1e.xlarge", "x1e.2xlarge", "x1e.4xlarge", "x1e.8xlarge", "x1e.16xlarge", "x1e.32xlarge", ), "ap-east-1": strset.New( "c5.large", "c5.xlarge", "c5.2xlarge", "c5.4xlarge", "c5.9xlarge", "c5.12xlarge", "c5.18xlarge", "c5.24xlarge", "c5.metal", "c5a.large", "c5a.xlarge", "c5a.2xlarge", "c5a.4xlarge", "c5a.8xlarge", "c5a.12xlarge", "c5a.16xlarge", "c5a.24xlarge", "c5d.large", "c5d.xlarge", "c5d.2xlarge", "c5d.4xlarge", "c5d.9xlarge", "c5d.18xlarge", "c5n.large", "c5n.xlarge", "c5n.2xlarge", "c5n.4xlarge", "c5n.9xlarge", "c5n.18xlarge", "c5n.metal", "c6g.medium", "c6g.large", "c6g.xlarge", "c6g.2xlarge", "c6g.4xlarge", "c6g.8xlarge", "c6g.12xlarge", "c6g.16xlarge", "c6g.metal", "c6gn.medium", "c6gn.large", "c6gn.xlarge", "c6gn.2xlarge", "c6gn.4xlarge", "c6gn.8xlarge", "c6gn.12xlarge", "c6gn.16xlarge", "d2.xlarge", "d2.2xlarge", "d2.4xlarge", "d2.8xlarge", "g4dn.xlarge", "g4dn.2xlarge", "g4dn.4xlarge", "g4dn.8xlarge", "g4dn.12xlarge", "g4dn.16xlarge", "g4dn.metal", "i3.large", "i3.xlarge", "i3.2xlarge", "i3.4xlarge", "i3.8xlarge", "i3.16xlarge", "i3.metal", "i3en.large", "i3en.xlarge", "i3en.2xlarge", "i3en.3xlarge", "i3en.6xlarge", "i3en.12xlarge", "i3en.24xlarge", "i3en.metal", "inf1.xlarge", "inf1.2xlarge", "inf1.6xlarge", "inf1.24xlarge", "m5.large", "m5.xlarge", "m5.2xlarge", "m5.4xlarge", "m5.8xlarge", "m5.12xlarge", "m5.16xlarge", "m5.24xlarge", "m5.metal", "m5d.large", "m5d.xlarge", "m5d.2xlarge", "m5d.4xlarge", "m5d.8xlarge", "m5d.12xlarge", "m5d.16xlarge", "m5d.24xlarge", "m5d.metal", "m6g.medium", "m6g.large", "m6g.xlarge", "m6g.2xlarge", "m6g.4xlarge", "m6g.8xlarge", "m6g.12xlarge", "m6g.16xlarge", "m6g.metal", "r5.large", "r5.xlarge", "r5.2xlarge", "r5.4xlarge", "r5.8xlarge", "r5.12xlarge", "r5.16xlarge", "r5.24xlarge", "r5.metal", "r5d.large", "r5d.xlarge", "r5d.2xlarge", "r5d.4xlarge", "r5d.8xlarge", "r5d.12xlarge", "r5d.16xlarge", "r5d.24xlarge", "r5d.metal", "r5n.large", "r5n.xlarge", "r5n.2xlarge", "r5n.4xlarge", "r5n.8xlarge", "r5n.12xlarge", "r5n.16xlarge", "r5n.24xlarge", "r5n.metal", "r6g.medium", "r6g.large", "r6g.xlarge", "r6g.2xlarge", "r6g.4xlarge", "r6g.8xlarge", "r6g.12xlarge", "r6g.16xlarge", "r6g.metal", "t3.nano", "t3.micro", "t3.small", "t3.medium", "t3.large", "t3.xlarge", "t3.2xlarge", "t4g.nano", "t4g.micro", "t4g.small", "t4g.medium", "t4g.large", "t4g.xlarge", "t4g.2xlarge", "x1.16xlarge", "x1.32xlarge", ), "ap-northeast-1": strset.New( "a1.medium", "a1.large", "a1.xlarge", "a1.2xlarge", "a1.4xlarge", "a1.metal", "c1.medium", "c1.xlarge", "c3.large", "c3.xlarge", "c3.2xlarge", "c3.4xlarge", "c3.8xlarge", "c4.large", "c4.xlarge", "c4.2xlarge", "c4.4xlarge", "c4.8xlarge", "c5.large", "c5.xlarge", "c5.2xlarge", "c5.4xlarge", "c5.9xlarge", "c5.12xlarge", "c5.18xlarge", "c5.24xlarge", "c5.metal", "c5a.large", "c5a.xlarge", "c5a.2xlarge", "c5a.4xlarge", "c5a.8xlarge", "c5a.12xlarge", "c5a.16xlarge", "c5a.24xlarge", "c5d.large", "c5d.xlarge", "c5d.2xlarge", "c5d.4xlarge", "c5d.9xlarge", "c5d.12xlarge", "c5d.18xlarge", "c5d.24xlarge", "c5d.metal", "c5n.large", "c5n.xlarge", "c5n.2xlarge", "c5n.4xlarge", "c5n.9xlarge", "c5n.18xlarge", "c5n.metal", "c6g.medium", "c6g.large", "c6g.xlarge", "c6g.2xlarge", "c6g.4xlarge", "c6g.8xlarge", "c6g.12xlarge", "c6g.16xlarge", "c6g.metal", "c6gd.medium", "c6gd.large", "c6gd.xlarge", "c6gd.2xlarge", "c6gd.4xlarge", "c6gd.8xlarge", "c6gd.12xlarge", "c6gd.16xlarge", "c6gd.metal", "c6gn.medium", "c6gn.large", "c6gn.xlarge", "c6gn.2xlarge", "c6gn.4xlarge", "c6gn.8xlarge", "c6gn.12xlarge", "c6gn.16xlarge", "c6gn.metal", "c6i.large", "c6i.xlarge", "c6i.2xlarge", "c6i.4xlarge", "c6i.8xlarge", "c6i.12xlarge", "c6i.16xlarge", "c6i.24xlarge", "c6i.32xlarge", "c6i.metal", "cc2.8xlarge", "cr1.8xlarge", "d2.xlarge", "d2.2xlarge", "d2.4xlarge", "d2.8xlarge", "d3.xlarge", "d3.2xlarge", "d3.4xlarge", "d3.8xlarge", "g2.2xlarge", "g2.8xlarge", "g3.4xlarge", "g3.8xlarge", "g3.16xlarge", "g3s.xlarge", "g4ad.xlarge", "g4ad.2xlarge", "g4ad.4xlarge", "g4ad.8xlarge", "g4ad.16xlarge", "g4dn.xlarge", "g4dn.2xlarge", "g4dn.4xlarge", "g4dn.8xlarge", "g4dn.12xlarge", "g4dn.16xlarge", "g4dn.metal", "g5g.xlarge", "g5g.2xlarge", "g5g.4xlarge", "g5g.8xlarge", "g5g.16xlarge", "g5g.metal", "hs1.8xlarge", "i2.xlarge", "i2.2xlarge", "i2.4xlarge", "i2.8xlarge", "i3.large", "i3.xlarge", "i3.2xlarge", "i3.4xlarge", "i3.8xlarge", "i3.16xlarge", "i3.metal", "i3en.large", "i3en.xlarge", "i3en.2xlarge", "i3en.3xlarge", "i3en.6xlarge", "i3en.12xlarge", "i3en.24xlarge", "i3en.metal", "inf1.xlarge", "inf1.2xlarge", "inf1.6xlarge", "inf1.24xlarge", "m1.small", "m1.medium", "m1.large", "m1.xlarge", "m2.xlarge", "m2.2xlarge", "m2.4xlarge", "m3.medium", "m3.large", "m3.xlarge", "m3.2xlarge", "m4.large", "m4.xlarge", "m4.2xlarge", "m4.4xlarge", "m4.10xlarge", "m4.16xlarge", "m5.large", "m5.xlarge", "m5.2xlarge", "m5.4xlarge", "m5.8xlarge", "m5.12xlarge", "m5.16xlarge", "m5.24xlarge", "m5.metal", "m5a.large", "m5a.xlarge", "m5a.2xlarge", "m5a.4xlarge", "m5a.8xlarge", "m5a.12xlarge", "m5a.16xlarge", "m5a.24xlarge", "m5ad.large", "m5ad.xlarge", "m5ad.2xlarge", "m5ad.4xlarge", "m5ad.8xlarge", "m5ad.12xlarge", "m5ad.16xlarge", "m5ad.24xlarge", "m5d.large", "m5d.xlarge", "m5d.2xlarge", "m5d.4xlarge", "m5d.8xlarge", "m5d.12xlarge", "m5d.16xlarge", "m5d.24xlarge", "m5d.metal", "m5dn.large", "m5dn.xlarge", "m5dn.2xlarge", "m5dn.4xlarge", "m5dn.8xlarge", "m5dn.12xlarge", "m5dn.16xlarge", "m5dn.24xlarge", "m5dn.metal", "m5n.large", "m5n.xlarge", "m5n.2xlarge", "m5n.4xlarge", "m5n.8xlarge", "m5n.12xlarge", "m5n.16xlarge", "m5n.24xlarge", "m5n.metal", "m5zn.large", "m5zn.xlarge", "m5zn.2xlarge", "m5zn.3xlarge", "m5zn.6xlarge", "m5zn.12xlarge", "m5zn.metal", "m6g.medium", "m6g.large", "m6g.xlarge", "m6g.2xlarge", "m6g.4xlarge", "m6g.8xlarge", "m6g.12xlarge", "m6g.16xlarge", "m6g.metal", "m6gd.medium", "m6gd.large", "m6gd.xlarge", "m6gd.2xlarge", "m6gd.4xlarge", "m6gd.8xlarge", "m6gd.12xlarge", "m6gd.16xlarge", "m6gd.metal", "m6i.large", "m6i.xlarge", "m6i.2xlarge", "m6i.4xlarge", "m6i.8xlarge", "m6i.12xlarge", "m6i.16xlarge", "m6i.24xlarge", "m6i.32xlarge", "m6i.metal", "mac1.metal", "p2.xlarge", "p2.8xlarge", "p2.16xlarge", "p3.2xlarge", "p3.8xlarge", "p3.16xlarge", "p3dn.24xlarge", "p4d.24xlarge", "r3.large", "r3.xlarge", "r3.2xlarge", "r3.4xlarge", "r3.8xlarge", "r4.large", "r4.xlarge", "r4.2xlarge", "r4.4xlarge", "r4.8xlarge", "r4.16xlarge", "r5.large", "r5.xlarge", "r5.2xlarge", "r5.4xlarge", "r5.8xlarge", "r5.12xlarge", "r5.16xlarge", "r5.24xlarge", "r5.metal", "r5a.large", "r5a.xlarge", "r5a.2xlarge", "r5a.4xlarge", "r5a.8xlarge", "r5a.12xlarge", "r5a.16xlarge", "r5a.24xlarge", "r5ad.large", "r5ad.xlarge", "r5ad.2xlarge", "r5ad.4xlarge", "r5ad.8xlarge", "r5ad.12xlarge", "r5ad.16xlarge", "r5ad.24xlarge", "r5b.large", "r5b.xlarge", "r5b.2xlarge", "r5b.4xlarge", "r5b.8xlarge", "r5b.12xlarge", "r5b.16xlarge", "r5b.24xlarge", "r5b.metal", "r5d.large", "r5d.xlarge", "r5d.2xlarge", "r5d.4xlarge", "r5d.8xlarge", "r5d.12xlarge", "r5d.16xlarge", "r5d.24xlarge", "r5d.metal", "r5dn.large", "r5dn.xlarge", "r5dn.2xlarge", "r5dn.4xlarge", "r5dn.8xlarge", "r5dn.12xlarge", "r5dn.16xlarge", "r5dn.24xlarge", "r5dn.metal", "r5n.large", "r5n.xlarge", "r5n.2xlarge", "r5n.4xlarge", "r5n.8xlarge", "r5n.12xlarge", "r5n.16xlarge", "r5n.24xlarge", "r5n.metal", "r6g.medium", "r6g.large", "r6g.xlarge", "r6g.2xlarge", "r6g.4xlarge", "r6g.8xlarge", "r6g.12xlarge", "r6g.16xlarge", "r6g.metal", "r6gd.medium", "r6gd.large", "r6gd.xlarge", "r6gd.2xlarge", "r6gd.4xlarge", "r6gd.8xlarge", "r6gd.12xlarge", "r6gd.16xlarge", "r6gd.metal", "r6i.large", "r6i.xlarge", "r6i.2xlarge", "r6i.4xlarge", "r6i.8xlarge", "r6i.12xlarge", "r6i.16xlarge", "r6i.24xlarge", "r6i.32xlarge", "r6i.metal", "t1.micro", "t2.nano", "t2.micro", "t2.small", "t2.medium", "t2.large", "t2.xlarge", "t2.2xlarge", "t3.nano", "t3.micro", "t3.small", "t3.medium", "t3.large", "t3.xlarge", "t3.2xlarge", "t3a.nano", "t3a.micro", "t3a.small", "t3a.medium", "t3a.large", "t3a.xlarge", "t3a.2xlarge", "t4g.nano", "t4g.micro", "t4g.small", "t4g.medium", "t4g.large", "t4g.xlarge", "t4g.2xlarge", "u-12tb1.metal", "u-6tb1.metal", "u-9tb1.metal", "vt1.3xlarge", "vt1.6xlarge", "vt1.24xlarge", "x1.16xlarge", "x1.32xlarge", "x1e.xlarge", "x1e.2xlarge", "x1e.4xlarge", "x1e.8xlarge", "x1e.16xlarge", "x1e.32xlarge", "x2idn.16xlarge", "x2idn.24xlarge", "x2idn.32xlarge", "x2iedn.xlarge", "x2iedn.2xlarge", "x2iedn.4xlarge", "x2iedn.8xlarge", "x2iedn.16xlarge", "x2iedn.24xlarge", "x2iedn.32xlarge", "x2iezn.2xlarge", "x2iezn.4xlarge", "x2iezn.6xlarge", "x2iezn.8xlarge", "x2iezn.12xlarge", "x2iezn.metal", "z1d.large", "z1d.xlarge", "z1d.2xlarge", "z1d.3xlarge", "z1d.6xlarge", "z1d.12xlarge", "z1d.metal", ), "ap-northeast-2": strset.New( "c3.large", "c3.xlarge", "c3.2xlarge", "c3.4xlarge", "c3.8xlarge", "c4.large", "c4.xlarge", "c4.2xlarge", "c4.4xlarge", "c4.8xlarge", "c5.large", "c5.xlarge", "c5.2xlarge", "c5.4xlarge", "c5.9xlarge", "c5.12xlarge", "c5.18xlarge", "c5.24xlarge", "c5.metal", "c5a.large", "c5a.xlarge", "c5a.2xlarge", "c5a.4xlarge", "c5a.8xlarge", "c5a.12xlarge", "c5a.16xlarge", "c5a.24xlarge", "c5d.large", "c5d.xlarge", "c5d.2xlarge", "c5d.4xlarge", "c5d.9xlarge", "c5d.12xlarge", "c5d.18xlarge", "c5d.24xlarge", "c5d.metal", "c5n.large", "c5n.xlarge", "c5n.2xlarge", "c5n.4xlarge", "c5n.9xlarge", "c5n.18xlarge", "c5n.metal", "c6g.medium", "c6g.large", "c6g.xlarge", "c6g.2xlarge", "c6g.4xlarge", "c6g.8xlarge", "c6g.12xlarge", "c6g.16xlarge", "c6g.metal", "c6i.large", "c6i.xlarge", "c6i.2xlarge", "c6i.4xlarge", "c6i.8xlarge", "c6i.12xlarge", "c6i.16xlarge", "c6i.24xlarge", "c6i.32xlarge", "c6i.metal", "d2.xlarge", "d2.2xlarge", "d2.4xlarge", "d2.8xlarge", "g2.2xlarge", "g2.8xlarge", "g3.4xlarge", "g3.8xlarge", "g3.16xlarge", "g3s.xlarge", "g4dn.xlarge", "g4dn.2xlarge", "g4dn.4xlarge", "g4dn.8xlarge", "g4dn.12xlarge", "g4dn.16xlarge", "g4dn.metal", "g5g.xlarge", "g5g.2xlarge", "g5g.4xlarge", "g5g.8xlarge", "g5g.16xlarge", "g5g.metal", "i2.xlarge", "i2.2xlarge", "i2.4xlarge", "i2.8xlarge", "i3.large", "i3.xlarge", "i3.2xlarge", "i3.4xlarge", "i3.8xlarge", "i3.16xlarge", "i3.metal", "i3en.large", "i3en.xlarge", "i3en.2xlarge", "i3en.3xlarge", "i3en.6xlarge", "i3en.12xlarge", "i3en.24xlarge", "i3en.metal", "inf1.xlarge", "inf1.2xlarge", "inf1.6xlarge", "inf1.24xlarge", "m3.medium", "m3.large", "m3.xlarge", "m3.2xlarge", "m4.large", "m4.xlarge", "m4.2xlarge", "m4.4xlarge", "m4.10xlarge", "m4.16xlarge", "m5.large", "m5.xlarge", "m5.2xlarge", "m5.4xlarge", "m5.8xlarge", "m5.12xlarge", "m5.16xlarge", "m5.24xlarge", "m5.metal", "m5a.large", "m5a.xlarge", "m5a.2xlarge", "m5a.4xlarge", "m5a.8xlarge", "m5a.12xlarge", "m5a.16xlarge", "m5a.24xlarge", "m5ad.large", "m5ad.xlarge", "m5ad.2xlarge", "m5ad.4xlarge", "m5ad.8xlarge", "m5ad.12xlarge", "m5ad.16xlarge", "m5ad.24xlarge", "m5d.large", "m5d.xlarge", "m5d.2xlarge", "m5d.4xlarge", "m5d.8xlarge", "m5d.12xlarge", "m5d.16xlarge", "m5d.24xlarge", "m5d.metal", "m5zn.large", "m5zn.xlarge", "m5zn.2xlarge", "m5zn.3xlarge", "m5zn.6xlarge", "m5zn.12xlarge", "m5zn.metal", "m6g.medium", "m6g.large", "m6g.xlarge", "m6g.2xlarge", "m6g.4xlarge", "m6g.8xlarge", "m6g.12xlarge", "m6g.16xlarge", "m6g.metal", "m6i.large", "m6i.xlarge", "m6i.2xlarge", "m6i.4xlarge", "m6i.8xlarge", "m6i.12xlarge", "m6i.16xlarge", "m6i.24xlarge", "m6i.32xlarge", "m6i.metal", "mac1.metal", "p2.xlarge", "p2.8xlarge", "p2.16xlarge", "p3.2xlarge", "p3.8xlarge", "p3.16xlarge", "p4d.24xlarge", "r3.large", "r3.xlarge", "r3.2xlarge", "r3.4xlarge", "r3.8xlarge", "r4.large", "r4.xlarge", "r4.2xlarge", "r4.4xlarge", "r4.8xlarge", "r4.16xlarge", "r5.large", "r5.xlarge", "r5.2xlarge", "r5.4xlarge", "r5.8xlarge", "r5.12xlarge", "r5.16xlarge", "r5.24xlarge", "r5.metal", "r5a.large", "r5a.xlarge", "r5a.2xlarge", "r5a.4xlarge", "r5a.8xlarge", "r5a.12xlarge", "r5a.16xlarge", "r5a.24xlarge", "r5ad.large", "r5ad.xlarge", "r5ad.2xlarge", "r5ad.4xlarge", "r5ad.8xlarge", "r5ad.12xlarge", "r5ad.16xlarge", "r5ad.24xlarge", "r5b.large", "r5b.xlarge", "r5b.2xlarge", "r5b.4xlarge", "r5b.8xlarge", "r5b.12xlarge", "r5b.16xlarge", "r5b.24xlarge", "r5b.metal", "r5d.large", "r5d.xlarge", "r5d.2xlarge", "r5d.4xlarge", "r5d.8xlarge", "r5d.12xlarge", "r5d.16xlarge", "r5d.24xlarge", "r5d.metal", "r5dn.large", "r5dn.xlarge", "r5dn.2xlarge", "r5dn.4xlarge", "r5dn.8xlarge", "r5dn.12xlarge", "r5dn.16xlarge", "r5dn.24xlarge", "r5dn.metal", "r5n.large", "r5n.xlarge", "r5n.2xlarge", "r5n.4xlarge", "r5n.8xlarge", "r5n.12xlarge", "r5n.16xlarge", "r5n.24xlarge", "r5n.metal", "r6g.medium", "r6g.large", "r6g.xlarge", "r6g.2xlarge", "r6g.4xlarge", "r6g.8xlarge", "r6g.12xlarge", "r6g.16xlarge", "r6g.metal", "r6i.large", "r6i.xlarge", "r6i.2xlarge", "r6i.4xlarge", "r6i.8xlarge", "r6i.12xlarge", "r6i.16xlarge", "r6i.24xlarge", "r6i.32xlarge", "r6i.metal", "t2.nano", "t2.micro", "t2.small", "t2.medium", "t2.large", "t2.xlarge", "t2.2xlarge", "t3.nano", "t3.micro", "t3.small", "t3.medium", "t3.large", "t3.xlarge", "t3.2xlarge", "t3a.nano", "t3a.micro", "t3a.small", "t3a.medium", "t3a.large", "t3a.xlarge", "t3a.2xlarge", "t4g.nano", "t4g.micro", "t4g.small", "t4g.medium", "t4g.large", "t4g.xlarge", "t4g.2xlarge", "u-12tb1.metal", "u-6tb1.56xlarge", "u-6tb1.112xlarge", "u-6tb1.metal", "u-9tb1.metal", "x1.16xlarge", "x1.32xlarge", "x1e.xlarge", "x1e.2xlarge", "x1e.4xlarge", "x1e.8xlarge", "x1e.16xlarge", "x1e.32xlarge", "z1d.large", "z1d.xlarge", "z1d.2xlarge", "z1d.3xlarge", "z1d.6xlarge", "z1d.12xlarge", "z1d.metal", ), "ap-northeast-3": strset.New( "c3.large", "c3.xlarge", "c3.2xlarge", "c3.4xlarge", "c3.8xlarge", "c4.large", "c4.xlarge", "c4.2xlarge", "c4.4xlarge", "c4.8xlarge", "c5.large", "c5.xlarge", "c5.2xlarge", "c5.4xlarge", "c5.9xlarge", "c5.12xlarge", "c5.18xlarge", "c5.24xlarge", "c5.metal", "c5d.large", "c5d.xlarge", "c5d.2xlarge", "c5d.4xlarge", "c5d.9xlarge", "c5d.12xlarge", "c5d.18xlarge", "c5d.24xlarge", "c5d.metal", "d2.xlarge", "d2.2xlarge", "d2.4xlarge", "d2.8xlarge", "g4dn.xlarge", "g4dn.2xlarge", "g4dn.4xlarge", "g4dn.8xlarge", "g4dn.12xlarge", "g4dn.16xlarge", "g4dn.metal", "i3.large", "i3.xlarge", "i3.2xlarge", "i3.4xlarge", "i3.8xlarge", "i3.16xlarge", "i3.metal", "i3en.large", "i3en.xlarge", "i3en.2xlarge", "i3en.3xlarge", "i3en.6xlarge", "i3en.12xlarge", "i3en.24xlarge", "i3en.metal", "m3.medium", "m3.large", "m3.xlarge", "m3.2xlarge", "m4.large", "m4.xlarge", "m4.2xlarge", "m4.4xlarge", "m4.10xlarge", "m4.16xlarge", "m5.large", "m5.xlarge", "m5.2xlarge", "m5.4xlarge", "m5.8xlarge", "m5.12xlarge", "m5.16xlarge", "m5.24xlarge", "m5.metal", "m5d.large", "m5d.xlarge", "m5d.2xlarge", "m5d.4xlarge", "m5d.8xlarge", "m5d.12xlarge", "m5d.16xlarge", "m5d.24xlarge", "m5d.metal", "r3.large", "r3.xlarge", "r3.2xlarge", "r3.4xlarge", "r3.8xlarge", "r4.large", "r4.xlarge", "r4.2xlarge", "r4.4xlarge", "r4.8xlarge", "r4.16xlarge", "r5.large", "r5.xlarge", "r5.2xlarge", "r5.4xlarge", "r5.8xlarge", "r5.12xlarge", "r5.16xlarge", "r5.24xlarge", "r5.metal", "r5d.large", "r5d.xlarge", "r5d.2xlarge", "r5d.4xlarge", "r5d.8xlarge", "r5d.12xlarge", "r5d.16xlarge", "r5d.24xlarge", "r5d.metal", "t2.nano", "t2.micro", "t2.small", "t2.medium", "t2.large", "t2.xlarge", "t2.2xlarge", "t3.nano", "t3.micro", "t3.small", "t3.medium", "t3.large", "t3.xlarge", "t3.2xlarge", "u-12tb1.metal", "u-6tb1.metal", "u-9tb1.metal", "x1.16xlarge", "x1.32xlarge", "x1e.xlarge", "x1e.2xlarge", "x1e.4xlarge", "x1e.8xlarge", "x1e.16xlarge", "x1e.32xlarge", ), "ap-south-1": strset.New( "a1.medium", "a1.large", "a1.xlarge", "a1.2xlarge", "a1.4xlarge", "a1.metal", "c4.large", "c4.xlarge", "c4.2xlarge", "c4.4xlarge", "c4.8xlarge", "c5.large", "c5.xlarge", "c5.2xlarge", "c5.4xlarge", "c5.9xlarge", "c5.12xlarge", "c5.18xlarge", "c5.24xlarge", "c5.metal", "c5a.large", "c5a.xlarge", "c5a.2xlarge", "c5a.4xlarge", "c5a.8xlarge", "c5a.12xlarge", "c5a.16xlarge", "c5a.24xlarge", "c5d.large", "c5d.xlarge", "c5d.2xlarge", "c5d.4xlarge", "c5d.9xlarge", "c5d.12xlarge", "c5d.18xlarge", "c5d.24xlarge", "c5d.metal", "c5n.large", "c5n.xlarge", "c5n.2xlarge", "c5n.4xlarge", "c5n.9xlarge", "c5n.18xlarge", "c5n.metal", "c6g.medium", "c6g.large", "c6g.xlarge", "c6g.2xlarge", "c6g.4xlarge", "c6g.8xlarge", "c6g.12xlarge", "c6g.16xlarge", "c6g.metal", "c6gd.medium", "c6gd.large", "c6gd.xlarge", "c6gd.2xlarge", "c6gd.4xlarge", "c6gd.8xlarge", "c6gd.12xlarge", "c6gd.16xlarge", "c6gd.metal", "c6gn.medium", "c6gn.large", "c6gn.xlarge", "c6gn.2xlarge", "c6gn.4xlarge", "c6gn.8xlarge", "c6gn.12xlarge", "c6gn.16xlarge", "c6i.large", "c6i.xlarge", "c6i.2xlarge", "c6i.4xlarge", "c6i.8xlarge", "c6i.12xlarge", "c6i.16xlarge", "c6i.24xlarge", "c6i.32xlarge", "c6i.metal", "d2.xlarge", "d2.2xlarge", "d2.4xlarge", "d2.8xlarge", "d3.xlarge", "d3.2xlarge", "d3.4xlarge", "d3.8xlarge", "g4dn.xlarge", "g4dn.2xlarge", "g4dn.4xlarge", "g4dn.8xlarge", "g4dn.12xlarge", "g4dn.16xlarge", "g4dn.metal", "i2.xlarge", "i2.2xlarge", "i2.4xlarge", "i2.8xlarge", "i3.large", "i3.xlarge", "i3.2xlarge", "i3.4xlarge", "i3.8xlarge", "i3.16xlarge", "i3.metal", "i3en.large", "i3en.xlarge", "i3en.2xlarge", "i3en.3xlarge", "i3en.6xlarge", "i3en.12xlarge", "i3en.24xlarge", "i3en.metal", "inf1.xlarge", "inf1.2xlarge", "inf1.6xlarge", "inf1.24xlarge", "m4.large", "m4.xlarge", "m4.2xlarge", "m4.4xlarge", "m4.10xlarge", "m4.16xlarge", "m5.large", "m5.xlarge", "m5.2xlarge", "m5.4xlarge", "m5.8xlarge", "m5.12xlarge", "m5.16xlarge", "m5.24xlarge", "m5.metal", "m5a.large", "m5a.xlarge", "m5a.2xlarge", "m5a.4xlarge", "m5a.8xlarge", "m5a.12xlarge", "m5a.16xlarge", "m5a.24xlarge", "m5ad.large", "m5ad.xlarge", "m5ad.2xlarge", "m5ad.4xlarge", "m5ad.8xlarge", "m5ad.12xlarge", "m5ad.16xlarge", "m5ad.24xlarge", "m5d.large", "m5d.xlarge", "m5d.2xlarge", "m5d.4xlarge", "m5d.8xlarge", "m5d.12xlarge", "m5d.16xlarge", "m5d.24xlarge", "m5d.metal", "m6a.large", "m6a.xlarge", "m6a.2xlarge", "m6a.4xlarge", "m6a.8xlarge", "m6a.12xlarge", "m6a.16xlarge", "m6a.24xlarge", "m6a.32xlarge", "m6a.48xlarge", "m6a.metal", "m6g.medium", "m6g.large", "m6g.xlarge", "m6g.2xlarge", "m6g.4xlarge", "m6g.8xlarge", "m6g.12xlarge", "m6g.16xlarge", "m6g.metal", "m6gd.medium", "m6gd.large", "m6gd.xlarge", "m6gd.2xlarge", "m6gd.4xlarge", "m6gd.8xlarge", "m6gd.12xlarge", "m6gd.16xlarge", "m6gd.metal", "m6i.large", "m6i.xlarge", "m6i.2xlarge", "m6i.4xlarge", "m6i.8xlarge", "m6i.12xlarge", "m6i.16xlarge", "m6i.24xlarge", "m6i.32xlarge", "m6i.metal", "mac1.metal", "p2.xlarge", "p2.8xlarge", "p2.16xlarge", "r3.large", "r3.xlarge", "r3.2xlarge", "r3.4xlarge", "r3.8xlarge", "r4.large", "r4.xlarge", "r4.2xlarge", "r4.4xlarge", "r4.8xlarge", "r4.16xlarge", "r5.large", "r5.xlarge", "r5.2xlarge", "r5.4xlarge", "r5.8xlarge", "r5.12xlarge", "r5.16xlarge", "r5.24xlarge", "r5.metal", "r5a.large", "r5a.xlarge", "r5a.2xlarge", "r5a.4xlarge", "r5a.8xlarge", "r5a.12xlarge", "r5a.16xlarge", "r5a.24xlarge", "r5ad.large", "r5ad.xlarge", "r5ad.2xlarge", "r5ad.4xlarge", "r5ad.8xlarge", "r5ad.12xlarge", "r5ad.16xlarge", "r5ad.24xlarge", "r5d.large", "r5d.xlarge", "r5d.2xlarge", "r5d.4xlarge", "r5d.8xlarge", "r5d.12xlarge", "r5d.16xlarge", "r5d.24xlarge", "r5d.metal", "r5n.large", "r5n.xlarge", "r5n.2xlarge", "r5n.4xlarge", "r5n.8xlarge", "r5n.12xlarge", "r5n.16xlarge", "r5n.24xlarge", "r5n.metal", "r6g.medium", "r6g.large", "r6g.xlarge", "r6g.2xlarge", "r6g.4xlarge", "r6g.8xlarge", "r6g.12xlarge", "r6g.16xlarge", "r6g.metal", "r6gd.medium", "r6gd.large", "r6gd.xlarge", "r6gd.2xlarge", "r6gd.4xlarge", "r6gd.8xlarge", "r6gd.12xlarge", "r6gd.16xlarge", "r6gd.metal", "r6i.large", "r6i.xlarge", "r6i.2xlarge", "r6i.4xlarge", "r6i.8xlarge", "r6i.12xlarge", "r6i.16xlarge", "r6i.24xlarge", "r6i.32xlarge", "r6i.metal", "t2.nano", "t2.micro", "t2.small", "t2.medium", "t2.large", "t2.xlarge", "t2.2xlarge", "t3.nano", "t3.micro", "t3.small", "t3.medium", "t3.large", "t3.xlarge", "t3.2xlarge", "t3a.nano", "t3a.micro", "t3a.small", "t3a.medium", "t3a.large", "t3a.xlarge", "t3a.2xlarge", "t4g.nano", "t4g.micro", "t4g.small", "t4g.medium", "t4g.large", "t4g.xlarge", "t4g.2xlarge", "u-12tb1.metal", "u-6tb1.56xlarge", "u-6tb1.112xlarge", "u-6tb1.metal", "u-9tb1.metal", "x1.16xlarge", "x1.32xlarge", "x1e.xlarge", "x1e.2xlarge", "x1e.4xlarge", "x1e.8xlarge", "x1e.16xlarge", "x1e.32xlarge", "x2idn.16xlarge", "x2idn.24xlarge", "x2idn.32xlarge", "z1d.large", "z1d.xlarge", "z1d.2xlarge", "z1d.3xlarge", "z1d.6xlarge", "z1d.12xlarge", "z1d.metal", ), "ap-southeast-1": strset.New( "a1.medium", "a1.large", "a1.xlarge", "a1.2xlarge", "a1.4xlarge", "a1.metal", "c1.medium", "c1.xlarge", "c3.large", "c3.xlarge", "c3.2xlarge", "c3.4xlarge", "c3.8xlarge", "c4.large", "c4.xlarge", "c4.2xlarge", "c4.4xlarge", "c4.8xlarge", "c5.large", "c5.xlarge", "c5.2xlarge", "c5.4xlarge", "c5.9xlarge", "c5.12xlarge", "c5.18xlarge", "c5.24xlarge", "c5.metal", "c5a.large", "c5a.xlarge", "c5a.2xlarge", "c5a.4xlarge", "c5a.8xlarge", "c5a.12xlarge", "c5a.16xlarge", "c5a.24xlarge", "c5ad.large", "c5ad.xlarge", "c5ad.2xlarge", "c5ad.4xlarge", "c5ad.8xlarge", "c5ad.12xlarge", "c5ad.16xlarge", "c5ad.24xlarge", "c5d.large", "c5d.xlarge", "c5d.2xlarge", "c5d.4xlarge", "c5d.9xlarge", "c5d.12xlarge", "c5d.18xlarge", "c5d.24xlarge", "c5d.metal", "c5n.large", "c5n.xlarge", "c5n.2xlarge", "c5n.4xlarge", "c5n.9xlarge", "c5n.18xlarge", "c5n.metal", "c6g.medium", "c6g.large", "c6g.xlarge", "c6g.2xlarge", "c6g.4xlarge", "c6g.8xlarge", "c6g.12xlarge", "c6g.16xlarge", "c6g.metal", "c6gd.medium", "c6gd.large", "c6gd.xlarge", "c6gd.2xlarge", "c6gd.4xlarge", "c6gd.8xlarge", "c6gd.12xlarge", "c6gd.16xlarge", "c6gd.metal", "c6gn.medium", "c6gn.large", "c6gn.xlarge", "c6gn.2xlarge", "c6gn.4xlarge", "c6gn.8xlarge", "c6gn.12xlarge", "c6gn.16xlarge", "c6i.large", "c6i.xlarge", "c6i.2xlarge", "c6i.4xlarge", "c6i.8xlarge", "c6i.12xlarge", "c6i.16xlarge", "c6i.24xlarge", "c6i.32xlarge", "c6i.metal", "d2.xlarge", "d2.2xlarge", "d2.4xlarge", "d2.8xlarge", "d3.xlarge", "d3.2xlarge", "d3.4xlarge", "d3.8xlarge", "g2.2xlarge", "g2.8xlarge", "g3.4xlarge", "g3.8xlarge", "g3.16xlarge", "g4dn.xlarge", "g4dn.2xlarge", "g4dn.4xlarge", "g4dn.8xlarge", "g4dn.12xlarge", "g4dn.16xlarge", "g4dn.metal", "g5g.xlarge", "g5g.2xlarge", "g5g.4xlarge", "g5g.8xlarge", "g5g.16xlarge", "g5g.metal", "hs1.8xlarge", "i2.xlarge", "i2.2xlarge", "i2.4xlarge", "i2.8xlarge", "i3.large", "i3.xlarge", "i3.2xlarge", "i3.4xlarge", "i3.8xlarge", "i3.16xlarge", "i3.metal", "i3en.large", "i3en.xlarge", "i3en.2xlarge", "i3en.3xlarge", "i3en.6xlarge", "i3en.12xlarge", "i3en.24xlarge", "i3en.metal", "inf1.xlarge", "inf1.2xlarge", "inf1.6xlarge", "inf1.24xlarge", "m1.small", "m1.medium", "m1.large", "m1.xlarge", "m2.xlarge", "m2.2xlarge", "m2.4xlarge", "m3.medium", "m3.large", "m3.xlarge", "m3.2xlarge", "m4.large", "m4.xlarge", "m4.2xlarge", "m4.4xlarge", "m4.10xlarge", "m4.16xlarge", "m5.large", "m5.xlarge", "m5.2xlarge", "m5.4xlarge", "m5.8xlarge", "m5.12xlarge", "m5.16xlarge", "m5.24xlarge", "m5.metal", "m5a.large", "m5a.xlarge", "m5a.2xlarge", "m5a.4xlarge", "m5a.8xlarge", "m5a.12xlarge", "m5a.16xlarge", "m5a.24xlarge", "m5ad.large", "m5ad.xlarge", "m5ad.2xlarge", "m5ad.4xlarge", "m5ad.8xlarge", "m5ad.12xlarge", "m5ad.16xlarge", "m5ad.24xlarge", "m5d.large", "m5d.xlarge", "m5d.2xlarge", "m5d.4xlarge", "m5d.8xlarge", "m5d.12xlarge", "m5d.16xlarge", "m5d.24xlarge", "m5d.metal", "m5dn.large", "m5dn.xlarge", "m5dn.2xlarge", "m5dn.4xlarge", "m5dn.8xlarge", "m5dn.12xlarge", "m5dn.16xlarge", "m5dn.24xlarge", "m5dn.metal", "m5n.large", "m5n.xlarge", "m5n.2xlarge", "m5n.4xlarge", "m5n.8xlarge", "m5n.12xlarge", "m5n.16xlarge", "m5n.24xlarge", "m5n.metal", "m5zn.large", "m5zn.xlarge", "m5zn.2xlarge", "m5zn.3xlarge", "m5zn.6xlarge", "m5zn.12xlarge", "m5zn.metal", "m6g.medium", "m6g.large", "m6g.xlarge", "m6g.2xlarge", "m6g.4xlarge", "m6g.8xlarge", "m6g.12xlarge", "m6g.16xlarge", "m6g.metal", "m6gd.medium", "m6gd.large", "m6gd.xlarge", "m6gd.2xlarge", "m6gd.4xlarge", "m6gd.8xlarge", "m6gd.12xlarge", "m6gd.16xlarge", "m6gd.metal", "m6i.large", "m6i.xlarge", "m6i.2xlarge", "m6i.4xlarge", "m6i.8xlarge", "m6i.12xlarge", "m6i.16xlarge", "m6i.24xlarge", "m6i.32xlarge", "m6i.metal", "mac1.metal", "p2.xlarge", "p2.8xlarge", "p2.16xlarge", "p3.2xlarge", "p3.8xlarge", "p3.16xlarge", "r3.large", "r3.xlarge", "r3.2xlarge", "r3.4xlarge", "r3.8xlarge", "r4.large", "r4.xlarge", "r4.2xlarge", "r4.4xlarge", "r4.8xlarge", "r4.16xlarge", "r5.large", "r5.xlarge", "r5.2xlarge", "r5.4xlarge", "r5.8xlarge", "r5.12xlarge", "r5.16xlarge", "r5.24xlarge", "r5.metal", "r5a.large", "r5a.xlarge", "r5a.2xlarge", "r5a.4xlarge", "r5a.8xlarge", "r5a.12xlarge", "r5a.16xlarge", "r5a.24xlarge", "r5ad.large", "r5ad.xlarge", "r5ad.2xlarge", "r5ad.4xlarge", "r5ad.8xlarge", "r5ad.12xlarge", "r5ad.16xlarge", "r5ad.24xlarge", "r5b.large", "r5b.xlarge", "r5b.2xlarge", "r5b.4xlarge", "r5b.8xlarge", "r5b.12xlarge", "r5b.16xlarge", "r5b.24xlarge", "r5b.metal", "r5d.large", "r5d.xlarge", "r5d.2xlarge", "r5d.4xlarge", "r5d.8xlarge", "r5d.12xlarge", "r5d.16xlarge", "r5d.24xlarge", "r5d.metal", "r5dn.large", "r5dn.xlarge", "r5dn.2xlarge", "r5dn.4xlarge", "r5dn.8xlarge", "r5dn.12xlarge", "r5dn.16xlarge", "r5dn.24xlarge", "r5dn.metal", "r5n.large", "r5n.xlarge", "r5n.2xlarge", "r5n.4xlarge", "r5n.8xlarge", "r5n.12xlarge", "r5n.16xlarge", "r5n.24xlarge", "r5n.metal", "r6g.medium", "r6g.large", "r6g.xlarge", "r6g.2xlarge", "r6g.4xlarge", "r6g.8xlarge", "r6g.12xlarge", "r6g.16xlarge", "r6g.metal", "r6gd.medium", "r6gd.large", "r6gd.xlarge", "r6gd.2xlarge", "r6gd.4xlarge", "r6gd.8xlarge", "r6gd.12xlarge", "r6gd.16xlarge", "r6gd.metal", "r6i.large", "r6i.xlarge", "r6i.2xlarge", "r6i.4xlarge", "r6i.8xlarge", "r6i.12xlarge", "r6i.16xlarge", "r6i.24xlarge", "r6i.32xlarge", "r6i.metal", "t1.micro", "t2.nano", "t2.micro", "t2.small", "t2.medium", "t2.large", "t2.xlarge", "t2.2xlarge", "t3.nano", "t3.micro", "t3.small", "t3.medium", "t3.large", "t3.xlarge", "t3.2xlarge", "t3a.nano", "t3a.micro", "t3a.small", "t3a.medium", "t3a.large", "t3a.xlarge", "t3a.2xlarge", "t4g.nano", "t4g.micro", "t4g.small", "t4g.medium", "t4g.large", "t4g.xlarge", "t4g.2xlarge", "u-12tb1.112xlarge", "u-12tb1.metal", "u-6tb1.56xlarge", "u-6tb1.112xlarge", "u-6tb1.metal", "u-9tb1.112xlarge", "u-9tb1.metal", "x1.16xlarge", "x1.32xlarge", "x1e.xlarge", "x1e.2xlarge", "x1e.4xlarge", "x1e.8xlarge", "x1e.16xlarge", "x1e.32xlarge", "x2idn.16xlarge", "x2idn.24xlarge", "x2idn.32xlarge", "x2iedn.xlarge", "x2iedn.2xlarge", "x2iedn.4xlarge", "x2iedn.8xlarge", "x2iedn.16xlarge", "x2iedn.24xlarge", "x2iedn.32xlarge", "z1d.large", "z1d.xlarge", "z1d.2xlarge", "z1d.3xlarge", "z1d.6xlarge", "z1d.12xlarge", "z1d.metal", ), "ap-southeast-2": strset.New( "a1.medium", "a1.large", "a1.xlarge", "a1.2xlarge", "a1.4xlarge", "a1.metal", "c1.medium", "c1.xlarge", "c3.large", "c3.xlarge", "c3.2xlarge", "c3.4xlarge", "c3.8xlarge", "c4.large", "c4.xlarge", "c4.2xlarge", "c4.4xlarge", "c4.8xlarge", "c5.large", "c5.xlarge", "c5.2xlarge", "c5.4xlarge", "c5.9xlarge", "c5.12xlarge", "c5.18xlarge", "c5.24xlarge", "c5.metal", "c5a.large", "c5a.xlarge", "c5a.2xlarge", "c5a.4xlarge", "c5a.8xlarge", "c5a.12xlarge", "c5a.16xlarge", "c5a.24xlarge", "c5ad.large", "c5ad.xlarge", "c5ad.2xlarge", "c5ad.4xlarge", "c5ad.8xlarge", "c5ad.12xlarge", "c5ad.16xlarge", "c5ad.24xlarge", "c5d.large", "c5d.xlarge", "c5d.2xlarge", "c5d.4xlarge", "c5d.9xlarge", "c5d.12xlarge", "c5d.18xlarge", "c5d.24xlarge", "c5d.metal", "c5n.large", "c5n.xlarge", "c5n.2xlarge", "c5n.4xlarge", "c5n.9xlarge", "c5n.18xlarge", "c5n.metal", "c6g.medium", "c6g.large", "c6g.xlarge", "c6g.2xlarge", "c6g.4xlarge", "c6g.8xlarge", "c6g.12xlarge", "c6g.16xlarge", "c6g.metal", "c6gd.medium", "c6gd.large", "c6gd.xlarge", "c6gd.2xlarge", "c6gd.4xlarge", "c6gd.8xlarge", "c6gd.12xlarge", "c6gd.16xlarge", "c6gd.metal", "c6gn.medium", "c6gn.large", "c6gn.xlarge", "c6gn.2xlarge", "c6gn.4xlarge", "c6gn.8xlarge", "c6gn.12xlarge", "c6gn.16xlarge", "c6i.large", "c6i.xlarge", "c6i.2xlarge", "c6i.4xlarge", "c6i.8xlarge", "c6i.12xlarge", "c6i.16xlarge", "c6i.24xlarge", "c6i.32xlarge", "c6i.metal", "d2.xlarge", "d2.2xlarge", "d2.4xlarge", "d2.8xlarge", "d3.xlarge", "d3.2xlarge", "d3.4xlarge", "d3.8xlarge", "f1.2xlarge", "f1.4xlarge", "f1.16xlarge", "g2.2xlarge", "g2.8xlarge", "g3.4xlarge", "g3.8xlarge", "g3.16xlarge", "g3s.xlarge", "g4dn.xlarge", "g4dn.2xlarge", "g4dn.4xlarge", "g4dn.8xlarge", "g4dn.12xlarge", "g4dn.16xlarge", "g4dn.metal", "hs1.8xlarge", "i2.xlarge", "i2.2xlarge", "i2.4xlarge", "i2.8xlarge", "i3.large", "i3.xlarge", "i3.2xlarge", "i3.4xlarge", "i3.8xlarge", "i3.16xlarge", "i3.metal", "i3en.large", "i3en.xlarge", "i3en.2xlarge", "i3en.3xlarge", "i3en.6xlarge", "i3en.12xlarge", "i3en.24xlarge", "i3en.metal", "inf1.xlarge", "inf1.2xlarge", "inf1.6xlarge", "inf1.24xlarge", "m1.small", "m1.medium", "m1.large", "m1.xlarge", "m2.xlarge", "m2.2xlarge", "m2.4xlarge", "m3.medium", "m3.large", "m3.xlarge", "m3.2xlarge", "m4.large", "m4.xlarge", "m4.2xlarge", "m4.4xlarge", "m4.10xlarge", "m4.16xlarge", "m5.large", "m5.xlarge", "m5.2xlarge", "m5.4xlarge", "m5.8xlarge", "m5.12xlarge", "m5.16xlarge", "m5.24xlarge", "m5.metal", "m5a.large", "m5a.xlarge", "m5a.2xlarge", "m5a.4xlarge", "m5a.8xlarge", "m5a.12xlarge", "m5a.16xlarge", "m5a.24xlarge", "m5ad.large", "m5ad.xlarge", "m5ad.2xlarge", "m5ad.4xlarge", "m5ad.8xlarge", "m5ad.12xlarge", "m5ad.16xlarge", "m5ad.24xlarge", "m5d.large", "m5d.xlarge", "m5d.2xlarge", "m5d.4xlarge", "m5d.8xlarge", "m5d.12xlarge", "m5d.16xlarge", "m5d.24xlarge", "m5d.metal", "m5zn.large", "m5zn.xlarge", "m5zn.2xlarge", "m5zn.3xlarge", "m5zn.6xlarge", "m5zn.12xlarge", "m5zn.metal", "m6g.medium", "m6g.large", "m6g.xlarge", "m6g.2xlarge", "m6g.4xlarge", "m6g.8xlarge", "m6g.12xlarge", "m6g.16xlarge", "m6g.metal", "m6gd.medium", "m6gd.large", "m6gd.xlarge", "m6gd.2xlarge", "m6gd.4xlarge", "m6gd.8xlarge", "m6gd.12xlarge", "m6gd.16xlarge", "m6gd.metal", "m6i.large", "m6i.xlarge", "m6i.2xlarge", "m6i.4xlarge", "m6i.8xlarge", "m6i.12xlarge", "m6i.16xlarge", "m6i.24xlarge", "m6i.32xlarge", "m6i.metal", "mac1.metal", "p2.xlarge", "p2.8xlarge", "p2.16xlarge", "p3.2xlarge", "p3.8xlarge", "p3.16xlarge", "r3.large", "r3.xlarge", "r3.2xlarge", "r3.4xlarge", "r3.8xlarge", "r4.large", "r4.xlarge", "r4.2xlarge", "r4.4xlarge", "r4.8xlarge", "r4.16xlarge", "r5.large", "r5.xlarge", "r5.2xlarge", "r5.4xlarge", "r5.8xlarge", "r5.12xlarge", "r5.16xlarge", "r5.24xlarge", "r5.metal", "r5a.large", "r5a.xlarge", "r5a.2xlarge", "r5a.4xlarge", "r5a.8xlarge", "r5a.12xlarge", "r5a.16xlarge", "r5a.24xlarge", "r5ad.large", "r5ad.xlarge", "r5ad.2xlarge", "r5ad.4xlarge", "r5ad.8xlarge", "r5ad.12xlarge", "r5ad.16xlarge", "r5ad.24xlarge", "r5d.large", "r5d.xlarge", "r5d.2xlarge", "r5d.4xlarge", "r5d.8xlarge", "r5d.12xlarge", "r5d.16xlarge", "r5d.24xlarge", "r5d.metal", "r5dn.large", "r5dn.xlarge", "r5dn.2xlarge", "r5dn.4xlarge", "r5dn.8xlarge", "r5dn.12xlarge", "r5dn.16xlarge", "r5dn.24xlarge", "r5dn.metal", "r5n.large", "r5n.xlarge", "r5n.2xlarge", "r5n.4xlarge", "r5n.8xlarge", "r5n.12xlarge", "r5n.16xlarge", "r5n.24xlarge", "r5n.metal", "r6g.medium", "r6g.large", "r6g.xlarge", "r6g.2xlarge", "r6g.4xlarge", "r6g.8xlarge", "r6g.12xlarge", "r6g.16xlarge", "r6g.metal", "r6gd.medium", "r6gd.large", "r6gd.xlarge", "r6gd.2xlarge", "r6gd.4xlarge", "r6gd.8xlarge", "r6gd.12xlarge", "r6gd.16xlarge", "r6gd.metal", "r6i.large", "r6i.xlarge", "r6i.2xlarge", "r6i.4xlarge", "r6i.8xlarge", "r6i.12xlarge", "r6i.16xlarge", "r6i.24xlarge", "r6i.32xlarge", "r6i.metal", "t1.micro", "t2.nano", "t2.micro", "t2.small", "t2.medium", "t2.large", "t2.xlarge", "t2.2xlarge", "t3.nano", "t3.micro", "t3.small", "t3.medium", "t3.large", "t3.xlarge", "t3.2xlarge", "t3a.nano", "t3a.micro", "t3a.small", "t3a.medium", "t3a.large", "t3a.xlarge", "t3a.2xlarge", "t4g.nano", "t4g.micro", "t4g.small", "t4g.medium", "t4g.large", "t4g.xlarge", "t4g.2xlarge", "u-12tb1.metal", "u-6tb1.56xlarge", "u-6tb1.112xlarge", "u-6tb1.metal", "u-9tb1.metal", "x1.16xlarge", "x1.32xlarge", "x1e.xlarge", "x1e.2xlarge", "x1e.4xlarge", "x1e.8xlarge", "x1e.16xlarge", "x1e.32xlarge", "z1d.large", "z1d.xlarge", "z1d.2xlarge", "z1d.3xlarge", "z1d.6xlarge", "z1d.12xlarge", "z1d.metal", ), "ca-central-1": strset.New( "c4.large", "c4.xlarge", "c4.2xlarge", "c4.4xlarge", "c4.8xlarge", "c5.large", "c5.xlarge", "c5.2xlarge", "c5.4xlarge", "c5.9xlarge", "c5.12xlarge", "c5.18xlarge", "c5.24xlarge", "c5.metal", "c5a.large", "c5a.xlarge", "c5a.2xlarge", "c5a.4xlarge", "c5a.8xlarge", "c5a.12xlarge", "c5a.16xlarge", "c5a.24xlarge", "c5d.large", "c5d.xlarge", "c5d.2xlarge", "c5d.4xlarge", "c5d.9xlarge", "c5d.12xlarge", "c5d.18xlarge", "c5d.24xlarge", "c5d.metal", "c5n.large", "c5n.xlarge", "c5n.2xlarge", "c5n.4xlarge", "c5n.9xlarge", "c5n.18xlarge", "c5n.metal", "c6g.medium", "c6g.large", "c6g.xlarge", "c6g.2xlarge", "c6g.4xlarge", "c6g.8xlarge", "c6g.12xlarge", "c6g.16xlarge", "c6g.metal", "c6gd.medium", "c6gd.large", "c6gd.xlarge", "c6gd.2xlarge", "c6gd.4xlarge", "c6gd.8xlarge", "c6gd.12xlarge", "c6gd.16xlarge", "c6gd.metal", "c6gn.medium", "c6gn.large", "c6gn.xlarge", "c6gn.2xlarge", "c6gn.4xlarge", "c6gn.8xlarge", "c6gn.12xlarge", "c6gn.16xlarge", "c6i.large", "c6i.xlarge", "c6i.2xlarge", "c6i.4xlarge", "c6i.8xlarge", "c6i.12xlarge", "c6i.16xlarge", "c6i.24xlarge", "c6i.32xlarge", "c6i.metal", "d2.xlarge", "d2.2xlarge", "d2.4xlarge", "d2.8xlarge", "g3.4xlarge", "g3.8xlarge", "g3.16xlarge", "g4ad.xlarge", "g4ad.2xlarge", "g4ad.4xlarge", "g4ad.8xlarge", "g4ad.16xlarge", "g4dn.xlarge", "g4dn.2xlarge", "g4dn.4xlarge", "g4dn.8xlarge", "g4dn.12xlarge", "g4dn.16xlarge", "g4dn.metal", "i3.large", "i3.xlarge", "i3.2xlarge", "i3.4xlarge", "i3.8xlarge", "i3.16xlarge", "i3.metal", "i3en.large", "i3en.xlarge", "i3en.2xlarge", "i3en.3xlarge", "i3en.6xlarge", "i3en.12xlarge", "i3en.24xlarge", "i3en.metal", "inf1.xlarge", "inf1.2xlarge", "inf1.6xlarge", "inf1.24xlarge", "m4.large", "m4.xlarge", "m4.2xlarge", "m4.4xlarge", "m4.10xlarge", "m4.16xlarge", "m5.large", "m5.xlarge", "m5.2xlarge", "m5.4xlarge", "m5.8xlarge", "m5.12xlarge", "m5.16xlarge", "m5.24xlarge", "m5.metal", "m5a.large", "m5a.xlarge", "m5a.2xlarge", "m5a.4xlarge", "m5a.8xlarge", "m5a.12xlarge", "m5a.16xlarge", "m5a.24xlarge", "m5ad.large", "m5ad.xlarge", "m5ad.2xlarge", "m5ad.4xlarge", "m5ad.8xlarge", "m5ad.12xlarge", "m5ad.16xlarge", "m5ad.24xlarge", "m5d.large", "m5d.xlarge", "m5d.2xlarge", "m5d.4xlarge", "m5d.8xlarge", "m5d.12xlarge", "m5d.16xlarge", "m5d.24xlarge", "m5d.metal", "m6g.medium", "m6g.large", "m6g.xlarge", "m6g.2xlarge", "m6g.4xlarge", "m6g.8xlarge", "m6g.12xlarge", "m6g.16xlarge", "m6g.metal", "m6i.large", "m6i.xlarge", "m6i.2xlarge", "m6i.4xlarge", "m6i.8xlarge", "m6i.12xlarge", "m6i.16xlarge", "m6i.24xlarge", "m6i.32xlarge", "m6i.metal", "p3.2xlarge", "p3.8xlarge", "p3.16xlarge", "r4.large", "r4.xlarge", "r4.2xlarge", "r4.4xlarge", "r4.8xlarge", "r4.16xlarge", "r5.large", "r5.xlarge", "r5.2xlarge", "r5.4xlarge", "r5.8xlarge", "r5.12xlarge", "r5.16xlarge", "r5.24xlarge", "r5.metal", "r5a.large", "r5a.xlarge", "r5a.2xlarge", "r5a.4xlarge", "r5a.8xlarge", "r5a.12xlarge", "r5a.16xlarge", "r5a.24xlarge", "r5ad.large", "r5ad.xlarge", "r5ad.2xlarge", "r5ad.4xlarge", "r5ad.8xlarge", "r5ad.12xlarge", "r5ad.16xlarge", "r5ad.24xlarge", "r5d.large", "r5d.xlarge", "r5d.2xlarge", "r5d.4xlarge", "r5d.8xlarge", "r5d.12xlarge", "r5d.16xlarge", "r5d.24xlarge", "r5d.metal", "r5n.large", "r5n.xlarge", "r5n.2xlarge", "r5n.4xlarge", "r5n.8xlarge", "r5n.12xlarge", "r5n.16xlarge", "r5n.24xlarge", "r5n.metal", "r6g.medium", "r6g.large", "r6g.xlarge", "r6g.2xlarge", "r6g.4xlarge", "r6g.8xlarge", "r6g.12xlarge", "r6g.16xlarge", "r6g.metal", "r6gd.medium", "r6gd.large", "r6gd.xlarge", "r6gd.2xlarge", "r6gd.4xlarge", "r6gd.8xlarge", "r6gd.12xlarge", "r6gd.16xlarge", "r6gd.metal", "r6i.large", "r6i.xlarge", "r6i.2xlarge", "r6i.4xlarge", "r6i.8xlarge", "r6i.12xlarge", "r6i.16xlarge", "r6i.24xlarge", "r6i.32xlarge", "r6i.metal", "t2.nano", "t2.micro", "t2.small", "t2.medium", "t2.large", "t2.xlarge", "t2.2xlarge", "t3.nano", "t3.micro", "t3.small", "t3.medium", "t3.large", "t3.xlarge", "t3.2xlarge", "t3a.nano", "t3a.micro", "t3a.small", "t3a.medium", "t3a.large", "t3a.xlarge", "t3a.2xlarge", "t4g.nano", "t4g.micro", "t4g.small", "t4g.medium", "t4g.large", "t4g.xlarge", "t4g.2xlarge", "x1.16xlarge", "x1.32xlarge", "x1e.xlarge", "x1e.2xlarge", "x1e.4xlarge", "x1e.8xlarge", "x1e.16xlarge", "x1e.32xlarge", ), "eu-central-1": strset.New( "a1.medium", "a1.large", "a1.xlarge", "a1.2xlarge", "a1.4xlarge", "a1.metal", "c3.large", "c3.xlarge", "c3.2xlarge", "c3.4xlarge", "c3.8xlarge", "c4.large", "c4.xlarge", "c4.2xlarge", "c4.4xlarge", "c4.8xlarge", "c5.large", "c5.xlarge", "c5.2xlarge", "c5.4xlarge", "c5.9xlarge", "c5.12xlarge", "c5.18xlarge", "c5.24xlarge", "c5.metal", "c5a.large", "c5a.xlarge", "c5a.2xlarge", "c5a.4xlarge", "c5a.8xlarge", "c5a.12xlarge", "c5a.16xlarge", "c5a.24xlarge", "c5ad.large", "c5ad.xlarge", "c5ad.2xlarge", "c5ad.4xlarge", "c5ad.8xlarge", "c5ad.12xlarge", "c5ad.16xlarge", "c5ad.24xlarge", "c5d.large", "c5d.xlarge", "c5d.2xlarge", "c5d.4xlarge", "c5d.9xlarge", "c5d.12xlarge", "c5d.18xlarge", "c5d.24xlarge", "c5d.metal", "c5n.large", "c5n.xlarge", "c5n.2xlarge", "c5n.4xlarge", "c5n.9xlarge", "c5n.18xlarge", "c5n.metal", "c6g.medium", "c6g.large", "c6g.xlarge", "c6g.2xlarge", "c6g.4xlarge", "c6g.8xlarge", "c6g.12xlarge", "c6g.16xlarge", "c6g.metal", "c6gd.medium", "c6gd.large", "c6gd.xlarge", "c6gd.2xlarge", "c6gd.4xlarge", "c6gd.8xlarge", "c6gd.12xlarge", "c6gd.16xlarge", "c6gd.metal", "c6gn.medium", "c6gn.large", "c6gn.xlarge", "c6gn.2xlarge", "c6gn.4xlarge", "c6gn.8xlarge", "c6gn.12xlarge", "c6gn.16xlarge", "d2.xlarge", "d2.2xlarge", "d2.4xlarge", "d2.8xlarge", "d3.xlarge", "d3.2xlarge", "d3.4xlarge", "d3.8xlarge", "f1.2xlarge", "f1.4xlarge", "f1.16xlarge", "g2.2xlarge", "g2.8xlarge", "g3.4xlarge", "g3.8xlarge", "g3.16xlarge", "g3s.xlarge", "g4ad.xlarge", "g4ad.2xlarge", "g4ad.4xlarge", "g4ad.8xlarge", "g4ad.16xlarge", "g4dn.xlarge", "g4dn.2xlarge", "g4dn.4xlarge", "g4dn.8xlarge", "g4dn.12xlarge", "g4dn.16xlarge", "g4dn.metal", "i2.xlarge", "i2.2xlarge", "i2.4xlarge", "i2.8xlarge", "i3.large", "i3.xlarge", "i3.2xlarge", "i3.4xlarge", "i3.8xlarge", "i3.16xlarge", "i3.metal", "i3en.large", "i3en.xlarge", "i3en.2xlarge", "i3en.3xlarge", "i3en.6xlarge", "i3en.12xlarge", "i3en.24xlarge", "i3en.metal", "inf1.xlarge", "inf1.2xlarge", "inf1.6xlarge", "inf1.24xlarge", "m3.medium", "m3.large", "m3.xlarge", "m3.2xlarge", "m4.large", "m4.xlarge", "m4.2xlarge", "m4.4xlarge", "m4.10xlarge", "m4.16xlarge", "m5.large", "m5.xlarge", "m5.2xlarge", "m5.4xlarge", "m5.8xlarge", "m5.12xlarge", "m5.16xlarge", "m5.24xlarge", "m5.metal", "m5a.large", "m5a.xlarge", "m5a.2xlarge", "m5a.4xlarge", "m5a.8xlarge", "m5a.12xlarge", "m5a.16xlarge", "m5a.24xlarge", "m5ad.large", "m5ad.xlarge", "m5ad.2xlarge", "m5ad.4xlarge", "m5ad.8xlarge", "m5ad.12xlarge", "m5ad.16xlarge", "m5ad.24xlarge", "m5d.large", "m5d.xlarge", "m5d.2xlarge", "m5d.4xlarge", "m5d.8xlarge", "m5d.12xlarge", "m5d.16xlarge", "m5d.24xlarge", "m5d.metal", "m5dn.large", "m5dn.xlarge", "m5dn.2xlarge", "m5dn.4xlarge", "m5dn.8xlarge", "m5dn.12xlarge", "m5dn.16xlarge", "m5dn.24xlarge", "m5dn.metal", "m5n.large", "m5n.xlarge", "m5n.2xlarge", "m5n.4xlarge", "m5n.8xlarge", "m5n.12xlarge", "m5n.16xlarge", "m5n.24xlarge", "m5n.metal", "m5zn.large", "m5zn.xlarge", "m5zn.2xlarge", "m5zn.3xlarge", "m5zn.6xlarge", "m5zn.12xlarge", "m5zn.metal", "m6g.medium", "m6g.large", "m6g.xlarge", "m6g.2xlarge", "m6g.4xlarge", "m6g.8xlarge", "m6g.12xlarge", "m6g.16xlarge", "m6g.metal", "m6gd.medium", "m6gd.large", "m6gd.xlarge", "m6gd.2xlarge", "m6gd.4xlarge", "m6gd.8xlarge", "m6gd.12xlarge", "m6gd.16xlarge", "m6gd.metal", "m6i.large", "m6i.xlarge", "m6i.2xlarge", "m6i.4xlarge", "m6i.8xlarge", "m6i.12xlarge", "m6i.16xlarge", "m6i.24xlarge", "m6i.32xlarge", "m6i.metal", "mac1.metal", "p2.xlarge", "p2.8xlarge", "p2.16xlarge", "p3.2xlarge", "p3.8xlarge", "p3.16xlarge", "p4d.24xlarge", "r3.large", "r3.xlarge", "r3.2xlarge", "r3.4xlarge", "r3.8xlarge", "r4.large", "r4.xlarge", "r4.2xlarge", "r4.4xlarge", "r4.8xlarge", "r4.16xlarge", "r5.large", "r5.xlarge", "r5.2xlarge", "r5.4xlarge", "r5.8xlarge", "r5.12xlarge", "r5.16xlarge", "r5.24xlarge", "r5.metal", "r5a.large", "r5a.xlarge", "r5a.2xlarge", "r5a.4xlarge", "r5a.8xlarge", "r5a.12xlarge", "r5a.16xlarge", "r5a.24xlarge", "r5ad.large", "r5ad.xlarge", "r5ad.2xlarge", "r5ad.4xlarge", "r5ad.8xlarge", "r5ad.12xlarge", "r5ad.16xlarge", "r5ad.24xlarge", "r5b.large", "r5b.xlarge", "r5b.2xlarge", "r5b.4xlarge", "r5b.8xlarge", "r5b.12xlarge", "r5b.16xlarge", "r5b.24xlarge", "r5b.metal", "r5d.large", "r5d.xlarge", "r5d.2xlarge", "r5d.4xlarge", "r5d.8xlarge", "r5d.12xlarge", "r5d.16xlarge", "r5d.24xlarge", "r5d.metal", "r5dn.large", "r5dn.xlarge", "r5dn.2xlarge", "r5dn.4xlarge", "r5dn.8xlarge", "r5dn.12xlarge", "r5dn.16xlarge", "r5dn.24xlarge", "r5dn.metal", "r5n.large", "r5n.xlarge", "r5n.2xlarge", "r5n.4xlarge", "r5n.8xlarge", "r5n.12xlarge", "r5n.16xlarge", "r5n.24xlarge", "r5n.metal", "r6g.medium", "r6g.large", "r6g.xlarge", "r6g.2xlarge", "r6g.4xlarge", "r6g.8xlarge", "r6g.12xlarge", "r6g.16xlarge", "r6g.metal", "r6gd.medium", "r6gd.large", "r6gd.xlarge", "r6gd.2xlarge", "r6gd.4xlarge", "r6gd.8xlarge", "r6gd.12xlarge", "r6gd.16xlarge", "r6gd.metal", "t2.nano", "t2.micro", "t2.small", "t2.medium", "t2.large", "t2.xlarge", "t2.2xlarge", "t3.nano", "t3.micro", "t3.small", "t3.medium", "t3.large", "t3.xlarge", "t3.2xlarge", "t3a.nano", "t3a.micro", "t3a.small", "t3a.medium", "t3a.large", "t3a.xlarge", "t3a.2xlarge", "t4g.nano", "t4g.micro", "t4g.small", "t4g.medium", "t4g.large", "t4g.xlarge", "t4g.2xlarge", "u-12tb1.112xlarge", "u-12tb1.metal", "u-3tb1.56xlarge", "u-6tb1.56xlarge", "u-6tb1.112xlarge", "u-6tb1.metal", "u-9tb1.112xlarge", "u-9tb1.metal", "x1.16xlarge", "x1.32xlarge", "x1e.xlarge", "x1e.2xlarge", "x1e.4xlarge", "x1e.8xlarge", "x1e.16xlarge", "x1e.32xlarge", "x2idn.16xlarge", "x2idn.24xlarge", "x2idn.32xlarge", "x2iedn.xlarge", "x2iedn.2xlarge", "x2iedn.4xlarge", "x2iedn.8xlarge", "x2iedn.16xlarge", "x2iedn.24xlarge", "x2iedn.32xlarge", "z1d.large", "z1d.xlarge", "z1d.2xlarge", "z1d.3xlarge", "z1d.6xlarge", "z1d.12xlarge", "z1d.metal", ), "eu-north-1": strset.New( "c5.large", "c5.xlarge", "c5.2xlarge", "c5.4xlarge", "c5.9xlarge", "c5.12xlarge", "c5.18xlarge", "c5.24xlarge", "c5.metal", "c5a.large", "c5a.xlarge", "c5a.2xlarge", "c5a.4xlarge", "c5a.8xlarge", "c5a.12xlarge", "c5a.16xlarge", "c5a.24xlarge", "c5d.large", "c5d.xlarge", "c5d.2xlarge", "c5d.4xlarge", "c5d.9xlarge", "c5d.12xlarge", "c5d.18xlarge", "c5d.24xlarge", "c5d.metal", "c5n.large", "c5n.xlarge", "c5n.2xlarge", "c5n.4xlarge", "c5n.9xlarge", "c5n.18xlarge", "c5n.metal", "c6g.medium", "c6g.large", "c6g.xlarge", "c6g.2xlarge", "c6g.4xlarge", "c6g.8xlarge", "c6g.12xlarge", "c6g.16xlarge", "c6g.metal", "c6gn.medium", "c6gn.large", "c6gn.xlarge", "c6gn.2xlarge", "c6gn.4xlarge", "c6gn.8xlarge", "c6gn.12xlarge", "c6gn.16xlarge", "d2.xlarge", "d2.2xlarge", "d2.4xlarge", "d2.8xlarge", "g4dn.xlarge", "g4dn.2xlarge", "g4dn.4xlarge", "g4dn.8xlarge", "g4dn.12xlarge", "g4dn.16xlarge", "g4dn.metal", "i3.large", "i3.xlarge", "i3.2xlarge", "i3.4xlarge", "i3.8xlarge", "i3.16xlarge", "i3.metal", "i3en.large", "i3en.xlarge", "i3en.2xlarge", "i3en.3xlarge", "i3en.6xlarge", "i3en.12xlarge", "i3en.24xlarge", "i3en.metal", "inf1.xlarge", "inf1.2xlarge", "inf1.6xlarge", "inf1.24xlarge", "m5.large", "m5.xlarge", "m5.2xlarge", "m5.4xlarge", "m5.8xlarge", "m5.12xlarge", "m5.16xlarge", "m5.24xlarge", "m5.metal", "m5d.large", "m5d.xlarge", "m5d.2xlarge", "m5d.4xlarge", "m5d.8xlarge", "m5d.12xlarge", "m5d.16xlarge", "m5d.24xlarge", "m5d.metal", "m6g.medium", "m6g.large", "m6g.xlarge", "m6g.2xlarge", "m6g.4xlarge", "m6g.8xlarge", "m6g.12xlarge", "m6g.16xlarge", "m6g.metal", "mac1.metal", "r5.large", "r5.xlarge", "r5.2xlarge", "r5.4xlarge", "r5.8xlarge", "r5.12xlarge", "r5.16xlarge", "r5.24xlarge", "r5.metal", "r5d.large", "r5d.xlarge", "r5d.2xlarge", "r5d.4xlarge", "r5d.8xlarge", "r5d.12xlarge", "r5d.16xlarge", "r5d.24xlarge", "r5d.metal", "r5dn.large", "r5dn.xlarge", "r5dn.2xlarge", "r5dn.4xlarge", "r5dn.8xlarge", "r5dn.12xlarge", "r5dn.16xlarge", "r5dn.24xlarge", "r5dn.metal", "r5n.large", "r5n.xlarge", "r5n.2xlarge", "r5n.4xlarge", "r5n.8xlarge", "r5n.12xlarge", "r5n.16xlarge", "r5n.24xlarge", "r5n.metal", "r6g.medium", "r6g.large", "r6g.xlarge", "r6g.2xlarge", "r6g.4xlarge", "r6g.8xlarge", "r6g.12xlarge", "r6g.16xlarge", "r6g.metal", "t3.nano", "t3.micro", "t3.small", "t3.medium", "t3.large", "t3.xlarge", "t3.2xlarge", "t4g.nano", "t4g.micro", "t4g.small", "t4g.medium", "t4g.large", "t4g.xlarge", "t4g.2xlarge", "u-6tb1.56xlarge", "u-6tb1.112xlarge", ), "eu-south-1": strset.New( "c5.large", "c5.xlarge", "c5.2xlarge", "c5.4xlarge", "c5.9xlarge", "c5.12xlarge", "c5.18xlarge", "c5.24xlarge", "c5.metal", "c5a.large", "c5a.xlarge", "c5a.2xlarge", "c5a.4xlarge", "c5a.8xlarge", "c5a.12xlarge", "c5a.16xlarge", "c5a.24xlarge", "c5ad.large", "c5ad.xlarge", "c5ad.2xlarge", "c5ad.4xlarge", "c5ad.8xlarge", "c5ad.12xlarge", "c5ad.16xlarge", "c5ad.24xlarge", "c5d.large", "c5d.xlarge", "c5d.2xlarge", "c5d.4xlarge", "c5d.9xlarge", "c5d.12xlarge", "c5d.18xlarge", "c5d.24xlarge", "c5d.metal", "c5n.large", "c5n.xlarge", "c5n.2xlarge", "c5n.4xlarge", "c5n.9xlarge", "c5n.18xlarge", "c5n.metal", "c6g.medium", "c6g.large", "c6g.xlarge", "c6g.2xlarge", "c6g.4xlarge", "c6g.8xlarge", "c6g.12xlarge", "c6g.16xlarge", "c6g.metal", "d2.xlarge", "d2.2xlarge", "d2.4xlarge", "d2.8xlarge", "g4dn.xlarge", "g4dn.2xlarge", "g4dn.4xlarge", "g4dn.8xlarge", "g4dn.12xlarge", "g4dn.16xlarge", "g4dn.metal", "i3.large", "i3.xlarge", "i3.2xlarge", "i3.4xlarge", "i3.8xlarge", "i3.16xlarge", "i3.metal", "i3en.large", "i3en.xlarge", "i3en.2xlarge", "i3en.3xlarge", "i3en.6xlarge", "i3en.12xlarge", "i3en.24xlarge", "i3en.metal", "inf1.xlarge", "inf1.2xlarge", "inf1.6xlarge", "inf1.24xlarge", "m5.large", "m5.xlarge", "m5.2xlarge", "m5.4xlarge", "m5.8xlarge", "m5.12xlarge", "m5.16xlarge", "m5.24xlarge", "m5.metal", "m5a.large", "m5a.xlarge", "m5a.2xlarge", "m5a.4xlarge", "m5a.8xlarge", "m5a.12xlarge", "m5a.16xlarge", "m5a.24xlarge", "m5d.large", "m5d.xlarge", "m5d.2xlarge", "m5d.4xlarge", "m5d.8xlarge", "m5d.12xlarge", "m5d.16xlarge", "m5d.24xlarge", "m5d.metal", "m6g.medium", "m6g.large", "m6g.xlarge", "m6g.2xlarge", "m6g.4xlarge", "m6g.8xlarge", "m6g.12xlarge", "m6g.16xlarge", "m6g.metal", "r5.large", "r5.xlarge", "r5.2xlarge", "r5.4xlarge", "r5.8xlarge", "r5.12xlarge", "r5.16xlarge", "r5.24xlarge", "r5.metal", "r5a.large", "r5a.xlarge", "r5a.2xlarge", "r5a.4xlarge", "r5a.8xlarge", "r5a.12xlarge", "r5a.16xlarge", "r5a.24xlarge", "r5d.large", "r5d.xlarge", "r5d.2xlarge", "r5d.4xlarge", "r5d.8xlarge", "r5d.12xlarge", "r5d.16xlarge", "r5d.24xlarge", "r5d.metal", "r5dn.large", "r5dn.xlarge", "r5dn.2xlarge", "r5dn.4xlarge", "r5dn.8xlarge", "r5dn.12xlarge", "r5dn.16xlarge", "r5dn.24xlarge", "r5dn.metal", "r6g.medium", "r6g.large", "r6g.xlarge", "r6g.2xlarge", "r6g.4xlarge", "r6g.8xlarge", "r6g.12xlarge", "r6g.16xlarge", "r6g.metal", "t3.nano", "t3.micro", "t3.small", "t3.medium", "t3.large", "t3.xlarge", "t3.2xlarge", "t3a.nano", "t3a.micro", "t3a.small", "t3a.medium", "t3a.large", "t3a.xlarge", "t3a.2xlarge", "t4g.nano", "t4g.micro", "t4g.small", "t4g.medium", "t4g.large", "t4g.xlarge", "t4g.2xlarge", "u-3tb1.56xlarge", "u-6tb1.56xlarge", "u-6tb1.112xlarge", ), "eu-west-1": strset.New( "a1.medium", "a1.large", "a1.xlarge", "a1.2xlarge", "a1.4xlarge", "a1.metal", "c1.medium", "c1.xlarge", "c3.large", "c3.xlarge", "c3.2xlarge", "c3.4xlarge", "c3.8xlarge", "c4.large", "c4.xlarge", "c4.2xlarge", "c4.4xlarge", "c4.8xlarge", "c5.large", "c5.xlarge", "c5.2xlarge", "c5.4xlarge", "c5.9xlarge", "c5.12xlarge", "c5.18xlarge", "c5.24xlarge", "c5.metal", "c5a.large", "c5a.xlarge", "c5a.2xlarge", "c5a.4xlarge", "c5a.8xlarge", "c5a.12xlarge", "c5a.16xlarge", "c5a.24xlarge", "c5ad.large", "c5ad.xlarge", "c5ad.2xlarge", "c5ad.4xlarge", "c5ad.8xlarge", "c5ad.12xlarge", "c5ad.16xlarge", "c5ad.24xlarge", "c5d.large", "c5d.xlarge", "c5d.2xlarge", "c5d.4xlarge", "c5d.9xlarge", "c5d.12xlarge", "c5d.18xlarge", "c5d.24xlarge", "c5d.metal", "c5n.large", "c5n.xlarge", "c5n.2xlarge", "c5n.4xlarge", "c5n.9xlarge", "c5n.18xlarge", "c5n.metal", "c6a.large", "c6a.xlarge", "c6a.2xlarge", "c6a.4xlarge", "c6a.8xlarge", "c6a.12xlarge", "c6a.16xlarge", "c6a.24xlarge", "c6a.32xlarge", "c6a.48xlarge", "c6a.metal", "c6g.medium", "c6g.large", "c6g.xlarge", "c6g.2xlarge", "c6g.4xlarge", "c6g.8xlarge", "c6g.12xlarge", "c6g.16xlarge", "c6g.metal", "c6gd.medium", "c6gd.large", "c6gd.xlarge", "c6gd.2xlarge", "c6gd.4xlarge", "c6gd.8xlarge", "c6gd.12xlarge", "c6gd.16xlarge", "c6gd.metal", "c6gn.medium", "c6gn.large", "c6gn.xlarge", "c6gn.2xlarge", "c6gn.4xlarge", "c6gn.8xlarge", "c6gn.12xlarge", "c6gn.16xlarge", "c6gn.metal", "c6i.large", "c6i.xlarge", "c6i.2xlarge", "c6i.4xlarge", "c6i.8xlarge", "c6i.12xlarge", "c6i.16xlarge", "c6i.24xlarge", "c6i.32xlarge", "c6i.metal", "cc2.8xlarge", "cr1.8xlarge", "d2.xlarge", "d2.2xlarge", "d2.4xlarge", "d2.8xlarge", "d3.xlarge", "d3.2xlarge", "d3.4xlarge", "d3.8xlarge", "d3en.xlarge", "d3en.2xlarge", "d3en.4xlarge", "d3en.6xlarge", "d3en.8xlarge", "d3en.12xlarge", "f1.2xlarge", "f1.4xlarge", "f1.16xlarge", "g2.2xlarge", "g2.8xlarge", "g3.4xlarge", "g3.8xlarge", "g3.16xlarge", "g3s.xlarge", "g4ad.xlarge", "g4ad.2xlarge", "g4ad.4xlarge", "g4ad.8xlarge", "g4ad.16xlarge", "g4dn.xlarge", "g4dn.2xlarge", "g4dn.4xlarge", "g4dn.8xlarge", "g4dn.12xlarge", "g4dn.16xlarge", "g4dn.metal", "g5.xlarge", "g5.2xlarge", "g5.4xlarge", "g5.8xlarge", "g5.12xlarge", "g5.16xlarge", "g5.24xlarge", "g5.48xlarge", "h1.2xlarge", "h1.4xlarge", "h1.8xlarge", "h1.16xlarge", "hs1.8xlarge", "i2.xlarge", "i2.2xlarge", "i2.4xlarge", "i2.8xlarge", "i3.large", "i3.xlarge", "i3.2xlarge", "i3.4xlarge", "i3.8xlarge", "i3.16xlarge", "i3.metal", "i3en.large", "i3en.xlarge", "i3en.2xlarge", "i3en.3xlarge", "i3en.6xlarge", "i3en.12xlarge", "i3en.24xlarge", "i3en.metal", "im4gn.large", "im4gn.xlarge", "im4gn.2xlarge", "im4gn.4xlarge", "im4gn.8xlarge", "im4gn.16xlarge", "inf1.xlarge", "inf1.2xlarge", "inf1.6xlarge", "inf1.24xlarge", "is4gen.medium", "is4gen.large", "is4gen.xlarge", "is4gen.2xlarge", "is4gen.4xlarge", "is4gen.8xlarge", "m1.small", "m1.medium", "m1.large", "m1.xlarge", "m2.xlarge", "m2.2xlarge", "m2.4xlarge", "m3.medium", "m3.large", "m3.xlarge", "m3.2xlarge", "m4.large", "m4.xlarge", "m4.2xlarge", "m4.4xlarge", "m4.10xlarge", "m4.16xlarge", "m5.large", "m5.xlarge", "m5.2xlarge", "m5.4xlarge", "m5.8xlarge", "m5.12xlarge", "m5.16xlarge", "m5.24xlarge", "m5.metal", "m5a.large", "m5a.xlarge", "m5a.2xlarge", "m5a.4xlarge", "m5a.8xlarge", "m5a.12xlarge", "m5a.16xlarge", "m5a.24xlarge", "m5ad.large", "m5ad.xlarge", "m5ad.2xlarge", "m5ad.4xlarge", "m5ad.8xlarge", "m5ad.12xlarge", "m5ad.16xlarge", "m5ad.24xlarge", "m5d.large", "m5d.xlarge", "m5d.2xlarge", "m5d.4xlarge", "m5d.8xlarge", "m5d.12xlarge", "m5d.16xlarge", "m5d.24xlarge", "m5d.metal", "m5dn.large", "m5dn.xlarge", "m5dn.2xlarge", "m5dn.4xlarge", "m5dn.8xlarge", "m5dn.12xlarge", "m5dn.16xlarge", "m5dn.24xlarge", "m5dn.metal", "m5n.large", "m5n.xlarge", "m5n.2xlarge", "m5n.4xlarge", "m5n.8xlarge", "m5n.12xlarge", "m5n.16xlarge", "m5n.24xlarge", "m5n.metal", "m5zn.large", "m5zn.xlarge", "m5zn.2xlarge", "m5zn.3xlarge", "m5zn.6xlarge", "m5zn.12xlarge", "m5zn.metal", "m6a.large", "m6a.xlarge", "m6a.2xlarge", "m6a.4xlarge", "m6a.8xlarge", "m6a.12xlarge", "m6a.16xlarge", "m6a.24xlarge", "m6a.32xlarge", "m6a.48xlarge", "m6a.metal", "m6g.medium", "m6g.large", "m6g.xlarge", "m6g.2xlarge", "m6g.4xlarge", "m6g.8xlarge", "m6g.12xlarge", "m6g.16xlarge", "m6g.metal", "m6gd.medium", "m6gd.large", "m6gd.xlarge", "m6gd.2xlarge", "m6gd.4xlarge", "m6gd.8xlarge", "m6gd.12xlarge", "m6gd.16xlarge", "m6gd.metal", "m6i.large", "m6i.xlarge", "m6i.2xlarge", "m6i.4xlarge", "m6i.8xlarge", "m6i.12xlarge", "m6i.16xlarge", "m6i.24xlarge", "m6i.32xlarge", "m6i.metal", "mac1.metal", "p2.xlarge", "p2.8xlarge", "p2.16xlarge", "p3.2xlarge", "p3.8xlarge", "p3.16xlarge", "p3dn.24xlarge", "p4d.24xlarge", "r3.large", "r3.xlarge", "r3.2xlarge", "r3.4xlarge", "r3.8xlarge", "r4.large", "r4.xlarge", "r4.2xlarge", "r4.4xlarge", "r4.8xlarge", "r4.16xlarge", "r5.large", "r5.xlarge", "r5.2xlarge", "r5.4xlarge", "r5.8xlarge", "r5.12xlarge", "r5.16xlarge", "r5.24xlarge", "r5.metal", "r5a.large", "r5a.xlarge", "r5a.2xlarge", "r5a.4xlarge", "r5a.8xlarge", "r5a.12xlarge", "r5a.16xlarge", "r5a.24xlarge", "r5ad.large", "r5ad.xlarge", "r5ad.2xlarge", "r5ad.4xlarge", "r5ad.8xlarge", "r5ad.12xlarge", "r5ad.16xlarge", "r5ad.24xlarge", "r5b.large", "r5b.xlarge", "r5b.2xlarge", "r5b.4xlarge", "r5b.8xlarge", "r5b.12xlarge", "r5b.16xlarge", "r5b.24xlarge", "r5b.metal", "r5d.large", "r5d.xlarge", "r5d.2xlarge", "r5d.4xlarge", "r5d.8xlarge", "r5d.12xlarge", "r5d.16xlarge", "r5d.24xlarge", "r5d.metal", "r5dn.large", "r5dn.xlarge", "r5dn.2xlarge", "r5dn.4xlarge", "r5dn.8xlarge", "r5dn.12xlarge", "r5dn.16xlarge", "r5dn.24xlarge", "r5dn.metal", "r5n.large", "r5n.xlarge", "r5n.2xlarge", "r5n.4xlarge", "r5n.8xlarge", "r5n.12xlarge", "r5n.16xlarge", "r5n.24xlarge", "r5n.metal", "r6g.medium", "r6g.large", "r6g.xlarge", "r6g.2xlarge", "r6g.4xlarge", "r6g.8xlarge", "r6g.12xlarge", "r6g.16xlarge", "r6g.metal", "r6gd.medium", "r6gd.large", "r6gd.xlarge", "r6gd.2xlarge", "r6gd.4xlarge", "r6gd.8xlarge", "r6gd.12xlarge", "r6gd.16xlarge", "r6gd.metal", "r6i.large", "r6i.xlarge", "r6i.2xlarge", "r6i.4xlarge", "r6i.8xlarge", "r6i.12xlarge", "r6i.16xlarge", "r6i.24xlarge", "r6i.32xlarge", "r6i.metal", "t1.micro", "t2.nano", "t2.micro", "t2.small", "t2.medium", "t2.large", "t2.xlarge", "t2.2xlarge", "t3.nano", "t3.micro", "t3.small", "t3.medium", "t3.large", "t3.xlarge", "t3.2xlarge", "t3a.nano", "t3a.micro", "t3a.small", "t3a.medium", "t3a.large", "t3a.xlarge", "t3a.2xlarge", "t4g.nano", "t4g.micro", "t4g.small", "t4g.medium", "t4g.large", "t4g.xlarge", "t4g.2xlarge", "u-12tb1.112xlarge", "u-12tb1.metal", "u-18tb1.metal", "u-24tb1.metal", "u-3tb1.56xlarge", "u-6tb1.56xlarge", "u-6tb1.112xlarge", "u-6tb1.metal", "u-9tb1.112xlarge", "u-9tb1.metal", "vt1.3xlarge", "vt1.6xlarge", "vt1.24xlarge", "x1.16xlarge", "x1.32xlarge", "x1e.xlarge", "x1e.2xlarge", "x1e.4xlarge", "x1e.8xlarge", "x1e.16xlarge", "x1e.32xlarge", "x2gd.medium", "x2gd.large", "x2gd.xlarge", "x2gd.2xlarge", "x2gd.4xlarge", "x2gd.8xlarge", "x2gd.12xlarge", "x2gd.16xlarge", "x2gd.metal", "x2idn.16xlarge", "x2idn.24xlarge", "x2idn.32xlarge", "x2iedn.xlarge", "x2iedn.2xlarge", "x2iedn.4xlarge", "x2iedn.8xlarge", "x2iedn.16xlarge", "x2iedn.24xlarge", "x2iedn.32xlarge", "x2iezn.2xlarge", "x2iezn.4xlarge", "x2iezn.6xlarge", "x2iezn.8xlarge", "x2iezn.12xlarge", "x2iezn.metal", "z1d.large", "z1d.xlarge", "z1d.2xlarge", "z1d.3xlarge", "z1d.6xlarge", "z1d.12xlarge", "z1d.metal", ), "eu-west-2": strset.New( "c4.large", "c4.xlarge", "c4.2xlarge", "c4.4xlarge", "c4.8xlarge", "c5.large", "c5.xlarge", "c5.2xlarge", "c5.4xlarge", "c5.9xlarge", "c5.12xlarge", "c5.18xlarge", "c5.24xlarge", "c5.metal", "c5a.large", "c5a.xlarge", "c5a.2xlarge", "c5a.4xlarge", "c5a.8xlarge", "c5a.12xlarge", "c5a.16xlarge", "c5a.24xlarge", "c5d.large", "c5d.xlarge", "c5d.2xlarge", "c5d.4xlarge", "c5d.9xlarge", "c5d.12xlarge", "c5d.18xlarge", "c5d.24xlarge", "c5d.metal", "c5n.large", "c5n.xlarge", "c5n.2xlarge", "c5n.4xlarge", "c5n.9xlarge", "c5n.18xlarge", "c5n.metal", "c6g.medium", "c6g.large", "c6g.xlarge", "c6g.2xlarge", "c6g.4xlarge", "c6g.8xlarge", "c6g.12xlarge", "c6g.16xlarge", "c6g.metal", "c6gd.medium", "c6gd.large", "c6gd.xlarge", "c6gd.2xlarge", "c6gd.4xlarge", "c6gd.8xlarge", "c6gd.12xlarge", "c6gd.16xlarge", "c6gd.metal", "c6gn.medium", "c6gn.large", "c6gn.xlarge", "c6gn.2xlarge", "c6gn.4xlarge", "c6gn.8xlarge", "c6gn.12xlarge", "c6gn.16xlarge", "c6i.large", "c6i.xlarge", "c6i.2xlarge", "c6i.4xlarge", "c6i.8xlarge", "c6i.12xlarge", "c6i.16xlarge", "c6i.24xlarge", "c6i.32xlarge", "c6i.metal", "d2.xlarge", "d2.2xlarge", "d2.4xlarge", "d2.8xlarge", "d3.xlarge", "d3.2xlarge", "d3.4xlarge", "d3.8xlarge", "f1.2xlarge", "f1.4xlarge", "g3.4xlarge", "g3.8xlarge", "g3.16xlarge", "g3s.xlarge", "g4ad.xlarge", "g4ad.2xlarge", "g4ad.4xlarge", "g4ad.8xlarge", "g4ad.16xlarge", "g4dn.xlarge", "g4dn.2xlarge", "g4dn.4xlarge", "g4dn.8xlarge", "g4dn.12xlarge", "g4dn.16xlarge", "g4dn.metal", "i3.large", "i3.xlarge", "i3.2xlarge", "i3.4xlarge", "i3.8xlarge", "i3.16xlarge", "i3.metal", "i3en.large", "i3en.xlarge", "i3en.2xlarge", "i3en.3xlarge", "i3en.6xlarge", "i3en.12xlarge", "i3en.24xlarge", "i3en.metal", "inf1.xlarge", "inf1.2xlarge", "inf1.6xlarge", "inf1.24xlarge", "m4.large", "m4.xlarge", "m4.2xlarge", "m4.4xlarge", "m4.10xlarge", "m4.16xlarge", "m5.large", "m5.xlarge", "m5.2xlarge", "m5.4xlarge", "m5.8xlarge", "m5.12xlarge", "m5.16xlarge", "m5.24xlarge", "m5.metal", "m5a.large", "m5a.xlarge", "m5a.2xlarge", "m5a.4xlarge", "m5a.8xlarge", "m5a.12xlarge", "m5a.16xlarge", "m5a.24xlarge", "m5ad.large", "m5ad.xlarge", "m5ad.2xlarge", "m5ad.4xlarge", "m5ad.8xlarge", "m5ad.12xlarge", "m5ad.16xlarge", "m5ad.24xlarge", "m5d.large", "m5d.xlarge", "m5d.2xlarge", "m5d.4xlarge", "m5d.8xlarge", "m5d.12xlarge", "m5d.16xlarge", "m5d.24xlarge", "m5d.metal", "m6g.medium", "m6g.large", "m6g.xlarge", "m6g.2xlarge", "m6g.4xlarge", "m6g.8xlarge", "m6g.12xlarge", "m6g.16xlarge", "m6g.metal", "m6gd.medium", "m6gd.large", "m6gd.xlarge", "m6gd.2xlarge", "m6gd.4xlarge", "m6gd.8xlarge", "m6gd.12xlarge", "m6gd.16xlarge", "m6gd.metal", "m6i.large", "m6i.xlarge", "m6i.2xlarge", "m6i.4xlarge", "m6i.8xlarge", "m6i.12xlarge", "m6i.16xlarge", "m6i.24xlarge", "m6i.32xlarge", "m6i.metal", "mac1.metal", "p3.2xlarge", "p3.8xlarge", "p3.16xlarge", "r4.large", "r4.xlarge", "r4.2xlarge", "r4.4xlarge", "r4.8xlarge", "r4.16xlarge", "r5.large", "r5.xlarge", "r5.2xlarge", "r5.4xlarge", "r5.8xlarge", "r5.12xlarge", "r5.16xlarge", "r5.24xlarge", "r5.metal", "r5a.large", "r5a.xlarge", "r5a.2xlarge", "r5a.4xlarge", "r5a.8xlarge", "r5a.12xlarge", "r5a.16xlarge", "r5a.24xlarge", "r5ad.large", "r5ad.xlarge", "r5ad.2xlarge", "r5ad.4xlarge", "r5ad.8xlarge", "r5ad.12xlarge", "r5ad.16xlarge", "r5ad.24xlarge", "r5b.large", "r5b.xlarge", "r5b.2xlarge", "r5b.4xlarge", "r5b.8xlarge", "r5b.12xlarge", "r5b.16xlarge", "r5b.24xlarge", "r5b.metal", "r5d.large", "r5d.xlarge", "r5d.2xlarge", "r5d.4xlarge", "r5d.8xlarge", "r5d.12xlarge", "r5d.16xlarge", "r5d.24xlarge", "r5d.metal", "r5n.large", "r5n.xlarge", "r5n.2xlarge", "r5n.4xlarge", "r5n.8xlarge", "r5n.12xlarge", "r5n.16xlarge", "r5n.24xlarge", "r5n.metal", "r6g.medium", "r6g.large", "r6g.xlarge", "r6g.2xlarge", "r6g.4xlarge", "r6g.8xlarge", "r6g.12xlarge", "r6g.16xlarge", "r6g.metal", "r6i.large", "r6i.xlarge", "r6i.2xlarge", "r6i.4xlarge", "r6i.8xlarge", "r6i.12xlarge", "r6i.16xlarge", "r6i.24xlarge", "r6i.32xlarge", "r6i.metal", "t2.nano", "t2.micro", "t2.small", "t2.medium", "t2.large", "t2.xlarge", "t2.2xlarge", "t3.nano", "t3.micro", "t3.small", "t3.medium", "t3.large", "t3.xlarge", "t3.2xlarge", "t3a.nano", "t3a.micro", "t3a.small", "t3a.medium", "t3a.large", "t3a.xlarge", "t3a.2xlarge", "t4g.nano", "t4g.micro", "t4g.small", "t4g.medium", "t4g.large", "t4g.xlarge", "t4g.2xlarge", "x1.16xlarge", "x1.32xlarge", "z1d.large", "z1d.xlarge", "z1d.2xlarge", "z1d.3xlarge", "z1d.6xlarge", "z1d.12xlarge", "z1d.metal", ), "eu-west-3": strset.New( "c5.large", "c5.xlarge", "c5.2xlarge", "c5.4xlarge", "c5.9xlarge", "c5.12xlarge", "c5.18xlarge", "c5.24xlarge", "c5.metal", "c5a.large", "c5a.xlarge", "c5a.2xlarge", "c5a.4xlarge", "c5a.8xlarge", "c5a.12xlarge", "c5a.16xlarge", "c5a.24xlarge", "c5d.large", "c5d.xlarge", "c5d.2xlarge", "c5d.4xlarge", "c5d.9xlarge", "c5d.18xlarge", "c5n.large", "c5n.xlarge", "c5n.2xlarge", "c5n.4xlarge", "c5n.9xlarge", "c5n.18xlarge", "c5n.metal", "c6g.medium", "c6g.large", "c6g.xlarge", "c6g.2xlarge", "c6g.4xlarge", "c6g.8xlarge", "c6g.12xlarge", "c6g.16xlarge", "c6g.metal", "c6i.large", "c6i.xlarge", "c6i.2xlarge", "c6i.4xlarge", "c6i.8xlarge", "c6i.12xlarge", "c6i.16xlarge", "c6i.24xlarge", "c6i.32xlarge", "c6i.metal", "d2.xlarge", "d2.2xlarge", "d2.4xlarge", "d2.8xlarge", "g4dn.xlarge", "g4dn.2xlarge", "g4dn.4xlarge", "g4dn.8xlarge", "g4dn.12xlarge", "g4dn.16xlarge", "g4dn.metal", "i3.large", "i3.xlarge", "i3.2xlarge", "i3.4xlarge", "i3.8xlarge", "i3.16xlarge", "i3.metal", "i3en.large", "i3en.xlarge", "i3en.2xlarge", "i3en.3xlarge", "i3en.6xlarge", "i3en.12xlarge", "i3en.24xlarge", "i3en.metal", "inf1.xlarge", "inf1.2xlarge", "inf1.6xlarge", "inf1.24xlarge", "m5.large", "m5.xlarge", "m5.2xlarge", "m5.4xlarge", "m5.8xlarge", "m5.12xlarge", "m5.16xlarge", "m5.24xlarge", "m5.metal", "m5a.large", "m5a.xlarge", "m5a.2xlarge", "m5a.4xlarge", "m5a.8xlarge", "m5a.12xlarge", "m5a.16xlarge", "m5a.24xlarge", "m5ad.large", "m5ad.xlarge", "m5ad.2xlarge", "m5ad.4xlarge", "m5ad.8xlarge", "m5ad.12xlarge", "m5ad.16xlarge", "m5ad.24xlarge", "m5d.large", "m5d.xlarge", "m5d.2xlarge", "m5d.4xlarge", "m5d.8xlarge", "m5d.12xlarge", "m5d.16xlarge", "m5d.24xlarge", "m5d.metal", "m6g.medium", "m6g.large", "m6g.xlarge", "m6g.2xlarge", "m6g.4xlarge", "m6g.8xlarge", "m6g.12xlarge", "m6g.16xlarge", "m6g.metal", "m6i.large", "m6i.xlarge", "m6i.2xlarge", "m6i.4xlarge", "m6i.8xlarge", "m6i.12xlarge", "m6i.16xlarge", "m6i.24xlarge", "m6i.32xlarge", "m6i.metal", "r4.large", "r4.xlarge", "r4.2xlarge", "r4.4xlarge", "r4.8xlarge", "r4.16xlarge", "r5.large", "r5.xlarge", "r5.2xlarge", "r5.4xlarge", "r5.8xlarge", "r5.12xlarge", "r5.16xlarge", "r5.24xlarge", "r5.metal", "r5a.large", "r5a.xlarge", "r5a.2xlarge", "r5a.4xlarge", "r5a.8xlarge", "r5a.12xlarge", "r5a.16xlarge", "r5a.24xlarge", "r5ad.large", "r5ad.xlarge", "r5ad.2xlarge", "r5ad.4xlarge", "r5ad.8xlarge", "r5ad.12xlarge", "r5ad.16xlarge", "r5ad.24xlarge", "r5d.large", "r5d.xlarge", "r5d.2xlarge", "r5d.4xlarge", "r5d.8xlarge", "r5d.12xlarge", "r5d.16xlarge", "r5d.24xlarge", "r5d.metal", "r5dn.large", "r5dn.xlarge", "r5dn.2xlarge", "r5dn.4xlarge", "r5dn.8xlarge", "r5dn.12xlarge", "r5dn.16xlarge", "r5dn.24xlarge", "r5dn.metal", "r5n.large", "r5n.xlarge", "r5n.2xlarge", "r5n.4xlarge", "r5n.8xlarge", "r5n.12xlarge", "r5n.16xlarge", "r5n.24xlarge", "r5n.metal", "r6g.medium", "r6g.large", "r6g.xlarge", "r6g.2xlarge", "r6g.4xlarge", "r6g.8xlarge", "r6g.12xlarge", "r6g.16xlarge", "r6g.metal", "r6gd.medium", "r6gd.large", "r6gd.xlarge", "r6gd.2xlarge", "r6gd.4xlarge", "r6gd.8xlarge", "r6gd.12xlarge", "r6gd.16xlarge", "r6gd.metal", "r6i.large", "r6i.xlarge", "r6i.2xlarge", "r6i.4xlarge", "r6i.8xlarge", "r6i.12xlarge", "r6i.16xlarge", "r6i.24xlarge", "r6i.32xlarge", "r6i.metal", "t2.nano", "t2.micro", "t2.small", "t2.medium", "t2.large", "t2.xlarge", "t2.2xlarge", "t3.nano", "t3.micro", "t3.small", "t3.medium", "t3.large", "t3.xlarge", "t3.2xlarge", "t3a.nano", "t3a.micro", "t3a.small", "t3a.medium", "t3a.large", "t3a.xlarge", "t3a.2xlarge", "t4g.nano", "t4g.micro", "t4g.small", "t4g.medium", "t4g.large", "t4g.xlarge", "t4g.2xlarge", "u-6tb1.56xlarge", "u-6tb1.112xlarge", "x1.16xlarge", "x1.32xlarge", ), "me-south-1": strset.New( "c5.large", "c5.xlarge", "c5.2xlarge", "c5.4xlarge", "c5.9xlarge", "c5.12xlarge", "c5.18xlarge", "c5.24xlarge", "c5.metal", "c5a.large", "c5a.xlarge", "c5a.2xlarge", "c5a.4xlarge", "c5a.8xlarge", "c5a.12xlarge", "c5a.16xlarge", "c5a.24xlarge", "c5ad.large", "c5ad.xlarge", "c5ad.2xlarge", "c5ad.4xlarge", "c5ad.8xlarge", "c5ad.12xlarge", "c5ad.16xlarge", "c5ad.24xlarge", "c5d.large", "c5d.xlarge", "c5d.2xlarge", "c5d.4xlarge", "c5d.9xlarge", "c5d.12xlarge", "c5d.18xlarge", "c5d.24xlarge", "c5d.metal", "c5n.large", "c5n.xlarge", "c5n.2xlarge", "c5n.4xlarge", "c5n.9xlarge", "c5n.18xlarge", "c5n.metal", "c6g.medium", "c6g.large", "c6g.xlarge", "c6g.2xlarge", "c6g.4xlarge", "c6g.8xlarge", "c6g.12xlarge", "c6g.16xlarge", "c6g.metal", "d2.xlarge", "d2.2xlarge", "d2.4xlarge", "d2.8xlarge", "g4dn.xlarge", "g4dn.2xlarge", "g4dn.4xlarge", "g4dn.8xlarge", "g4dn.12xlarge", "g4dn.16xlarge", "g4dn.metal", "i3.large", "i3.xlarge", "i3.2xlarge", "i3.4xlarge", "i3.8xlarge", "i3.16xlarge", "i3en.large", "i3en.xlarge", "i3en.2xlarge", "i3en.3xlarge", "i3en.6xlarge", "i3en.12xlarge", "i3en.24xlarge", "i3en.metal", "inf1.xlarge", "inf1.2xlarge", "inf1.6xlarge", "inf1.24xlarge", "m5.large", "m5.xlarge", "m5.2xlarge", "m5.4xlarge", "m5.8xlarge", "m5.12xlarge", "m5.16xlarge", "m5.24xlarge", "m5.metal", "m5d.large", "m5d.xlarge", "m5d.2xlarge", "m5d.4xlarge", "m5d.8xlarge", "m5d.12xlarge", "m5d.16xlarge", "m5d.24xlarge", "m5d.metal", "m6g.medium", "m6g.large", "m6g.xlarge", "m6g.2xlarge", "m6g.4xlarge", "m6g.8xlarge", "m6g.12xlarge", "m6g.16xlarge", "m6g.metal", "r5.large", "r5.xlarge", "r5.2xlarge", "r5.4xlarge", "r5.8xlarge", "r5.12xlarge", "r5.16xlarge", "r5.24xlarge", "r5.metal", "r5d.large", "r5d.xlarge", "r5d.2xlarge", "r5d.4xlarge", "r5d.8xlarge", "r5d.12xlarge", "r5d.16xlarge", "r5d.24xlarge", "r5d.metal", "t3.nano", "t3.micro", "t3.small", "t3.medium", "t3.large", "t3.xlarge", "t3.2xlarge", ), "sa-east-1": strset.New( "c1.medium", "c1.xlarge", "c3.large", "c3.xlarge", "c3.2xlarge", "c3.4xlarge", "c3.8xlarge", "c4.large", "c4.xlarge", "c4.2xlarge", "c4.4xlarge", "c4.8xlarge", "c5.large", "c5.xlarge", "c5.2xlarge", "c5.4xlarge", "c5.9xlarge", "c5.12xlarge", "c5.18xlarge", "c5.24xlarge", "c5.metal", "c5a.large", "c5a.xlarge", "c5a.2xlarge", "c5a.4xlarge", "c5a.8xlarge", "c5a.12xlarge", "c5a.16xlarge", "c5a.24xlarge", "c5ad.large", "c5ad.xlarge", "c5ad.2xlarge", "c5ad.4xlarge", "c5ad.8xlarge", "c5ad.12xlarge", "c5ad.16xlarge", "c5ad.24xlarge", "c5d.large", "c5d.xlarge", "c5d.2xlarge", "c5d.4xlarge", "c5d.9xlarge", "c5d.12xlarge", "c5d.18xlarge", "c5d.24xlarge", "c5d.metal", "c5n.large", "c5n.xlarge", "c5n.2xlarge", "c5n.4xlarge", "c5n.9xlarge", "c5n.18xlarge", "c5n.metal", "c6g.medium", "c6g.large", "c6g.xlarge", "c6g.2xlarge", "c6g.4xlarge", "c6g.8xlarge", "c6g.12xlarge", "c6g.16xlarge", "c6g.metal", "c6gn.medium", "c6gn.large", "c6gn.xlarge", "c6gn.2xlarge", "c6gn.4xlarge", "c6gn.8xlarge", "c6gn.12xlarge", "c6gn.16xlarge", "c6i.large", "c6i.xlarge", "c6i.2xlarge", "c6i.4xlarge", "c6i.8xlarge", "c6i.12xlarge", "c6i.16xlarge", "c6i.24xlarge", "c6i.32xlarge", "c6i.metal", "d2.xlarge", "d2.2xlarge", "d2.4xlarge", "d2.8xlarge", "g2.2xlarge", "g2.8xlarge", "g4dn.xlarge", "g4dn.2xlarge", "g4dn.4xlarge", "g4dn.8xlarge", "g4dn.12xlarge", "g4dn.16xlarge", "g4dn.metal", "i2.xlarge", "i2.2xlarge", "i2.4xlarge", "i2.8xlarge", "i3.large", "i3.xlarge", "i3.2xlarge", "i3.4xlarge", "i3.8xlarge", "i3.16xlarge", "i3.metal", "i3en.large", "i3en.xlarge", "i3en.2xlarge", "i3en.3xlarge", "i3en.6xlarge", "i3en.12xlarge", "i3en.24xlarge", "i3en.metal", "inf1.xlarge", "inf1.2xlarge", "inf1.6xlarge", "inf1.24xlarge", "m1.small", "m1.medium", "m1.large", "m1.xlarge", "m2.xlarge", "m2.2xlarge", "m2.4xlarge", "m3.medium", "m3.large", "m3.xlarge", "m3.2xlarge", "m4.large", "m4.xlarge", "m4.2xlarge", "m4.4xlarge", "m4.10xlarge", "m4.16xlarge", "m5.large", "m5.xlarge", "m5.2xlarge", "m5.4xlarge", "m5.8xlarge", "m5.12xlarge", "m5.16xlarge", "m5.24xlarge", "m5.metal", "m5a.large", "m5a.xlarge", "m5a.2xlarge", "m5a.4xlarge", "m5a.8xlarge", "m5a.12xlarge", "m5a.16xlarge", "m5a.24xlarge", "m5ad.large", "m5ad.xlarge", "m5ad.2xlarge", "m5ad.4xlarge", "m5ad.8xlarge", "m5ad.12xlarge", "m5ad.16xlarge", "m5ad.24xlarge", "m5d.large", "m5d.xlarge", "m5d.2xlarge", "m5d.4xlarge", "m5d.8xlarge", "m5d.12xlarge", "m5d.16xlarge", "m5d.24xlarge", "m5d.metal", "m5zn.large", "m5zn.xlarge", "m5zn.2xlarge", "m5zn.3xlarge", "m5zn.6xlarge", "m5zn.12xlarge", "m5zn.metal", "m6g.medium", "m6g.large", "m6g.xlarge", "m6g.2xlarge", "m6g.4xlarge", "m6g.8xlarge", "m6g.12xlarge", "m6g.16xlarge", "m6g.metal", "m6i.large", "m6i.xlarge", "m6i.2xlarge", "m6i.4xlarge", "m6i.8xlarge", "m6i.12xlarge", "m6i.16xlarge", "m6i.24xlarge", "m6i.32xlarge", "m6i.metal", "r3.large", "r3.xlarge", "r3.2xlarge", "r3.4xlarge", "r3.8xlarge", "r4.large", "r4.xlarge", "r4.2xlarge", "r4.4xlarge", "r4.8xlarge", "r4.16xlarge", "r5.large", "r5.xlarge", "r5.2xlarge", "r5.4xlarge", "r5.8xlarge", "r5.12xlarge", "r5.16xlarge", "r5.24xlarge", "r5.metal", "r5a.large", "r5a.xlarge", "r5a.2xlarge", "r5a.4xlarge", "r5a.8xlarge", "r5a.12xlarge", "r5a.16xlarge", "r5a.24xlarge", "r5ad.large", "r5ad.xlarge", "r5ad.2xlarge", "r5ad.4xlarge", "r5ad.8xlarge", "r5ad.12xlarge", "r5ad.16xlarge", "r5ad.24xlarge", "r5d.large", "r5d.xlarge", "r5d.2xlarge", "r5d.4xlarge", "r5d.8xlarge", "r5d.12xlarge", "r5d.16xlarge", "r5d.24xlarge", "r5d.metal", "r5n.large", "r5n.xlarge", "r5n.2xlarge", "r5n.4xlarge", "r5n.8xlarge", "r5n.12xlarge", "r5n.16xlarge", "r5n.24xlarge", "r5n.metal", "r6g.medium", "r6g.large", "r6g.xlarge", "r6g.2xlarge", "r6g.4xlarge", "r6g.8xlarge", "r6g.12xlarge", "r6g.16xlarge", "r6g.metal", "r6i.large", "r6i.xlarge", "r6i.2xlarge", "r6i.4xlarge", "r6i.8xlarge", "r6i.12xlarge", "r6i.16xlarge", "r6i.24xlarge", "r6i.32xlarge", "r6i.metal", "t1.micro", "t2.nano", "t2.micro", "t2.small", "t2.medium", "t2.large", "t2.xlarge", "t2.2xlarge", "t3.nano", "t3.micro", "t3.small", "t3.medium", "t3.large", "t3.xlarge", "t3.2xlarge", "t3a.nano", "t3a.micro", "t3a.small", "t3a.medium", "t3a.large", "t3a.xlarge", "t3a.2xlarge", "t4g.nano", "t4g.micro", "t4g.small", "t4g.medium", "t4g.large", "t4g.xlarge", "t4g.2xlarge", "u-12tb1.metal", "u-6tb1.metal", "u-9tb1.metal", "x1.16xlarge", "x1.32xlarge", "x1e.xlarge", "x1e.2xlarge", "x1e.4xlarge", "x1e.8xlarge", "x1e.16xlarge", "x1e.32xlarge", ), "us-east-1": strset.New( "a1.medium", "a1.large", "a1.xlarge", "a1.2xlarge", "a1.4xlarge", "a1.metal", "c1.medium", "c1.xlarge", "c3.large", "c3.xlarge", "c3.2xlarge", "c3.4xlarge", "c3.8xlarge", "c4.large", "c4.xlarge", "c4.2xlarge", "c4.4xlarge", "c4.8xlarge", "c5.large", "c5.xlarge", "c5.2xlarge", "c5.4xlarge", "c5.9xlarge", "c5.12xlarge", "c5.18xlarge", "c5.24xlarge", "c5.metal", "c5a.large", "c5a.xlarge", "c5a.2xlarge", "c5a.4xlarge", "c5a.8xlarge", "c5a.12xlarge", "c5a.16xlarge", "c5a.24xlarge", "c5ad.large", "c5ad.xlarge", "c5ad.2xlarge", "c5ad.4xlarge", "c5ad.8xlarge", "c5ad.12xlarge", "c5ad.16xlarge", "c5ad.24xlarge", "c5d.large", "c5d.xlarge", "c5d.2xlarge", "c5d.4xlarge", "c5d.9xlarge", "c5d.12xlarge", "c5d.18xlarge", "c5d.24xlarge", "c5d.metal", "c5n.large", "c5n.xlarge", "c5n.2xlarge", "c5n.4xlarge", "c5n.9xlarge", "c5n.18xlarge", "c5n.metal", "c6a.large", "c6a.xlarge", "c6a.2xlarge", "c6a.4xlarge", "c6a.8xlarge", "c6a.12xlarge", "c6a.16xlarge", "c6a.24xlarge", "c6a.32xlarge", "c6a.48xlarge", "c6a.metal", "c6g.medium", "c6g.large", "c6g.xlarge", "c6g.2xlarge", "c6g.4xlarge", "c6g.8xlarge", "c6g.12xlarge", "c6g.16xlarge", "c6g.metal", "c6gd.medium", "c6gd.large", "c6gd.xlarge", "c6gd.2xlarge", "c6gd.4xlarge", "c6gd.8xlarge", "c6gd.12xlarge", "c6gd.16xlarge", "c6gd.metal", "c6gn.medium", "c6gn.large", "c6gn.xlarge", "c6gn.2xlarge", "c6gn.4xlarge", "c6gn.8xlarge", "c6gn.12xlarge", "c6gn.16xlarge", "c6gn.metal", "c6i.large", "c6i.xlarge", "c6i.2xlarge", "c6i.4xlarge", "c6i.8xlarge", "c6i.12xlarge", "c6i.16xlarge", "c6i.24xlarge", "c6i.32xlarge", "c6i.metal", "cc2.8xlarge", "cr1.8xlarge", "d2.xlarge", "d2.2xlarge", "d2.4xlarge", "d2.8xlarge", "d3.xlarge", "d3.2xlarge", "d3.4xlarge", "d3.8xlarge", "d3en.xlarge", "d3en.2xlarge", "d3en.4xlarge", "d3en.6xlarge", "d3en.8xlarge", "d3en.12xlarge", "dl1.24xlarge", "f1.2xlarge", "f1.4xlarge", "f1.16xlarge", "g2.2xlarge", "g2.8xlarge", "g3.4xlarge", "g3.8xlarge", "g3.16xlarge", "g3s.xlarge", "g4ad.xlarge", "g4ad.2xlarge", "g4ad.4xlarge", "g4ad.8xlarge", "g4ad.16xlarge", "g4dn.xlarge", "g4dn.2xlarge", "g4dn.4xlarge", "g4dn.8xlarge", "g4dn.12xlarge", "g4dn.16xlarge", "g4dn.metal", "g5.xlarge", "g5.2xlarge", "g5.4xlarge", "g5.8xlarge", "g5.12xlarge", "g5.16xlarge", "g5.24xlarge", "g5.48xlarge", "g5g.xlarge", "g5g.2xlarge", "g5g.4xlarge", "g5g.8xlarge", "g5g.16xlarge", "g5g.metal", "h1.2xlarge", "h1.4xlarge", "h1.8xlarge", "h1.16xlarge", "hs1.8xlarge", "i2.xlarge", "i2.2xlarge", "i2.4xlarge", "i2.8xlarge", "i3.large", "i3.xlarge", "i3.2xlarge", "i3.4xlarge", "i3.8xlarge", "i3.16xlarge", "i3.metal", "i3en.large", "i3en.xlarge", "i3en.2xlarge", "i3en.3xlarge", "i3en.6xlarge", "i3en.12xlarge", "i3en.24xlarge", "i3en.metal", "im4gn.large", "im4gn.xlarge", "im4gn.2xlarge", "im4gn.4xlarge", "im4gn.8xlarge", "im4gn.16xlarge", "inf1.xlarge", "inf1.2xlarge", "inf1.6xlarge", "inf1.24xlarge", "is4gen.medium", "is4gen.large", "is4gen.xlarge", "is4gen.2xlarge", "is4gen.4xlarge", "is4gen.8xlarge", "m1.small", "m1.medium", "m1.large", "m1.xlarge", "m2.xlarge", "m2.2xlarge", "m2.4xlarge", "m3.medium", "m3.large", "m3.xlarge", "m3.2xlarge", "m4.large", "m4.xlarge", "m4.2xlarge", "m4.4xlarge", "m4.10xlarge", "m4.16xlarge", "m5.large", "m5.xlarge", "m5.2xlarge", "m5.4xlarge", "m5.8xlarge", "m5.12xlarge", "m5.16xlarge", "m5.24xlarge", "m5.metal", "m5a.large", "m5a.xlarge", "m5a.2xlarge", "m5a.4xlarge", "m5a.8xlarge", "m5a.12xlarge", "m5a.16xlarge", "m5a.24xlarge", "m5ad.large", "m5ad.xlarge", "m5ad.2xlarge", "m5ad.4xlarge", "m5ad.8xlarge", "m5ad.12xlarge", "m5ad.16xlarge", "m5ad.24xlarge", "m5d.large", "m5d.xlarge", "m5d.2xlarge", "m5d.4xlarge", "m5d.8xlarge", "m5d.12xlarge", "m5d.16xlarge", "m5d.24xlarge", "m5d.metal", "m5dn.large", "m5dn.xlarge", "m5dn.2xlarge", "m5dn.4xlarge", "m5dn.8xlarge", "m5dn.12xlarge", "m5dn.16xlarge", "m5dn.24xlarge", "m5dn.metal", "m5n.large", "m5n.xlarge", "m5n.2xlarge", "m5n.4xlarge", "m5n.8xlarge", "m5n.12xlarge", "m5n.16xlarge", "m5n.24xlarge", "m5n.metal", "m5zn.large", "m5zn.xlarge", "m5zn.2xlarge", "m5zn.3xlarge", "m5zn.6xlarge", "m5zn.12xlarge", "m5zn.metal", "m6a.large", "m6a.xlarge", "m6a.2xlarge", "m6a.4xlarge", "m6a.8xlarge", "m6a.12xlarge", "m6a.16xlarge", "m6a.24xlarge", "m6a.32xlarge", "m6a.48xlarge", "m6a.metal", "m6g.medium", "m6g.large", "m6g.xlarge", "m6g.2xlarge", "m6g.4xlarge", "m6g.8xlarge", "m6g.12xlarge", "m6g.16xlarge", "m6g.metal", "m6gd.medium", "m6gd.large", "m6gd.xlarge", "m6gd.2xlarge", "m6gd.4xlarge", "m6gd.8xlarge", "m6gd.12xlarge", "m6gd.16xlarge", "m6gd.metal", "m6i.large", "m6i.xlarge", "m6i.2xlarge", "m6i.4xlarge", "m6i.8xlarge", "m6i.12xlarge", "m6i.16xlarge", "m6i.24xlarge", "m6i.32xlarge", "m6i.metal", "mac1.metal", "mac2.metal", "p2.xlarge", "p2.8xlarge", "p2.16xlarge", "p3.2xlarge", "p3.8xlarge", "p3.16xlarge", "p3dn.24xlarge", "p4d.24xlarge", "r3.large", "r3.xlarge", "r3.2xlarge", "r3.4xlarge", "r3.8xlarge", "r4.large", "r4.xlarge", "r4.2xlarge", "r4.4xlarge", "r4.8xlarge", "r4.16xlarge", "r5.large", "r5.xlarge", "r5.2xlarge", "r5.4xlarge", "r5.8xlarge", "r5.12xlarge", "r5.16xlarge", "r5.24xlarge", "r5.metal", "r5a.large", "r5a.xlarge", "r5a.2xlarge", "r5a.4xlarge", "r5a.8xlarge", "r5a.12xlarge", "r5a.16xlarge", "r5a.24xlarge", "r5ad.large", "r5ad.xlarge", "r5ad.2xlarge", "r5ad.4xlarge", "r5ad.8xlarge", "r5ad.12xlarge", "r5ad.16xlarge", "r5ad.24xlarge", "r5b.large", "r5b.xlarge", "r5b.2xlarge", "r5b.4xlarge", "r5b.8xlarge", "r5b.12xlarge", "r5b.16xlarge", "r5b.24xlarge", "r5b.metal", "r5d.large", "r5d.xlarge", "r5d.2xlarge", "r5d.4xlarge", "r5d.8xlarge", "r5d.12xlarge", "r5d.16xlarge", "r5d.24xlarge", "r5d.metal", "r5dn.large", "r5dn.xlarge", "r5dn.2xlarge", "r5dn.4xlarge", "r5dn.8xlarge", "r5dn.12xlarge", "r5dn.16xlarge", "r5dn.24xlarge", "r5dn.metal", "r5n.large", "r5n.xlarge", "r5n.2xlarge", "r5n.4xlarge", "r5n.8xlarge", "r5n.12xlarge", "r5n.16xlarge", "r5n.24xlarge", "r5n.metal", "r6g.medium", "r6g.large", "r6g.xlarge", "r6g.2xlarge", "r6g.4xlarge", "r6g.8xlarge", "r6g.12xlarge", "r6g.16xlarge", "r6g.metal", "r6gd.medium", "r6gd.large", "r6gd.xlarge", "r6gd.2xlarge", "r6gd.4xlarge", "r6gd.8xlarge", "r6gd.12xlarge", "r6gd.16xlarge", "r6gd.metal", "r6i.large", "r6i.xlarge", "r6i.2xlarge", "r6i.4xlarge", "r6i.8xlarge", "r6i.12xlarge", "r6i.16xlarge", "r6i.24xlarge", "r6i.32xlarge", "r6i.metal", "t1.micro", "t2.nano", "t2.micro", "t2.small", "t2.medium", "t2.large", "t2.xlarge", "t2.2xlarge", "t3.nano", "t3.micro", "t3.small", "t3.medium", "t3.large", "t3.xlarge", "t3.2xlarge", "t3a.nano", "t3a.micro", "t3a.small", "t3a.medium", "t3a.large", "t3a.xlarge", "t3a.2xlarge", "t4g.nano", "t4g.micro", "t4g.small", "t4g.medium", "t4g.large", "t4g.xlarge", "t4g.2xlarge", "u-12tb1.112xlarge", "u-12tb1.metal", "u-18tb1.metal", "u-24tb1.metal", "u-3tb1.56xlarge", "u-6tb1.56xlarge", "u-6tb1.112xlarge", "u-6tb1.metal", "u-9tb1.112xlarge", "u-9tb1.metal", "vt1.3xlarge", "vt1.6xlarge", "vt1.24xlarge", "x1.16xlarge", "x1.32xlarge", "x1e.xlarge", "x1e.2xlarge", "x1e.4xlarge", "x1e.8xlarge", "x1e.16xlarge", "x1e.32xlarge", "x2gd.medium", "x2gd.large", "x2gd.xlarge", "x2gd.2xlarge", "x2gd.4xlarge", "x2gd.8xlarge", "x2gd.12xlarge", "x2gd.16xlarge", "x2gd.metal", "x2idn.16xlarge", "x2idn.24xlarge", "x2idn.32xlarge", "x2iedn.xlarge", "x2iedn.2xlarge", "x2iedn.4xlarge", "x2iedn.8xlarge", "x2iedn.16xlarge", "x2iedn.24xlarge", "x2iedn.32xlarge", "x2iezn.2xlarge", "x2iezn.4xlarge", "x2iezn.6xlarge", "x2iezn.8xlarge", "x2iezn.12xlarge", "x2iezn.metal", "z1d.large", "z1d.xlarge", "z1d.2xlarge", "z1d.3xlarge", "z1d.6xlarge", "z1d.12xlarge", "z1d.metal", ), "us-east-2": strset.New( "a1.medium", "a1.large", "a1.xlarge", "a1.2xlarge", "a1.4xlarge", "a1.metal", "c4.large", "c4.xlarge", "c4.2xlarge", "c4.4xlarge", "c4.8xlarge", "c5.large", "c5.xlarge", "c5.2xlarge", "c5.4xlarge", "c5.9xlarge", "c5.12xlarge", "c5.18xlarge", "c5.24xlarge", "c5.metal", "c5a.large", "c5a.xlarge", "c5a.2xlarge", "c5a.4xlarge", "c5a.8xlarge", "c5a.12xlarge", "c5a.16xlarge", "c5a.24xlarge", "c5ad.large", "c5ad.xlarge", "c5ad.2xlarge", "c5ad.4xlarge", "c5ad.8xlarge", "c5ad.12xlarge", "c5ad.16xlarge", "c5ad.24xlarge", "c5d.large", "c5d.xlarge", "c5d.2xlarge", "c5d.4xlarge", "c5d.9xlarge", "c5d.12xlarge", "c5d.18xlarge", "c5d.24xlarge", "c5d.metal", "c5n.large", "c5n.xlarge", "c5n.2xlarge", "c5n.4xlarge", "c5n.9xlarge", "c5n.18xlarge", "c5n.metal", "c6g.medium", "c6g.large", "c6g.xlarge", "c6g.2xlarge", "c6g.4xlarge", "c6g.8xlarge", "c6g.12xlarge", "c6g.16xlarge", "c6g.metal", "c6gd.medium", "c6gd.large", "c6gd.xlarge", "c6gd.2xlarge", "c6gd.4xlarge", "c6gd.8xlarge", "c6gd.12xlarge", "c6gd.16xlarge", "c6gd.metal", "c6gn.medium", "c6gn.large", "c6gn.xlarge", "c6gn.2xlarge", "c6gn.4xlarge", "c6gn.8xlarge", "c6gn.12xlarge", "c6gn.16xlarge", "c6gn.metal", "c6i.large", "c6i.xlarge", "c6i.2xlarge", "c6i.4xlarge", "c6i.8xlarge", "c6i.12xlarge", "c6i.16xlarge", "c6i.24xlarge", "c6i.32xlarge", "c6i.metal", "d2.xlarge", "d2.2xlarge", "d2.4xlarge", "d2.8xlarge", "d3.xlarge", "d3.2xlarge", "d3.4xlarge", "d3.8xlarge", "g3.4xlarge", "g3.8xlarge", "g3.16xlarge", "g3s.xlarge", "g4ad.xlarge", "g4ad.2xlarge", "g4ad.4xlarge", "g4ad.8xlarge", "g4ad.16xlarge", "g4dn.xlarge", "g4dn.2xlarge", "g4dn.4xlarge", "g4dn.8xlarge", "g4dn.12xlarge", "g4dn.16xlarge", "g4dn.metal", "h1.2xlarge", "h1.4xlarge", "h1.8xlarge", "h1.16xlarge", "hpc6a.48xlarge", "i2.xlarge", "i2.2xlarge", "i2.4xlarge", "i2.8xlarge", "i3.large", "i3.xlarge", "i3.2xlarge", "i3.4xlarge", "i3.8xlarge", "i3.16xlarge", "i3.metal", "i3en.large", "i3en.xlarge", "i3en.2xlarge", "i3en.3xlarge", "i3en.6xlarge", "i3en.12xlarge", "i3en.24xlarge", "i3en.metal", "im4gn.large", "im4gn.xlarge", "im4gn.2xlarge", "im4gn.4xlarge", "im4gn.8xlarge", "im4gn.16xlarge", "inf1.xlarge", "inf1.2xlarge", "inf1.6xlarge", "inf1.24xlarge", "is4gen.medium", "is4gen.large", "is4gen.xlarge", "is4gen.2xlarge", "is4gen.4xlarge", "is4gen.8xlarge", "m4.large", "m4.xlarge", "m4.2xlarge", "m4.4xlarge", "m4.10xlarge", "m4.16xlarge", "m5.large", "m5.xlarge", "m5.2xlarge", "m5.4xlarge", "m5.8xlarge", "m5.12xlarge", "m5.16xlarge", "m5.24xlarge", "m5.metal", "m5a.large", "m5a.xlarge", "m5a.2xlarge", "m5a.4xlarge", "m5a.8xlarge", "m5a.12xlarge", "m5a.16xlarge", "m5a.24xlarge", "m5ad.large", "m5ad.xlarge", "m5ad.2xlarge", "m5ad.4xlarge", "m5ad.8xlarge", "m5ad.12xlarge", "m5ad.16xlarge", "m5ad.24xlarge", "m5d.large", "m5d.xlarge", "m5d.2xlarge", "m5d.4xlarge", "m5d.8xlarge", "m5d.12xlarge", "m5d.16xlarge", "m5d.24xlarge", "m5d.metal", "m5dn.large", "m5dn.xlarge", "m5dn.2xlarge", "m5dn.4xlarge", "m5dn.8xlarge", "m5dn.12xlarge", "m5dn.16xlarge", "m5dn.24xlarge", "m5dn.metal", "m5n.large", "m5n.xlarge", "m5n.2xlarge", "m5n.4xlarge", "m5n.8xlarge", "m5n.12xlarge", "m5n.16xlarge", "m5n.24xlarge", "m5n.metal", "m5zn.large", "m5zn.xlarge", "m5zn.2xlarge", "m5zn.3xlarge", "m5zn.6xlarge", "m5zn.12xlarge", "m5zn.metal", "m6g.medium", "m6g.large", "m6g.xlarge", "m6g.2xlarge", "m6g.4xlarge", "m6g.8xlarge", "m6g.12xlarge", "m6g.16xlarge", "m6g.metal", "m6gd.medium", "m6gd.large", "m6gd.xlarge", "m6gd.2xlarge", "m6gd.4xlarge", "m6gd.8xlarge", "m6gd.12xlarge", "m6gd.16xlarge", "m6gd.metal", "m6i.large", "m6i.xlarge", "m6i.2xlarge", "m6i.4xlarge", "m6i.8xlarge", "m6i.12xlarge", "m6i.16xlarge", "m6i.24xlarge", "m6i.32xlarge", "m6i.metal", "mac1.metal", "p2.xlarge", "p2.8xlarge", "p2.16xlarge", "p3.2xlarge", "p3.8xlarge", "p3.16xlarge", "p4d.24xlarge", "r3.large", "r3.xlarge", "r3.2xlarge", "r3.4xlarge", "r3.8xlarge", "r4.large", "r4.xlarge", "r4.2xlarge", "r4.4xlarge", "r4.8xlarge", "r4.16xlarge", "r5.large", "r5.xlarge", "r5.2xlarge", "r5.4xlarge", "r5.8xlarge", "r5.12xlarge", "r5.16xlarge", "r5.24xlarge", "r5.metal", "r5a.large", "r5a.xlarge", "r5a.2xlarge", "r5a.4xlarge", "r5a.8xlarge", "r5a.12xlarge", "r5a.16xlarge", "r5a.24xlarge", "r5ad.large", "r5ad.xlarge", "r5ad.2xlarge", "r5ad.4xlarge", "r5ad.8xlarge", "r5ad.12xlarge", "r5ad.16xlarge", "r5ad.24xlarge", "r5b.large", "r5b.xlarge", "r5b.2xlarge", "r5b.4xlarge", "r5b.8xlarge", "r5b.12xlarge", "r5b.16xlarge", "r5b.24xlarge", "r5b.metal", "r5d.large", "r5d.xlarge", "r5d.2xlarge", "r5d.4xlarge", "r5d.8xlarge", "r5d.12xlarge", "r5d.16xlarge", "r5d.24xlarge", "r5d.metal", "r5dn.large", "r5dn.xlarge", "r5dn.2xlarge", "r5dn.4xlarge", "r5dn.8xlarge", "r5dn.12xlarge", "r5dn.16xlarge", "r5dn.24xlarge", "r5dn.metal", "r5n.large", "r5n.xlarge", "r5n.2xlarge", "r5n.4xlarge", "r5n.8xlarge", "r5n.12xlarge", "r5n.16xlarge", "r5n.24xlarge", "r5n.metal", "r6g.medium", "r6g.large", "r6g.xlarge", "r6g.2xlarge", "r6g.4xlarge", "r6g.8xlarge", "r6g.12xlarge", "r6g.16xlarge", "r6g.metal", "r6gd.medium", "r6gd.large", "r6gd.xlarge", "r6gd.2xlarge", "r6gd.4xlarge", "r6gd.8xlarge", "r6gd.12xlarge", "r6gd.16xlarge", "r6gd.metal", "r6i.large", "r6i.xlarge", "r6i.2xlarge", "r6i.4xlarge", "r6i.8xlarge", "r6i.12xlarge", "r6i.16xlarge", "r6i.24xlarge", "r6i.32xlarge", "r6i.metal", "t2.nano", "t2.micro", "t2.small", "t2.medium", "t2.large", "t2.xlarge", "t2.2xlarge", "t3.nano", "t3.micro", "t3.small", "t3.medium", "t3.large", "t3.xlarge", "t3.2xlarge", "t3a.nano", "t3a.micro", "t3a.small", "t3a.medium", "t3a.large", "t3a.xlarge", "t3a.2xlarge", "t4g.nano", "t4g.micro", "t4g.small", "t4g.medium", "t4g.large", "t4g.xlarge", "t4g.2xlarge", "u-12tb1.metal", "u-6tb1.56xlarge", "u-6tb1.112xlarge", "u-6tb1.metal", "u-9tb1.metal", "x1.16xlarge", "x1.32xlarge", "x1e.xlarge", "x1e.2xlarge", "x1e.4xlarge", "x1e.8xlarge", "x1e.16xlarge", "x1e.32xlarge", "x2gd.medium", "x2gd.large", "x2gd.xlarge", "x2gd.2xlarge", "x2gd.4xlarge", "x2gd.8xlarge", "x2gd.12xlarge", "x2gd.16xlarge", "x2gd.metal", "x2iedn.xlarge", "x2iedn.2xlarge", "x2iedn.4xlarge", "x2iedn.8xlarge", "x2iedn.16xlarge", "x2iedn.24xlarge", "x2iedn.32xlarge", "z1d.large", "z1d.xlarge", "z1d.2xlarge", "z1d.3xlarge", "z1d.6xlarge", "z1d.12xlarge", "z1d.metal", ), "us-gov-east-1": strset.New( "c5.large", "c5.xlarge", "c5.2xlarge", "c5.4xlarge", "c5.9xlarge", "c5.12xlarge", "c5.18xlarge", "c5.24xlarge", "c5.metal", "c5a.large", "c5a.xlarge", "c5a.2xlarge", "c5a.4xlarge", "c5a.8xlarge", "c5a.12xlarge", "c5a.16xlarge", "c5a.24xlarge", "c5d.large", "c5d.xlarge", "c5d.2xlarge", "c5d.4xlarge", "c5d.9xlarge", "c5d.18xlarge", "c5n.large", "c5n.xlarge", "c5n.2xlarge", "c5n.4xlarge", "c5n.9xlarge", "c5n.18xlarge", "c5n.metal", "c6g.medium", "c6g.large", "c6g.xlarge", "c6g.2xlarge", "c6g.4xlarge", "c6g.8xlarge", "c6g.12xlarge", "c6g.16xlarge", "c6g.metal", "d2.xlarge", "d2.2xlarge", "d2.4xlarge", "d2.8xlarge", "g4dn.xlarge", "g4dn.2xlarge", "g4dn.4xlarge", "g4dn.8xlarge", "g4dn.12xlarge", "g4dn.16xlarge", "i3.large", "i3.xlarge", "i3.2xlarge", "i3.4xlarge", "i3.8xlarge", "i3.16xlarge", "i3.metal", "i3en.large", "i3en.xlarge", "i3en.2xlarge", "i3en.3xlarge", "i3en.6xlarge", "i3en.12xlarge", "i3en.24xlarge", "i3en.metal", "inf1.xlarge", "inf1.2xlarge", "inf1.6xlarge", "inf1.24xlarge", "m5.large", "m5.xlarge", "m5.2xlarge", "m5.4xlarge", "m5.8xlarge", "m5.12xlarge", "m5.16xlarge", "m5.24xlarge", "m5.metal", "m5a.large", "m5a.xlarge", "m5a.2xlarge", "m5a.4xlarge", "m5a.8xlarge", "m5a.12xlarge", "m5a.16xlarge", "m5a.24xlarge", "m5d.large", "m5d.xlarge", "m5d.2xlarge", "m5d.4xlarge", "m5d.8xlarge", "m5d.12xlarge", "m5d.16xlarge", "m5d.24xlarge", "m5d.metal", "m5dn.large", "m5dn.xlarge", "m5dn.2xlarge", "m5dn.4xlarge", "m5dn.8xlarge", "m5dn.12xlarge", "m5dn.16xlarge", "m5dn.24xlarge", "m5dn.metal", "m5n.large", "m5n.xlarge", "m5n.2xlarge", "m5n.4xlarge", "m5n.8xlarge", "m5n.12xlarge", "m5n.16xlarge", "m5n.24xlarge", "m5n.metal", "m6g.medium", "m6g.large", "m6g.xlarge", "m6g.2xlarge", "m6g.4xlarge", "m6g.8xlarge", "m6g.12xlarge", "m6g.16xlarge", "m6g.metal", "p3dn.24xlarge", "r5.large", "r5.xlarge", "r5.2xlarge", "r5.4xlarge", "r5.8xlarge", "r5.12xlarge", "r5.16xlarge", "r5.24xlarge", "r5.metal", "r5a.large", "r5a.xlarge", "r5a.2xlarge", "r5a.4xlarge", "r5a.8xlarge", "r5a.12xlarge", "r5a.16xlarge", "r5a.24xlarge", "r5d.large", "r5d.xlarge", "r5d.2xlarge", "r5d.4xlarge", "r5d.8xlarge", "r5d.12xlarge", "r5d.16xlarge", "r5d.24xlarge", "r5d.metal", "r5dn.large", "r5dn.xlarge", "r5dn.2xlarge", "r5dn.4xlarge", "r5dn.8xlarge", "r5dn.12xlarge", "r5dn.16xlarge", "r5dn.24xlarge", "r5dn.metal", "r5n.large", "r5n.xlarge", "r5n.2xlarge", "r5n.4xlarge", "r5n.8xlarge", "r5n.12xlarge", "r5n.16xlarge", "r5n.24xlarge", "r5n.metal", "r6g.medium", "r6g.large", "r6g.xlarge", "r6g.2xlarge", "r6g.4xlarge", "r6g.8xlarge", "r6g.12xlarge", "r6g.16xlarge", "r6g.metal", "t3.nano", "t3.micro", "t3.small", "t3.medium", "t3.large", "t3.xlarge", "t3.2xlarge", "t3a.nano", "t3a.micro", "t3a.small", "t3a.medium", "t3a.large", "t3a.xlarge", "t3a.2xlarge", "t4g.nano", "t4g.micro", "t4g.small", "t4g.medium", "t4g.large", "t4g.xlarge", "t4g.2xlarge", "u-12tb1.metal", "u-18tb1.metal", "u-24tb1.metal", "u-6tb1.56xlarge", "u-6tb1.112xlarge", "u-6tb1.metal", "u-9tb1.metal", "x1.16xlarge", "x1.32xlarge", "x1e.xlarge", "x1e.2xlarge", "x1e.4xlarge", "x1e.8xlarge", "x1e.16xlarge", "x1e.32xlarge", ), "us-gov-west-1": strset.New( "c1.medium", "c1.xlarge", "c3.large", "c3.xlarge", "c3.2xlarge", "c3.4xlarge", "c3.8xlarge", "c4.large", "c4.xlarge", "c4.2xlarge", "c4.4xlarge", "c4.8xlarge", "c5.large", "c5.xlarge", "c5.2xlarge", "c5.4xlarge", "c5.9xlarge", "c5.12xlarge", "c5.18xlarge", "c5.24xlarge", "c5.metal", "c5a.large", "c5a.xlarge", "c5a.2xlarge", "c5a.4xlarge", "c5a.8xlarge", "c5a.12xlarge", "c5a.16xlarge", "c5a.24xlarge", "c5d.large", "c5d.xlarge", "c5d.2xlarge", "c5d.4xlarge", "c5d.9xlarge", "c5d.12xlarge", "c5d.18xlarge", "c5d.24xlarge", "c5d.metal", "c5n.large", "c5n.xlarge", "c5n.2xlarge", "c5n.4xlarge", "c5n.9xlarge", "c5n.18xlarge", "c5n.metal", "c6g.medium", "c6g.large", "c6g.xlarge", "c6g.2xlarge", "c6g.4xlarge", "c6g.8xlarge", "c6g.12xlarge", "c6g.16xlarge", "c6g.metal", "cc2.8xlarge", "d2.xlarge", "d2.2xlarge", "d2.4xlarge", "d2.8xlarge", "d3.xlarge", "d3.2xlarge", "d3.4xlarge", "d3.8xlarge", "f1.2xlarge", "f1.4xlarge", "f1.16xlarge", "g3.4xlarge", "g3.8xlarge", "g3.16xlarge", "g3s.xlarge", "g4dn.xlarge", "g4dn.2xlarge", "g4dn.4xlarge", "g4dn.8xlarge", "g4dn.12xlarge", "g4dn.16xlarge", "g4dn.metal", "hpc6a.48xlarge", "hs1.8xlarge", "i2.xlarge", "i2.2xlarge", "i2.4xlarge", "i2.8xlarge", "i3.large", "i3.xlarge", "i3.2xlarge", "i3.4xlarge", "i3.8xlarge", "i3.16xlarge", "i3.metal", "i3en.large", "i3en.xlarge", "i3en.2xlarge", "i3en.3xlarge", "i3en.6xlarge", "i3en.12xlarge", "i3en.24xlarge", "i3en.metal", "i3p.16xlarge", "inf1.xlarge", "inf1.2xlarge", "inf1.6xlarge", "inf1.24xlarge", "m1.small", "m1.medium", "m1.large", "m1.xlarge", "m2.xlarge", "m2.2xlarge", "m2.4xlarge", "m3.medium", "m3.large", "m3.xlarge", "m3.2xlarge", "m4.large", "m4.xlarge", "m4.2xlarge", "m4.4xlarge", "m4.10xlarge", "m4.16xlarge", "m5.large", "m5.xlarge", "m5.2xlarge", "m5.4xlarge", "m5.8xlarge", "m5.12xlarge", "m5.16xlarge", "m5.24xlarge", "m5.metal", "m5a.large", "m5a.xlarge", "m5a.2xlarge", "m5a.4xlarge", "m5a.8xlarge", "m5a.12xlarge", "m5a.16xlarge", "m5a.24xlarge", "m5ad.large", "m5ad.xlarge", "m5ad.2xlarge", "m5ad.4xlarge", "m5ad.8xlarge", "m5ad.12xlarge", "m5ad.16xlarge", "m5ad.24xlarge", "m5d.large", "m5d.xlarge", "m5d.2xlarge", "m5d.4xlarge", "m5d.8xlarge", "m5d.12xlarge", "m5d.16xlarge", "m5d.24xlarge", "m5d.metal", "m5dn.large", "m5dn.xlarge", "m5dn.2xlarge", "m5dn.4xlarge", "m5dn.8xlarge", "m5dn.12xlarge", "m5dn.16xlarge", "m5dn.24xlarge", "m5dn.metal", "m5n.large", "m5n.xlarge", "m5n.2xlarge", "m5n.4xlarge", "m5n.8xlarge", "m5n.12xlarge", "m5n.16xlarge", "m5n.24xlarge", "m5n.metal", "m6g.medium", "m6g.large", "m6g.xlarge", "m6g.2xlarge", "m6g.4xlarge", "m6g.8xlarge", "m6g.12xlarge", "m6g.16xlarge", "m6g.metal", "p2.xlarge", "p2.8xlarge", "p2.16xlarge", "p3.2xlarge", "p3.8xlarge", "p3.16xlarge", "p3dn.24xlarge", "r3.large", "r3.xlarge", "r3.2xlarge", "r3.4xlarge", "r3.8xlarge", "r4.large", "r4.xlarge", "r4.2xlarge", "r4.4xlarge", "r4.8xlarge", "r4.16xlarge", "r5.large", "r5.xlarge", "r5.2xlarge", "r5.4xlarge", "r5.8xlarge", "r5.12xlarge", "r5.16xlarge", "r5.24xlarge", "r5.metal", "r5a.large", "r5a.xlarge", "r5a.2xlarge", "r5a.4xlarge", "r5a.8xlarge", "r5a.12xlarge", "r5a.16xlarge", "r5a.24xlarge", "r5ad.large", "r5ad.xlarge", "r5ad.2xlarge", "r5ad.4xlarge", "r5ad.8xlarge", "r5ad.12xlarge", "r5ad.16xlarge", "r5ad.24xlarge", "r5d.large", "r5d.xlarge", "r5d.2xlarge", "r5d.4xlarge", "r5d.8xlarge", "r5d.12xlarge", "r5d.16xlarge", "r5d.24xlarge", "r5d.metal", "r5dn.large", "r5dn.xlarge", "r5dn.2xlarge", "r5dn.4xlarge", "r5dn.8xlarge", "r5dn.12xlarge", "r5dn.16xlarge", "r5dn.24xlarge", "r5dn.metal", "r5n.large", "r5n.xlarge", "r5n.2xlarge", "r5n.4xlarge", "r5n.8xlarge", "r5n.12xlarge", "r5n.16xlarge", "r5n.24xlarge", "r5n.metal", "r6g.medium", "r6g.large", "r6g.xlarge", "r6g.2xlarge", "r6g.4xlarge", "r6g.8xlarge", "r6g.12xlarge", "r6g.16xlarge", "r6g.metal", "t1.micro", "t2.nano", "t2.micro", "t2.small", "t2.medium", "t2.large", "t2.xlarge", "t2.2xlarge", "t3.nano", "t3.micro", "t3.small", "t3.medium", "t3.large", "t3.xlarge", "t3.2xlarge", "t3a.nano", "t3a.micro", "t3a.small", "t3a.medium", "t3a.large", "t3a.xlarge", "t3a.2xlarge", "t4g.nano", "t4g.micro", "t4g.small", "t4g.medium", "t4g.large", "t4g.xlarge", "t4g.2xlarge", "u-12tb1.112xlarge", "u-12tb1.metal", "u-18tb1.metal", "u-24tb1.metal", "u-6tb1.56xlarge", "u-6tb1.112xlarge", "u-6tb1.metal", "u-9tb1.112xlarge", "u-9tb1.metal", "x1.16xlarge", "x1.32xlarge", "x1e.xlarge", "x1e.2xlarge", "x1e.4xlarge", "x1e.8xlarge", "x1e.16xlarge", "x1e.32xlarge", ), "us-west-1": strset.New( "c1.medium", "c1.xlarge", "c3.large", "c3.xlarge", "c3.2xlarge", "c3.4xlarge", "c3.8xlarge", "c4.large", "c4.xlarge", "c4.2xlarge", "c4.4xlarge", "c4.8xlarge", "c5.large", "c5.xlarge", "c5.2xlarge", "c5.4xlarge", "c5.9xlarge", "c5.12xlarge", "c5.18xlarge", "c5.24xlarge", "c5.metal", "c5a.large", "c5a.xlarge", "c5a.2xlarge", "c5a.4xlarge", "c5a.8xlarge", "c5a.12xlarge", "c5a.16xlarge", "c5a.24xlarge", "c5d.large", "c5d.xlarge", "c5d.2xlarge", "c5d.4xlarge", "c5d.9xlarge", "c5d.12xlarge", "c5d.18xlarge", "c5d.24xlarge", "c5d.metal", "c5n.large", "c5n.xlarge", "c5n.2xlarge", "c5n.4xlarge", "c5n.9xlarge", "c5n.18xlarge", "c5n.metal", "c6g.medium", "c6g.large", "c6g.xlarge", "c6g.2xlarge", "c6g.4xlarge", "c6g.8xlarge", "c6g.12xlarge", "c6g.16xlarge", "c6g.metal", "c6gd.medium", "c6gd.large", "c6gd.xlarge", "c6gd.2xlarge", "c6gd.4xlarge", "c6gd.8xlarge", "c6gd.12xlarge", "c6gd.16xlarge", "c6gd.metal", "c6gn.medium", "c6gn.large", "c6gn.xlarge", "c6gn.2xlarge", "c6gn.4xlarge", "c6gn.8xlarge", "c6gn.12xlarge", "c6gn.16xlarge", "c6i.large", "c6i.xlarge", "c6i.2xlarge", "c6i.4xlarge", "c6i.8xlarge", "c6i.12xlarge", "c6i.16xlarge", "c6i.24xlarge", "c6i.32xlarge", "c6i.metal", "d2.xlarge", "d2.2xlarge", "d2.4xlarge", "d2.8xlarge", "g2.2xlarge", "g2.8xlarge", "g3.4xlarge", "g3.8xlarge", "g3.16xlarge", "g4dn.xlarge", "g4dn.2xlarge", "g4dn.4xlarge", "g4dn.8xlarge", "g4dn.12xlarge", "g4dn.16xlarge", "g4dn.metal", "i2.xlarge", "i2.2xlarge", "i2.4xlarge", "i2.8xlarge", "i3.large", "i3.xlarge", "i3.2xlarge", "i3.4xlarge", "i3.8xlarge", "i3.16xlarge", "i3.metal", "i3en.large", "i3en.xlarge", "i3en.2xlarge", "i3en.3xlarge", "i3en.6xlarge", "i3en.12xlarge", "i3en.24xlarge", "i3en.metal", "inf1.xlarge", "inf1.2xlarge", "inf1.6xlarge", "inf1.24xlarge", "m1.small", "m1.medium", "m1.large", "m1.xlarge", "m2.xlarge", "m2.2xlarge", "m2.4xlarge", "m3.medium", "m3.large", "m3.xlarge", "m3.2xlarge", "m4.large", "m4.xlarge", "m4.2xlarge", "m4.4xlarge", "m4.10xlarge", "m4.16xlarge", "m5.large", "m5.xlarge", "m5.2xlarge", "m5.4xlarge", "m5.8xlarge", "m5.12xlarge", "m5.16xlarge", "m5.24xlarge", "m5.metal", "m5a.large", "m5a.xlarge", "m5a.2xlarge", "m5a.4xlarge", "m5a.8xlarge", "m5a.12xlarge", "m5a.16xlarge", "m5a.24xlarge", "m5ad.large", "m5ad.xlarge", "m5ad.2xlarge", "m5ad.4xlarge", "m5ad.8xlarge", "m5ad.12xlarge", "m5ad.16xlarge", "m5ad.24xlarge", "m5d.large", "m5d.xlarge", "m5d.2xlarge", "m5d.4xlarge", "m5d.8xlarge", "m5d.12xlarge", "m5d.16xlarge", "m5d.24xlarge", "m5d.metal", "m5zn.large", "m5zn.xlarge", "m5zn.2xlarge", "m5zn.3xlarge", "m5zn.6xlarge", "m5zn.12xlarge", "m5zn.metal", "m6g.medium", "m6g.large", "m6g.xlarge", "m6g.2xlarge", "m6g.4xlarge", "m6g.8xlarge", "m6g.12xlarge", "m6g.16xlarge", "m6g.metal", "m6gd.medium", "m6gd.large", "m6gd.xlarge", "m6gd.2xlarge", "m6gd.4xlarge", "m6gd.8xlarge", "m6gd.12xlarge", "m6gd.16xlarge", "m6gd.metal", "m6i.large", "m6i.xlarge", "m6i.2xlarge", "m6i.4xlarge", "m6i.8xlarge", "m6i.12xlarge", "m6i.16xlarge", "m6i.24xlarge", "m6i.32xlarge", "m6i.metal", "r3.large", "r3.xlarge", "r3.2xlarge", "r3.4xlarge", "r3.8xlarge", "r4.large", "r4.xlarge", "r4.2xlarge", "r4.4xlarge", "r4.8xlarge", "r4.16xlarge", "r5.large", "r5.xlarge", "r5.2xlarge", "r5.4xlarge", "r5.8xlarge", "r5.12xlarge", "r5.16xlarge", "r5.24xlarge", "r5.metal", "r5a.large", "r5a.xlarge", "r5a.2xlarge", "r5a.4xlarge", "r5a.8xlarge", "r5a.12xlarge", "r5a.16xlarge", "r5a.24xlarge", "r5ad.large", "r5ad.xlarge", "r5ad.2xlarge", "r5ad.4xlarge", "r5ad.8xlarge", "r5ad.12xlarge", "r5ad.16xlarge", "r5ad.24xlarge", "r5d.large", "r5d.xlarge", "r5d.2xlarge", "r5d.4xlarge", "r5d.8xlarge", "r5d.12xlarge", "r5d.16xlarge", "r5d.24xlarge", "r5d.metal", "r5n.large", "r5n.xlarge", "r5n.2xlarge", "r5n.4xlarge", "r5n.8xlarge", "r5n.12xlarge", "r5n.16xlarge", "r5n.24xlarge", "r5n.metal", "r6g.medium", "r6g.large", "r6g.xlarge", "r6g.2xlarge", "r6g.4xlarge", "r6g.8xlarge", "r6g.12xlarge", "r6g.16xlarge", "r6g.metal", "r6gd.medium", "r6gd.large", "r6gd.xlarge", "r6gd.2xlarge", "r6gd.4xlarge", "r6gd.8xlarge", "r6gd.12xlarge", "r6gd.16xlarge", "r6gd.metal", "r6i.large", "r6i.xlarge", "r6i.2xlarge", "r6i.4xlarge", "r6i.8xlarge", "r6i.12xlarge", "r6i.16xlarge", "r6i.24xlarge", "r6i.32xlarge", "r6i.metal", "t1.micro", "t2.nano", "t2.micro", "t2.small", "t2.medium", "t2.large", "t2.xlarge", "t2.2xlarge", "t3.nano", "t3.micro", "t3.small", "t3.medium", "t3.large", "t3.xlarge", "t3.2xlarge", "t3a.nano", "t3a.micro", "t3a.small", "t3a.medium", "t3a.large", "t3a.xlarge", "t3a.2xlarge", "t4g.nano", "t4g.micro", "t4g.small", "t4g.medium", "t4g.large", "t4g.xlarge", "t4g.2xlarge", "z1d.large", "z1d.xlarge", "z1d.2xlarge", "z1d.3xlarge", "z1d.6xlarge", "z1d.12xlarge", "z1d.metal", ), "us-west-2": strset.New( "a1.medium", "a1.large", "a1.xlarge", "a1.2xlarge", "a1.4xlarge", "a1.metal", "c1.medium", "c1.xlarge", "c3.large", "c3.xlarge", "c3.2xlarge", "c3.4xlarge", "c3.8xlarge", "c4.large", "c4.xlarge", "c4.2xlarge", "c4.4xlarge", "c4.8xlarge", "c5.large", "c5.xlarge", "c5.2xlarge", "c5.4xlarge", "c5.9xlarge", "c5.12xlarge", "c5.18xlarge", "c5.24xlarge", "c5.metal", "c5a.large", "c5a.xlarge", "c5a.2xlarge", "c5a.4xlarge", "c5a.8xlarge", "c5a.12xlarge", "c5a.16xlarge", "c5a.24xlarge", "c5ad.large", "c5ad.xlarge", "c5ad.2xlarge", "c5ad.4xlarge", "c5ad.8xlarge", "c5ad.12xlarge", "c5ad.16xlarge", "c5ad.24xlarge", "c5d.large", "c5d.xlarge", "c5d.2xlarge", "c5d.4xlarge", "c5d.9xlarge", "c5d.12xlarge", "c5d.18xlarge", "c5d.24xlarge", "c5d.metal", "c5n.large", "c5n.xlarge", "c5n.2xlarge", "c5n.4xlarge", "c5n.9xlarge", "c5n.18xlarge", "c5n.metal", "c6a.large", "c6a.xlarge", "c6a.2xlarge", "c6a.4xlarge", "c6a.8xlarge", "c6a.12xlarge", "c6a.16xlarge", "c6a.24xlarge", "c6a.32xlarge", "c6a.48xlarge", "c6a.metal", "c6g.medium", "c6g.large", "c6g.xlarge", "c6g.2xlarge", "c6g.4xlarge", "c6g.8xlarge", "c6g.12xlarge", "c6g.16xlarge", "c6g.metal", "c6gd.medium", "c6gd.large", "c6gd.xlarge", "c6gd.2xlarge", "c6gd.4xlarge", "c6gd.8xlarge", "c6gd.12xlarge", "c6gd.16xlarge", "c6gd.metal", "c6gn.medium", "c6gn.large", "c6gn.xlarge", "c6gn.2xlarge", "c6gn.4xlarge", "c6gn.8xlarge", "c6gn.12xlarge", "c6gn.16xlarge", "c6gn.metal", "c6i.large", "c6i.xlarge", "c6i.2xlarge", "c6i.4xlarge", "c6i.8xlarge", "c6i.12xlarge", "c6i.16xlarge", "c6i.24xlarge", "c6i.32xlarge", "c6i.metal", "cc2.8xlarge", "cr1.8xlarge", "d2.xlarge", "d2.2xlarge", "d2.4xlarge", "d2.8xlarge", "d3.xlarge", "d3.2xlarge", "d3.4xlarge", "d3.8xlarge", "d3en.xlarge", "d3en.2xlarge", "d3en.4xlarge", "d3en.6xlarge", "d3en.8xlarge", "d3en.12xlarge", "dl1.24xlarge", "f1.2xlarge", "f1.4xlarge", "f1.16xlarge", "g2.2xlarge", "g2.8xlarge", "g3.4xlarge", "g3.8xlarge", "g3.16xlarge", "g3s.xlarge", "g4ad.xlarge", "g4ad.2xlarge", "g4ad.4xlarge", "g4ad.8xlarge", "g4ad.16xlarge", "g4dn.xlarge", "g4dn.2xlarge", "g4dn.4xlarge", "g4dn.8xlarge", "g4dn.12xlarge", "g4dn.16xlarge", "g4dn.metal", "g5.xlarge", "g5.2xlarge", "g5.4xlarge", "g5.8xlarge", "g5.12xlarge", "g5.16xlarge", "g5.24xlarge", "g5.48xlarge", "g5g.xlarge", "g5g.2xlarge", "g5g.4xlarge", "g5g.8xlarge", "g5g.16xlarge", "g5g.metal", "h1.2xlarge", "h1.4xlarge", "h1.8xlarge", "h1.16xlarge", "hs1.8xlarge", "i2.xlarge", "i2.2xlarge", "i2.4xlarge", "i2.8xlarge", "i3.large", "i3.xlarge", "i3.2xlarge", "i3.4xlarge", "i3.8xlarge", "i3.16xlarge", "i3.metal", "i3en.large", "i3en.xlarge", "i3en.2xlarge", "i3en.3xlarge", "i3en.6xlarge", "i3en.12xlarge", "i3en.24xlarge", "i3en.metal", "im4gn.large", "im4gn.xlarge", "im4gn.2xlarge", "im4gn.4xlarge", "im4gn.8xlarge", "im4gn.16xlarge", "inf1.xlarge", "inf1.2xlarge", "inf1.6xlarge", "inf1.24xlarge", "is4gen.medium", "is4gen.large", "is4gen.xlarge", "is4gen.2xlarge", "is4gen.4xlarge", "is4gen.8xlarge", "m1.small", "m1.medium", "m1.large", "m1.xlarge", "m2.xlarge", "m2.2xlarge", "m2.4xlarge", "m3.medium", "m3.large", "m3.xlarge", "m3.2xlarge", "m4.large", "m4.xlarge", "m4.2xlarge", "m4.4xlarge", "m4.10xlarge", "m4.16xlarge", "m5.large", "m5.xlarge", "m5.2xlarge", "m5.4xlarge", "m5.8xlarge", "m5.12xlarge", "m5.16xlarge", "m5.24xlarge", "m5.metal", "m5a.large", "m5a.xlarge", "m5a.2xlarge", "m5a.4xlarge", "m5a.8xlarge", "m5a.12xlarge", "m5a.16xlarge", "m5a.24xlarge", "m5ad.large", "m5ad.xlarge", "m5ad.2xlarge", "m5ad.4xlarge", "m5ad.8xlarge", "m5ad.12xlarge", "m5ad.16xlarge", "m5ad.24xlarge", "m5d.large", "m5d.xlarge", "m5d.2xlarge", "m5d.4xlarge", "m5d.8xlarge", "m5d.12xlarge", "m5d.16xlarge", "m5d.24xlarge", "m5d.metal", "m5dn.large", "m5dn.xlarge", "m5dn.2xlarge", "m5dn.4xlarge", "m5dn.8xlarge", "m5dn.12xlarge", "m5dn.16xlarge", "m5dn.24xlarge", "m5dn.metal", "m5n.large", "m5n.xlarge", "m5n.2xlarge", "m5n.4xlarge", "m5n.8xlarge", "m5n.12xlarge", "m5n.16xlarge", "m5n.24xlarge", "m5n.metal", "m5zn.large", "m5zn.xlarge", "m5zn.2xlarge", "m5zn.3xlarge", "m5zn.6xlarge", "m5zn.12xlarge", "m5zn.metal", "m6a.large", "m6a.xlarge", "m6a.2xlarge", "m6a.4xlarge", "m6a.8xlarge", "m6a.12xlarge", "m6a.16xlarge", "m6a.24xlarge", "m6a.32xlarge", "m6a.48xlarge", "m6a.metal", "m6g.medium", "m6g.large", "m6g.xlarge", "m6g.2xlarge", "m6g.4xlarge", "m6g.8xlarge", "m6g.12xlarge", "m6g.16xlarge", "m6g.metal", "m6gd.medium", "m6gd.large", "m6gd.xlarge", "m6gd.2xlarge", "m6gd.4xlarge", "m6gd.8xlarge", "m6gd.12xlarge", "m6gd.16xlarge", "m6gd.metal", "m6i.large", "m6i.xlarge", "m6i.2xlarge", "m6i.4xlarge", "m6i.8xlarge", "m6i.12xlarge", "m6i.16xlarge", "m6i.24xlarge", "m6i.32xlarge", "m6i.metal", "mac1.metal", "mac2.metal", "p2.xlarge", "p2.8xlarge", "p2.16xlarge", "p3.2xlarge", "p3.8xlarge", "p3.16xlarge", "p3dn.24xlarge", "p4d.24xlarge", "r3.large", "r3.xlarge", "r3.2xlarge", "r3.4xlarge", "r3.8xlarge", "r4.large", "r4.xlarge", "r4.2xlarge", "r4.4xlarge", "r4.8xlarge", "r4.16xlarge", "r5.large", "r5.xlarge", "r5.2xlarge", "r5.4xlarge", "r5.8xlarge", "r5.12xlarge", "r5.16xlarge", "r5.24xlarge", "r5.metal", "r5a.large", "r5a.xlarge", "r5a.2xlarge", "r5a.4xlarge", "r5a.8xlarge", "r5a.12xlarge", "r5a.16xlarge", "r5a.24xlarge", "r5ad.large", "r5ad.xlarge", "r5ad.2xlarge", "r5ad.4xlarge", "r5ad.8xlarge", "r5ad.12xlarge", "r5ad.16xlarge", "r5ad.24xlarge", "r5b.large", "r5b.xlarge", "r5b.2xlarge", "r5b.4xlarge", "r5b.8xlarge", "r5b.12xlarge", "r5b.16xlarge", "r5b.24xlarge", "r5b.metal", "r5d.large", "r5d.xlarge", "r5d.2xlarge", "r5d.4xlarge", "r5d.8xlarge", "r5d.12xlarge", "r5d.16xlarge", "r5d.24xlarge", "r5d.metal", "r5dn.large", "r5dn.xlarge", "r5dn.2xlarge", "r5dn.4xlarge", "r5dn.8xlarge", "r5dn.12xlarge", "r5dn.16xlarge", "r5dn.24xlarge", "r5dn.metal", "r5n.large", "r5n.xlarge", "r5n.2xlarge", "r5n.4xlarge", "r5n.8xlarge", "r5n.12xlarge", "r5n.16xlarge", "r5n.24xlarge", "r5n.metal", "r6g.medium", "r6g.large", "r6g.xlarge", "r6g.2xlarge", "r6g.4xlarge", "r6g.8xlarge", "r6g.12xlarge", "r6g.16xlarge", "r6g.metal", "r6gd.medium", "r6gd.large", "r6gd.xlarge", "r6gd.2xlarge", "r6gd.4xlarge", "r6gd.8xlarge", "r6gd.12xlarge", "r6gd.16xlarge", "r6gd.metal", "r6i.large", "r6i.xlarge", "r6i.2xlarge", "r6i.4xlarge", "r6i.8xlarge", "r6i.12xlarge", "r6i.16xlarge", "r6i.24xlarge", "r6i.32xlarge", "r6i.metal", "t1.micro", "t2.nano", "t2.micro", "t2.small", "t2.medium", "t2.large", "t2.xlarge", "t2.2xlarge", "t3.nano", "t3.micro", "t3.small", "t3.medium", "t3.large", "t3.xlarge", "t3.2xlarge", "t3a.nano", "t3a.micro", "t3a.small", "t3a.medium", "t3a.large", "t3a.xlarge", "t3a.2xlarge", "t4g.nano", "t4g.micro", "t4g.small", "t4g.medium", "t4g.large", "t4g.xlarge", "t4g.2xlarge", "u-12tb1.112xlarge", "u-12tb1.metal", "u-3tb1.56xlarge", "u-6tb1.56xlarge", "u-6tb1.112xlarge", "u-6tb1.metal", "u-9tb1.112xlarge", "u-9tb1.metal", "vt1.3xlarge", "vt1.6xlarge", "vt1.24xlarge", "x1.16xlarge", "x1.32xlarge", "x1e.xlarge", "x1e.2xlarge", "x1e.4xlarge", "x1e.8xlarge", "x1e.16xlarge", "x1e.32xlarge", "x2gd.medium", "x2gd.large", "x2gd.xlarge", "x2gd.2xlarge", "x2gd.4xlarge", "x2gd.8xlarge", "x2gd.12xlarge", "x2gd.16xlarge", "x2gd.metal", "x2iedn.xlarge", "x2iedn.2xlarge", "x2iedn.4xlarge", "x2iedn.8xlarge", "x2iedn.16xlarge", "x2iedn.24xlarge", "x2iedn.32xlarge", "x2iezn.2xlarge", "x2iezn.4xlarge", "x2iezn.6xlarge", "x2iezn.8xlarge", "x2iezn.12xlarge", "x2iezn.metal", "z1d.large", "z1d.xlarge", "z1d.2xlarge", "z1d.3xlarge", "z1d.6xlarge", "z1d.12xlarge", "z1d.metal", ), } // region -> instance type -> instance metadata var InstanceMetadatas = map[string]map[string]InstanceMetadata{ "af-south-1": { "c5.large": {Region: "af-south-1", Type: "c5.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.114}, "c5.xlarge": {Region: "af-south-1", Type: "c5.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.228}, "c5.2xlarge": {Region: "af-south-1", Type: "c5.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.456}, "c5.4xlarge": {Region: "af-south-1", Type: "c5.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.912}, "c5.9xlarge": {Region: "af-south-1", Type: "c5.9xlarge", Memory: kresource.MustParse("73728Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 2.052}, "c5.12xlarge": {Region: "af-south-1", Type: "c5.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.736}, "c5.18xlarge": {Region: "af-south-1", Type: "c5.18xlarge", Memory: kresource.MustParse("147456Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 4.104}, "c5.24xlarge": {Region: "af-south-1", Type: "c5.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.472}, "c5.metal": {Region: "af-south-1", Type: "c5.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.472}, "c5a.large": {Region: "af-south-1", Type: "c5a.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.103}, "c5a.xlarge": {Region: "af-south-1", Type: "c5a.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.206}, "c5a.2xlarge": {Region: "af-south-1", Type: "c5a.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.412}, "c5a.4xlarge": {Region: "af-south-1", Type: "c5a.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.824}, "c5a.8xlarge": {Region: "af-south-1", Type: "c5a.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.648}, "c5a.12xlarge": {Region: "af-south-1", Type: "c5a.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.472}, "c5a.16xlarge": {Region: "af-south-1", Type: "c5a.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.296}, "c5a.24xlarge": {Region: "af-south-1", Type: "c5a.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.944}, "c5ad.large": {Region: "af-south-1", Type: "c5ad.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.117}, "c5ad.xlarge": {Region: "af-south-1", Type: "c5ad.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.234}, "c5ad.2xlarge": {Region: "af-south-1", Type: "c5ad.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.468}, "c5ad.4xlarge": {Region: "af-south-1", Type: "c5ad.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.936}, "c5ad.8xlarge": {Region: "af-south-1", Type: "c5ad.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.872}, "c5ad.12xlarge": {Region: "af-south-1", Type: "c5ad.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.808}, "c5ad.16xlarge": {Region: "af-south-1", Type: "c5ad.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.744}, "c5ad.24xlarge": {Region: "af-south-1", Type: "c5ad.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.616}, "c5d.large": {Region: "af-south-1", Type: "c5d.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.13}, "c5d.xlarge": {Region: "af-south-1", Type: "c5d.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.26}, "c5d.2xlarge": {Region: "af-south-1", Type: "c5d.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.52}, "c5d.4xlarge": {Region: "af-south-1", Type: "c5d.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.04}, "c5d.9xlarge": {Region: "af-south-1", Type: "c5d.9xlarge", Memory: kresource.MustParse("73728Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 2.34}, "c5d.12xlarge": {Region: "af-south-1", Type: "c5d.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.12}, "c5d.18xlarge": {Region: "af-south-1", Type: "c5d.18xlarge", Memory: kresource.MustParse("147456Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 4.68}, "c5d.24xlarge": {Region: "af-south-1", Type: "c5d.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.24}, "c5d.metal": {Region: "af-south-1", Type: "c5d.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.24}, "c5n.large": {Region: "af-south-1", Type: "c5n.large", Memory: kresource.MustParse("5376Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.145}, "c5n.xlarge": {Region: "af-south-1", Type: "c5n.xlarge", Memory: kresource.MustParse("10752Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.29}, "c5n.2xlarge": {Region: "af-south-1", Type: "c5n.2xlarge", Memory: kresource.MustParse("21504Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.581}, "c5n.4xlarge": {Region: "af-south-1", Type: "c5n.4xlarge", Memory: kresource.MustParse("43008Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.162}, "c5n.9xlarge": {Region: "af-south-1", Type: "c5n.9xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 2.614}, "c5n.18xlarge": {Region: "af-south-1", Type: "c5n.18xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 5.227}, "c5n.metal": {Region: "af-south-1", Type: "c5n.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 5.227}, "d2.xlarge": {Region: "af-south-1", Type: "d2.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.875}, "d2.2xlarge": {Region: "af-south-1", Type: "d2.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.75}, "d2.4xlarge": {Region: "af-south-1", Type: "d2.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 3.5}, "d2.8xlarge": {Region: "af-south-1", Type: "d2.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 7.0}, "g4dn.xlarge": {Region: "af-south-1", Type: "g4dn.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 1, Inf: 0, Price: 0.698}, "g4dn.2xlarge": {Region: "af-south-1", Type: "g4dn.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 1, Inf: 0, Price: 0.998}, "g4dn.4xlarge": {Region: "af-south-1", Type: "g4dn.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 1, Inf: 0, Price: 1.597}, "g4dn.8xlarge": {Region: "af-south-1", Type: "g4dn.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 1, Inf: 0, Price: 2.887}, "g4dn.12xlarge": {Region: "af-south-1", Type: "g4dn.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 4, Inf: 0, Price: 5.19}, "g4dn.16xlarge": {Region: "af-south-1", Type: "g4dn.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 1, Inf: 0, Price: 5.774}, "g4dn.metal": {Region: "af-south-1", Type: "g4dn.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 8, Inf: 0, Price: 10.381}, "i3.large": {Region: "af-south-1", Type: "i3.large", Memory: kresource.MustParse("15616Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.205}, "i3.xlarge": {Region: "af-south-1", Type: "i3.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.41}, "i3.2xlarge": {Region: "af-south-1", Type: "i3.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.82}, "i3.4xlarge": {Region: "af-south-1", Type: "i3.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.64}, "i3.8xlarge": {Region: "af-south-1", Type: "i3.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 3.28}, "i3.16xlarge": {Region: "af-south-1", Type: "i3.16xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 6.56}, "i3.metal": {Region: "af-south-1", Type: "i3.metal", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 6.56}, "i3en.large": {Region: "af-south-1", Type: "i3en.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.297}, "i3en.xlarge": {Region: "af-south-1", Type: "i3en.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.594}, "i3en.2xlarge": {Region: "af-south-1", Type: "i3en.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.188}, "i3en.3xlarge": {Region: "af-south-1", Type: "i3en.3xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("12"), GPU: 0, Inf: 0, Price: 1.782}, "i3en.6xlarge": {Region: "af-south-1", Type: "i3en.6xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 0, Price: 3.564}, "i3en.12xlarge": {Region: "af-south-1", Type: "i3en.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 7.128}, "i3en.24xlarge": {Region: "af-south-1", Type: "i3en.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 14.256}, "i3en.metal": {Region: "af-south-1", Type: "i3en.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 14.256}, "inf1.24xlarge": {Region: "af-south-1", Type: "inf1.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 16, Price: 6.249}, "m5.large": {Region: "af-south-1", Type: "m5.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.127}, "m5.xlarge": {Region: "af-south-1", Type: "m5.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.254}, "m5.2xlarge": {Region: "af-south-1", Type: "m5.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.508}, "m5.4xlarge": {Region: "af-south-1", Type: "m5.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.016}, "m5.8xlarge": {Region: "af-south-1", Type: "m5.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.032}, "m5.12xlarge": {Region: "af-south-1", Type: "m5.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.048}, "m5.16xlarge": {Region: "af-south-1", Type: "m5.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.064}, "m5.24xlarge": {Region: "af-south-1", Type: "m5.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.096}, "m5.metal": {Region: "af-south-1", Type: "m5.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.096}, "m5d.large": {Region: "af-south-1", Type: "m5d.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.15}, "m5d.xlarge": {Region: "af-south-1", Type: "m5d.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.3}, "m5d.2xlarge": {Region: "af-south-1", Type: "m5d.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.6}, "m5d.4xlarge": {Region: "af-south-1", Type: "m5d.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.2}, "m5d.8xlarge": {Region: "af-south-1", Type: "m5d.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.4}, "m5d.12xlarge": {Region: "af-south-1", Type: "m5d.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.6}, "m5d.16xlarge": {Region: "af-south-1", Type: "m5d.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.8}, "m5d.24xlarge": {Region: "af-south-1", Type: "m5d.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.2}, "m5d.metal": {Region: "af-south-1", Type: "m5d.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.2}, "r5.large": {Region: "af-south-1", Type: "r5.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.168}, "r5.xlarge": {Region: "af-south-1", Type: "r5.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.336}, "r5.2xlarge": {Region: "af-south-1", Type: "r5.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.672}, "r5.4xlarge": {Region: "af-south-1", Type: "r5.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.344}, "r5.8xlarge": {Region: "af-south-1", Type: "r5.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.688}, "r5.12xlarge": {Region: "af-south-1", Type: "r5.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.032}, "r5.16xlarge": {Region: "af-south-1", Type: "r5.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.376}, "r5.24xlarge": {Region: "af-south-1", Type: "r5.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.064}, "r5.metal": {Region: "af-south-1", Type: "r5.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.064}, "r5d.large": {Region: "af-south-1", Type: "r5d.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.19}, "r5d.xlarge": {Region: "af-south-1", Type: "r5d.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.38}, "r5d.2xlarge": {Region: "af-south-1", Type: "r5d.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.76}, "r5d.4xlarge": {Region: "af-south-1", Type: "r5d.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.52}, "r5d.8xlarge": {Region: "af-south-1", Type: "r5d.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 3.04}, "r5d.12xlarge": {Region: "af-south-1", Type: "r5d.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.56}, "r5d.16xlarge": {Region: "af-south-1", Type: "r5d.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 6.08}, "r5d.24xlarge": {Region: "af-south-1", Type: "r5d.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 9.12}, "r5d.metal": {Region: "af-south-1", Type: "r5d.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 9.12}, "r5dn.large": {Region: "af-south-1", Type: "r5dn.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.221}, "r5dn.xlarge": {Region: "af-south-1", Type: "r5dn.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.442}, "r5dn.2xlarge": {Region: "af-south-1", Type: "r5dn.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.884}, "r5dn.4xlarge": {Region: "af-south-1", Type: "r5dn.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.768}, "r5dn.8xlarge": {Region: "af-south-1", Type: "r5dn.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 3.536}, "r5dn.12xlarge": {Region: "af-south-1", Type: "r5dn.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 5.304}, "r5dn.16xlarge": {Region: "af-south-1", Type: "r5dn.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 7.072}, "r5dn.24xlarge": {Region: "af-south-1", Type: "r5dn.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 10.608}, "r5dn.metal": {Region: "af-south-1", Type: "r5dn.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 10.608}, "t3.nano": {Region: "af-south-1", Type: "t3.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0068}, "t3.micro": {Region: "af-south-1", Type: "t3.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0136}, "t3.small": {Region: "af-south-1", Type: "t3.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0271}, "t3.medium": {Region: "af-south-1", Type: "t3.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0542}, "t3.large": {Region: "af-south-1", Type: "t3.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.1085}, "t3.xlarge": {Region: "af-south-1", Type: "t3.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.217}, "t3.2xlarge": {Region: "af-south-1", Type: "t3.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.4339}, "x1.16xlarge": {Region: "af-south-1", Type: "x1.16xlarge", Memory: kresource.MustParse("999424Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 9.524}, "x1.32xlarge": {Region: "af-south-1", Type: "x1.32xlarge", Memory: kresource.MustParse("1998848Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 19.048}, "x1e.xlarge": {Region: "af-south-1", Type: "x1e.xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 1.19}, "x1e.2xlarge": {Region: "af-south-1", Type: "x1e.2xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 2.38}, "x1e.4xlarge": {Region: "af-south-1", Type: "x1e.4xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 4.76}, "x1e.8xlarge": {Region: "af-south-1", Type: "x1e.8xlarge", Memory: kresource.MustParse("999424Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 9.52}, "x1e.16xlarge": {Region: "af-south-1", Type: "x1e.16xlarge", Memory: kresource.MustParse("1998848Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 19.04}, "x1e.32xlarge": {Region: "af-south-1", Type: "x1e.32xlarge", Memory: kresource.MustParse("3997696Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 38.08}, }, "ap-east-1": { "c5.large": {Region: "ap-east-1", Type: "c5.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.108}, "c5.xlarge": {Region: "ap-east-1", Type: "c5.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.216}, "c5.2xlarge": {Region: "ap-east-1", Type: "c5.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.432}, "c5.4xlarge": {Region: "ap-east-1", Type: "c5.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.864}, "c5.9xlarge": {Region: "ap-east-1", Type: "c5.9xlarge", Memory: kresource.MustParse("73728Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 1.944}, "c5.12xlarge": {Region: "ap-east-1", Type: "c5.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.592}, "c5.18xlarge": {Region: "ap-east-1", Type: "c5.18xlarge", Memory: kresource.MustParse("147456Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 3.888}, "c5.24xlarge": {Region: "ap-east-1", Type: "c5.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.184}, "c5.metal": {Region: "ap-east-1", Type: "c5.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.184}, "c5a.large": {Region: "ap-east-1", Type: "c5a.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.097}, "c5a.xlarge": {Region: "ap-east-1", Type: "c5a.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.194}, "c5a.2xlarge": {Region: "ap-east-1", Type: "c5a.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.388}, "c5a.4xlarge": {Region: "ap-east-1", Type: "c5a.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.776}, "c5a.8xlarge": {Region: "ap-east-1", Type: "c5a.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.552}, "c5a.12xlarge": {Region: "ap-east-1", Type: "c5a.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.328}, "c5a.16xlarge": {Region: "ap-east-1", Type: "c5a.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.104}, "c5a.24xlarge": {Region: "ap-east-1", Type: "c5a.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.656}, "c5d.large": {Region: "ap-east-1", Type: "c5d.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.123}, "c5d.xlarge": {Region: "ap-east-1", Type: "c5d.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.246}, "c5d.2xlarge": {Region: "ap-east-1", Type: "c5d.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.492}, "c5d.4xlarge": {Region: "ap-east-1", Type: "c5d.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.984}, "c5d.9xlarge": {Region: "ap-east-1", Type: "c5d.9xlarge", Memory: kresource.MustParse("73728Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 2.214}, "c5d.18xlarge": {Region: "ap-east-1", Type: "c5d.18xlarge", Memory: kresource.MustParse("147456Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 4.428}, "c5n.large": {Region: "ap-east-1", Type: "c5n.large", Memory: kresource.MustParse("5376Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.143}, "c5n.xlarge": {Region: "ap-east-1", Type: "c5n.xlarge", Memory: kresource.MustParse("10752Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.286}, "c5n.2xlarge": {Region: "ap-east-1", Type: "c5n.2xlarge", Memory: kresource.MustParse("21504Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.572}, "c5n.4xlarge": {Region: "ap-east-1", Type: "c5n.4xlarge", Memory: kresource.MustParse("43008Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.144}, "c5n.9xlarge": {Region: "ap-east-1", Type: "c5n.9xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 2.574}, "c5n.18xlarge": {Region: "ap-east-1", Type: "c5n.18xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 5.148}, "c5n.metal": {Region: "ap-east-1", Type: "c5n.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 5.148}, "c6g.medium": {Region: "ap-east-1", Type: "c6g.medium", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.043}, "c6g.large": {Region: "ap-east-1", Type: "c6g.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.086}, "c6g.xlarge": {Region: "ap-east-1", Type: "c6g.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.172}, "c6g.2xlarge": {Region: "ap-east-1", Type: "c6g.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.344}, "c6g.4xlarge": {Region: "ap-east-1", Type: "c6g.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.688}, "c6g.8xlarge": {Region: "ap-east-1", Type: "c6g.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.376}, "c6g.12xlarge": {Region: "ap-east-1", Type: "c6g.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.064}, "c6g.16xlarge": {Region: "ap-east-1", Type: "c6g.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.752}, "c6g.metal": {Region: "ap-east-1", Type: "c6g.metal", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.752}, "c6gn.medium": {Region: "ap-east-1", Type: "c6gn.medium", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.05725}, "c6gn.large": {Region: "ap-east-1", Type: "c6gn.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.1145}, "c6gn.xlarge": {Region: "ap-east-1", Type: "c6gn.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.229}, "c6gn.2xlarge": {Region: "ap-east-1", Type: "c6gn.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.458}, "c6gn.4xlarge": {Region: "ap-east-1", Type: "c6gn.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.916}, "c6gn.8xlarge": {Region: "ap-east-1", Type: "c6gn.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.832}, "c6gn.12xlarge": {Region: "ap-east-1", Type: "c6gn.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.748}, "c6gn.16xlarge": {Region: "ap-east-1", Type: "c6gn.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.664}, "d2.xlarge": {Region: "ap-east-1", Type: "d2.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.957}, "d2.2xlarge": {Region: "ap-east-1", Type: "d2.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.914}, "d2.4xlarge": {Region: "ap-east-1", Type: "d2.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 3.828}, "d2.8xlarge": {Region: "ap-east-1", Type: "d2.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 7.656}, "g4dn.xlarge": {Region: "ap-east-1", Type: "g4dn.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 1, Inf: 0, Price: 0.81}, "g4dn.2xlarge": {Region: "ap-east-1", Type: "g4dn.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 1, Inf: 0, Price: 1.158}, "g4dn.4xlarge": {Region: "ap-east-1", Type: "g4dn.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 1, Inf: 0, Price: 1.854}, "g4dn.8xlarge": {Region: "ap-east-1", Type: "g4dn.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 1, Inf: 0, Price: 3.351}, "g4dn.12xlarge": {Region: "ap-east-1", Type: "g4dn.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 4, Inf: 0, Price: 6.024}, "g4dn.16xlarge": {Region: "ap-east-1", Type: "g4dn.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 1, Inf: 0, Price: 6.702}, "g4dn.metal": {Region: "ap-east-1", Type: "g4dn.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 8, Inf: 0, Price: 12.048}, "i3.large": {Region: "ap-east-1", Type: "i3.large", Memory: kresource.MustParse("15616Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.206}, "i3.xlarge": {Region: "ap-east-1", Type: "i3.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.412}, "i3.2xlarge": {Region: "ap-east-1", Type: "i3.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.824}, "i3.4xlarge": {Region: "ap-east-1", Type: "i3.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.648}, "i3.8xlarge": {Region: "ap-east-1", Type: "i3.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 3.296}, "i3.16xlarge": {Region: "ap-east-1", Type: "i3.16xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 6.592}, "i3.metal": {Region: "ap-east-1", Type: "i3.metal", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 6.592}, "i3en.large": {Region: "ap-east-1", Type: "i3en.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.298}, "i3en.xlarge": {Region: "ap-east-1", Type: "i3en.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.597}, "i3en.2xlarge": {Region: "ap-east-1", Type: "i3en.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.193}, "i3en.3xlarge": {Region: "ap-east-1", Type: "i3en.3xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("12"), GPU: 0, Inf: 0, Price: 1.79}, "i3en.6xlarge": {Region: "ap-east-1", Type: "i3en.6xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 0, Price: 3.58}, "i3en.12xlarge": {Region: "ap-east-1", Type: "i3en.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 7.16}, "i3en.24xlarge": {Region: "ap-east-1", Type: "i3en.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 14.319}, "i3en.metal": {Region: "ap-east-1", Type: "i3en.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 14.319}, "inf1.xlarge": {Region: "ap-east-1", Type: "inf1.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 1, Price: 0.352}, "inf1.2xlarge": {Region: "ap-east-1", Type: "inf1.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 1, Price: 0.558}, "inf1.6xlarge": {Region: "ap-east-1", Type: "inf1.6xlarge", Memory: kresource.MustParse("49152Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 4, Price: 1.819}, "inf1.24xlarge": {Region: "ap-east-1", Type: "inf1.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 16, Price: 7.274}, "m5.large": {Region: "ap-east-1", Type: "m5.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.132}, "m5.xlarge": {Region: "ap-east-1", Type: "m5.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.264}, "m5.2xlarge": {Region: "ap-east-1", Type: "m5.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.528}, "m5.4xlarge": {Region: "ap-east-1", Type: "m5.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.056}, "m5.8xlarge": {Region: "ap-east-1", Type: "m5.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.112}, "m5.12xlarge": {Region: "ap-east-1", Type: "m5.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.168}, "m5.16xlarge": {Region: "ap-east-1", Type: "m5.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.224}, "m5.24xlarge": {Region: "ap-east-1", Type: "m5.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.336}, "m5.metal": {Region: "ap-east-1", Type: "m5.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.336}, "m5d.large": {Region: "ap-east-1", Type: "m5d.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.155}, "m5d.xlarge": {Region: "ap-east-1", Type: "m5d.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.31}, "m5d.2xlarge": {Region: "ap-east-1", Type: "m5d.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.62}, "m5d.4xlarge": {Region: "ap-east-1", Type: "m5d.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.24}, "m5d.8xlarge": {Region: "ap-east-1", Type: "m5d.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.48}, "m5d.12xlarge": {Region: "ap-east-1", Type: "m5d.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.72}, "m5d.16xlarge": {Region: "ap-east-1", Type: "m5d.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.96}, "m5d.24xlarge": {Region: "ap-east-1", Type: "m5d.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.44}, "m5d.metal": {Region: "ap-east-1", Type: "m5d.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.44}, "m6g.medium": {Region: "ap-east-1", Type: "m6g.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.053}, "m6g.large": {Region: "ap-east-1", Type: "m6g.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.106}, "m6g.xlarge": {Region: "ap-east-1", Type: "m6g.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.212}, "m6g.2xlarge": {Region: "ap-east-1", Type: "m6g.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.424}, "m6g.4xlarge": {Region: "ap-east-1", Type: "m6g.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.848}, "m6g.8xlarge": {Region: "ap-east-1", Type: "m6g.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.696}, "m6g.12xlarge": {Region: "ap-east-1", Type: "m6g.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.544}, "m6g.16xlarge": {Region: "ap-east-1", Type: "m6g.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.392}, "m6g.metal": {Region: "ap-east-1", Type: "m6g.metal", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.392}, "r5.large": {Region: "ap-east-1", Type: "r5.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.167}, "r5.xlarge": {Region: "ap-east-1", Type: "r5.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.334}, "r5.2xlarge": {Region: "ap-east-1", Type: "r5.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.668}, "r5.4xlarge": {Region: "ap-east-1", Type: "r5.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.336}, "r5.8xlarge": {Region: "ap-east-1", Type: "r5.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.672}, "r5.12xlarge": {Region: "ap-east-1", Type: "r5.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.008}, "r5.16xlarge": {Region: "ap-east-1", Type: "r5.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.344}, "r5.24xlarge": {Region: "ap-east-1", Type: "r5.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.016}, "r5.metal": {Region: "ap-east-1", Type: "r5.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.016}, "r5d.large": {Region: "ap-east-1", Type: "r5d.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.175}, "r5d.xlarge": {Region: "ap-east-1", Type: "r5d.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.35}, "r5d.2xlarge": {Region: "ap-east-1", Type: "r5d.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.7}, "r5d.4xlarge": {Region: "ap-east-1", Type: "r5d.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.4}, "r5d.8xlarge": {Region: "ap-east-1", Type: "r5d.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.8}, "r5d.12xlarge": {Region: "ap-east-1", Type: "r5d.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.2}, "r5d.16xlarge": {Region: "ap-east-1", Type: "r5d.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.6}, "r5d.24xlarge": {Region: "ap-east-1", Type: "r5d.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.4}, "r5d.metal": {Region: "ap-east-1", Type: "r5d.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.4}, "r5n.large": {Region: "ap-east-1", Type: "r5n.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.202}, "r5n.xlarge": {Region: "ap-east-1", Type: "r5n.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.404}, "r5n.2xlarge": {Region: "ap-east-1", Type: "r5n.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.808}, "r5n.4xlarge": {Region: "ap-east-1", Type: "r5n.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.616}, "r5n.8xlarge": {Region: "ap-east-1", Type: "r5n.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 3.232}, "r5n.12xlarge": {Region: "ap-east-1", Type: "r5n.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.848}, "r5n.16xlarge": {Region: "ap-east-1", Type: "r5n.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 6.464}, "r5n.24xlarge": {Region: "ap-east-1", Type: "r5n.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 9.696}, "r5n.metal": {Region: "ap-east-1", Type: "r5n.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 9.696}, "r6g.medium": {Region: "ap-east-1", Type: "r6g.medium", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.067}, "r6g.large": {Region: "ap-east-1", Type: "r6g.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.134}, "r6g.xlarge": {Region: "ap-east-1", Type: "r6g.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.268}, "r6g.2xlarge": {Region: "ap-east-1", Type: "r6g.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.536}, "r6g.4xlarge": {Region: "ap-east-1", Type: "r6g.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.072}, "r6g.8xlarge": {Region: "ap-east-1", Type: "r6g.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.144}, "r6g.12xlarge": {Region: "ap-east-1", Type: "r6g.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.216}, "r6g.16xlarge": {Region: "ap-east-1", Type: "r6g.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.288}, "r6g.metal": {Region: "ap-east-1", Type: "r6g.metal", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.288}, "t3.nano": {Region: "ap-east-1", Type: "t3.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0073}, "t3.micro": {Region: "ap-east-1", Type: "t3.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0146}, "t3.small": {Region: "ap-east-1", Type: "t3.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0292}, "t3.medium": {Region: "ap-east-1", Type: "t3.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0584}, "t3.large": {Region: "ap-east-1", Type: "t3.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.1168}, "t3.xlarge": {Region: "ap-east-1", Type: "t3.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.2336}, "t3.2xlarge": {Region: "ap-east-1", Type: "t3.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.4672}, "t4g.nano": {Region: "ap-east-1", Type: "t4g.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0058}, "t4g.micro": {Region: "ap-east-1", Type: "t4g.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0116}, "t4g.small": {Region: "ap-east-1", Type: "t4g.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0232}, "t4g.medium": {Region: "ap-east-1", Type: "t4g.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0464}, "t4g.large": {Region: "ap-east-1", Type: "t4g.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0928}, "t4g.xlarge": {Region: "ap-east-1", Type: "t4g.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1856}, "t4g.2xlarge": {Region: "ap-east-1", Type: "t4g.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3712}, "x1.16xlarge": {Region: "ap-east-1", Type: "x1.16xlarge", Memory: kresource.MustParse("999424Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 10.638}, "x1.32xlarge": {Region: "ap-east-1", Type: "x1.32xlarge", Memory: kresource.MustParse("1998848Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 21.276}, }, "ap-northeast-1": { "a1.medium": {Region: "ap-northeast-1", Type: "a1.medium", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0321}, "a1.large": {Region: "ap-northeast-1", Type: "a1.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0642}, "a1.xlarge": {Region: "ap-northeast-1", Type: "a1.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1284}, "a1.2xlarge": {Region: "ap-northeast-1", Type: "a1.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.2568}, "a1.4xlarge": {Region: "ap-northeast-1", Type: "a1.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.5136}, "a1.metal": {Region: "ap-northeast-1", Type: "a1.metal", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.514}, "c1.medium": {Region: "ap-northeast-1", Type: "c1.medium", Memory: kresource.MustParse("1740Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.158}, "c1.xlarge": {Region: "ap-northeast-1", Type: "c1.xlarge", Memory: kresource.MustParse("7168Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.632}, "c3.large": {Region: "ap-northeast-1", Type: "c3.large", Memory: kresource.MustParse("3840Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.128}, "c3.xlarge": {Region: "ap-northeast-1", Type: "c3.xlarge", Memory: kresource.MustParse("7680Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.255}, "c3.2xlarge": {Region: "ap-northeast-1", Type: "c3.2xlarge", Memory: kresource.MustParse("15360Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.511}, "c3.4xlarge": {Region: "ap-northeast-1", Type: "c3.4xlarge", Memory: kresource.MustParse("30720Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.021}, "c3.8xlarge": {Region: "ap-northeast-1", Type: "c3.8xlarge", Memory: kresource.MustParse("61440Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.043}, "c4.large": {Region: "ap-northeast-1", Type: "c4.large", Memory: kresource.MustParse("3840Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.126}, "c4.xlarge": {Region: "ap-northeast-1", Type: "c4.xlarge", Memory: kresource.MustParse("7680Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.252}, "c4.2xlarge": {Region: "ap-northeast-1", Type: "c4.2xlarge", Memory: kresource.MustParse("15360Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.504}, "c4.4xlarge": {Region: "ap-northeast-1", Type: "c4.4xlarge", Memory: kresource.MustParse("30720Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.008}, "c4.8xlarge": {Region: "ap-northeast-1", Type: "c4.8xlarge", Memory: kresource.MustParse("61440Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 2.016}, "c5.large": {Region: "ap-northeast-1", Type: "c5.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.107}, "c5.xlarge": {Region: "ap-northeast-1", Type: "c5.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.214}, "c5.2xlarge": {Region: "ap-northeast-1", Type: "c5.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.428}, "c5.4xlarge": {Region: "ap-northeast-1", Type: "c5.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.856}, "c5.9xlarge": {Region: "ap-northeast-1", Type: "c5.9xlarge", Memory: kresource.MustParse("73728Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 1.926}, "c5.12xlarge": {Region: "ap-northeast-1", Type: "c5.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.568}, "c5.18xlarge": {Region: "ap-northeast-1", Type: "c5.18xlarge", Memory: kresource.MustParse("147456Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 3.852}, "c5.24xlarge": {Region: "ap-northeast-1", Type: "c5.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.136}, "c5.metal": {Region: "ap-northeast-1", Type: "c5.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.136}, "c5a.large": {Region: "ap-northeast-1", Type: "c5a.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.096}, "c5a.xlarge": {Region: "ap-northeast-1", Type: "c5a.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.192}, "c5a.2xlarge": {Region: "ap-northeast-1", Type: "c5a.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.384}, "c5a.4xlarge": {Region: "ap-northeast-1", Type: "c5a.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.768}, "c5a.8xlarge": {Region: "ap-northeast-1", Type: "c5a.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.536}, "c5a.12xlarge": {Region: "ap-northeast-1", Type: "c5a.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.304}, "c5a.16xlarge": {Region: "ap-northeast-1", Type: "c5a.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.072}, "c5a.24xlarge": {Region: "ap-northeast-1", Type: "c5a.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.608}, "c5d.large": {Region: "ap-northeast-1", Type: "c5d.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.122}, "c5d.xlarge": {Region: "ap-northeast-1", Type: "c5d.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.244}, "c5d.2xlarge": {Region: "ap-northeast-1", Type: "c5d.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.488}, "c5d.4xlarge": {Region: "ap-northeast-1", Type: "c5d.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.976}, "c5d.9xlarge": {Region: "ap-northeast-1", Type: "c5d.9xlarge", Memory: kresource.MustParse("73728Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 2.196}, "c5d.12xlarge": {Region: "ap-northeast-1", Type: "c5d.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.928}, "c5d.18xlarge": {Region: "ap-northeast-1", Type: "c5d.18xlarge", Memory: kresource.MustParse("147456Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 4.392}, "c5d.24xlarge": {Region: "ap-northeast-1", Type: "c5d.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.856}, "c5d.metal": {Region: "ap-northeast-1", Type: "c5d.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.856}, "c5n.large": {Region: "ap-northeast-1", Type: "c5n.large", Memory: kresource.MustParse("5376Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.136}, "c5n.xlarge": {Region: "ap-northeast-1", Type: "c5n.xlarge", Memory: kresource.MustParse("10752Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.272}, "c5n.2xlarge": {Region: "ap-northeast-1", Type: "c5n.2xlarge", Memory: kresource.MustParse("21504Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.544}, "c5n.4xlarge": {Region: "ap-northeast-1", Type: "c5n.4xlarge", Memory: kresource.MustParse("43008Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.088}, "c5n.9xlarge": {Region: "ap-northeast-1", Type: "c5n.9xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 2.448}, "c5n.18xlarge": {Region: "ap-northeast-1", Type: "c5n.18xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 4.896}, "c5n.metal": {Region: "ap-northeast-1", Type: "c5n.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 4.896}, "c6g.medium": {Region: "ap-northeast-1", Type: "c6g.medium", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0428}, "c6g.large": {Region: "ap-northeast-1", Type: "c6g.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0856}, "c6g.xlarge": {Region: "ap-northeast-1", Type: "c6g.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1712}, "c6g.2xlarge": {Region: "ap-northeast-1", Type: "c6g.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3424}, "c6g.4xlarge": {Region: "ap-northeast-1", Type: "c6g.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.6848}, "c6g.8xlarge": {Region: "ap-northeast-1", Type: "c6g.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.3696}, "c6g.12xlarge": {Region: "ap-northeast-1", Type: "c6g.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.0544}, "c6g.16xlarge": {Region: "ap-northeast-1", Type: "c6g.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.7392}, "c6g.metal": {Region: "ap-northeast-1", Type: "c6g.metal", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.7392}, "c6gd.medium": {Region: "ap-northeast-1", Type: "c6gd.medium", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.049}, "c6gd.large": {Region: "ap-northeast-1", Type: "c6gd.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.098}, "c6gd.xlarge": {Region: "ap-northeast-1", Type: "c6gd.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.196}, "c6gd.2xlarge": {Region: "ap-northeast-1", Type: "c6gd.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.392}, "c6gd.4xlarge": {Region: "ap-northeast-1", Type: "c6gd.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.784}, "c6gd.8xlarge": {Region: "ap-northeast-1", Type: "c6gd.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.568}, "c6gd.12xlarge": {Region: "ap-northeast-1", Type: "c6gd.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.352}, "c6gd.16xlarge": {Region: "ap-northeast-1", Type: "c6gd.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.136}, "c6gd.metal": {Region: "ap-northeast-1", Type: "c6gd.metal", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.136}, "c6gn.medium": {Region: "ap-northeast-1", Type: "c6gn.medium", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0545}, "c6gn.large": {Region: "ap-northeast-1", Type: "c6gn.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.109}, "c6gn.xlarge": {Region: "ap-northeast-1", Type: "c6gn.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.218}, "c6gn.2xlarge": {Region: "ap-northeast-1", Type: "c6gn.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.436}, "c6gn.4xlarge": {Region: "ap-northeast-1", Type: "c6gn.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.872}, "c6gn.8xlarge": {Region: "ap-northeast-1", Type: "c6gn.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.744}, "c6gn.12xlarge": {Region: "ap-northeast-1", Type: "c6gn.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.616}, "c6gn.16xlarge": {Region: "ap-northeast-1", Type: "c6gn.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.488}, "c6gn.metal": {Region: "ap-northeast-1", Type: "c6gn.metal", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.488}, "c6i.large": {Region: "ap-northeast-1", Type: "c6i.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.107}, "c6i.xlarge": {Region: "ap-northeast-1", Type: "c6i.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.214}, "c6i.2xlarge": {Region: "ap-northeast-1", Type: "c6i.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.428}, "c6i.4xlarge": {Region: "ap-northeast-1", Type: "c6i.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.856}, "c6i.8xlarge": {Region: "ap-northeast-1", Type: "c6i.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.712}, "c6i.12xlarge": {Region: "ap-northeast-1", Type: "c6i.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.568}, "c6i.16xlarge": {Region: "ap-northeast-1", Type: "c6i.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.424}, "c6i.24xlarge": {Region: "ap-northeast-1", Type: "c6i.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.136}, "c6i.32xlarge": {Region: "ap-northeast-1", Type: "c6i.32xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 6.848}, "c6i.metal": {Region: "ap-northeast-1", Type: "c6i.metal", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 6.848}, "cc2.8xlarge": {Region: "ap-northeast-1", Type: "cc2.8xlarge", Memory: kresource.MustParse("61952Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.349}, "cr1.8xlarge": {Region: "ap-northeast-1", Type: "cr1.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 4.105}, "d2.xlarge": {Region: "ap-northeast-1", Type: "d2.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.844}, "d2.2xlarge": {Region: "ap-northeast-1", Type: "d2.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.688}, "d2.4xlarge": {Region: "ap-northeast-1", Type: "d2.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 3.376}, "d2.8xlarge": {Region: "ap-northeast-1", Type: "d2.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 6.752}, "d3.xlarge": {Region: "ap-northeast-1", Type: "d3.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.724}, "d3.2xlarge": {Region: "ap-northeast-1", Type: "d3.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.448}, "d3.4xlarge": {Region: "ap-northeast-1", Type: "d3.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 2.897}, "d3.8xlarge": {Region: "ap-northeast-1", Type: "d3.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 5.79344}, "g2.2xlarge": {Region: "ap-northeast-1", Type: "g2.2xlarge", Memory: kresource.MustParse("15360Mi"), CPU: kresource.MustParse("8"), GPU: 1, Inf: 0, Price: 0.898}, "g2.8xlarge": {Region: "ap-northeast-1", Type: "g2.8xlarge", Memory: kresource.MustParse("61440Mi"), CPU: kresource.MustParse("32"), GPU: 4, Inf: 0, Price: 3.592}, "g3.4xlarge": {Region: "ap-northeast-1", Type: "g3.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 1, Inf: 0, Price: 1.58}, "g3.8xlarge": {Region: "ap-northeast-1", Type: "g3.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 2, Inf: 0, Price: 3.16}, "g3.16xlarge": {Region: "ap-northeast-1", Type: "g3.16xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("64"), GPU: 4, Inf: 0, Price: 6.32}, "g3s.xlarge": {Region: "ap-northeast-1", Type: "g3s.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 1, Inf: 0, Price: 1.04}, "g4ad.xlarge": {Region: "ap-northeast-1", Type: "g4ad.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 1, Inf: 0, Price: 0.51082}, "g4ad.2xlarge": {Region: "ap-northeast-1", Type: "g4ad.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 1, Inf: 0, Price: 0.7303}, "g4ad.4xlarge": {Region: "ap-northeast-1", Type: "g4ad.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 1, Inf: 0, Price: 1.17}, "g4ad.8xlarge": {Region: "ap-northeast-1", Type: "g4ad.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 2, Inf: 0, Price: 2.34}, "g4ad.16xlarge": {Region: "ap-northeast-1", Type: "g4ad.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 4, Inf: 0, Price: 4.68}, "g4dn.xlarge": {Region: "ap-northeast-1", Type: "g4dn.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 1, Inf: 0, Price: 0.71}, "g4dn.2xlarge": {Region: "ap-northeast-1", Type: "g4dn.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 1, Inf: 0, Price: 1.015}, "g4dn.4xlarge": {Region: "ap-northeast-1", Type: "g4dn.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 1, Inf: 0, Price: 1.625}, "g4dn.8xlarge": {Region: "ap-northeast-1", Type: "g4dn.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 1, Inf: 0, Price: 2.938}, "g4dn.12xlarge": {Region: "ap-northeast-1", Type: "g4dn.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 4, Inf: 0, Price: 5.281}, "g4dn.16xlarge": {Region: "ap-northeast-1", Type: "g4dn.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 1, Inf: 0, Price: 5.875}, "g4dn.metal": {Region: "ap-northeast-1", Type: "g4dn.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 8, Inf: 0, Price: 10.562}, "g5g.xlarge": {Region: "ap-northeast-1", Type: "g5g.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 1, Inf: 0, Price: 0.5669}, "g5g.2xlarge": {Region: "ap-northeast-1", Type: "g5g.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 1, Inf: 0, Price: 0.7505}, "g5g.4xlarge": {Region: "ap-northeast-1", Type: "g5g.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 1, Inf: 0, Price: 1.1176}, "g5g.8xlarge": {Region: "ap-northeast-1", Type: "g5g.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 1, Inf: 0, Price: 1.8519}, "g5g.16xlarge": {Region: "ap-northeast-1", Type: "g5g.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 2, Inf: 0, Price: 3.7039}, "g5g.metal": {Region: "ap-northeast-1", Type: "g5g.metal", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 2, Inf: 0, Price: 3.7039}, "hs1.8xlarge": {Region: "ap-northeast-1", Type: "hs1.8xlarge", Memory: kresource.MustParse("119808Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 5.4}, "i2.xlarge": {Region: "ap-northeast-1", Type: "i2.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 1.001}, "i2.2xlarge": {Region: "ap-northeast-1", Type: "i2.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 2.001}, "i2.4xlarge": {Region: "ap-northeast-1", Type: "i2.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 4.002}, "i2.8xlarge": {Region: "ap-northeast-1", Type: "i2.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 8.004}, "i3.large": {Region: "ap-northeast-1", Type: "i3.large", Memory: kresource.MustParse("15616Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.183}, "i3.xlarge": {Region: "ap-northeast-1", Type: "i3.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.366}, "i3.2xlarge": {Region: "ap-northeast-1", Type: "i3.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.732}, "i3.4xlarge": {Region: "ap-northeast-1", Type: "i3.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.464}, "i3.8xlarge": {Region: "ap-northeast-1", Type: "i3.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.928}, "i3.16xlarge": {Region: "ap-northeast-1", Type: "i3.16xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.856}, "i3.metal": {Region: "ap-northeast-1", Type: "i3.metal", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.856}, "i3en.large": {Region: "ap-northeast-1", Type: "i3en.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.266}, "i3en.xlarge": {Region: "ap-northeast-1", Type: "i3en.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.532}, "i3en.2xlarge": {Region: "ap-northeast-1", Type: "i3en.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.064}, "i3en.3xlarge": {Region: "ap-northeast-1", Type: "i3en.3xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("12"), GPU: 0, Inf: 0, Price: 1.596}, "i3en.6xlarge": {Region: "ap-northeast-1", Type: "i3en.6xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 0, Price: 3.192}, "i3en.12xlarge": {Region: "ap-northeast-1", Type: "i3en.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 6.384}, "i3en.24xlarge": {Region: "ap-northeast-1", Type: "i3en.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 12.768}, "i3en.metal": {Region: "ap-northeast-1", Type: "i3en.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 12.768}, "inf1.xlarge": {Region: "ap-northeast-1", Type: "inf1.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 1, Price: 0.308}, "inf1.2xlarge": {Region: "ap-northeast-1", Type: "inf1.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 1, Price: 0.489}, "inf1.6xlarge": {Region: "ap-northeast-1", Type: "inf1.6xlarge", Memory: kresource.MustParse("49152Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 4, Price: 1.594}, "inf1.24xlarge": {Region: "ap-northeast-1", Type: "inf1.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 16, Price: 6.376}, "m1.small": {Region: "ap-northeast-1", Type: "m1.small", Memory: kresource.MustParse("1740Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.061}, "m1.medium": {Region: "ap-northeast-1", Type: "m1.medium", Memory: kresource.MustParse("3840Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.122}, "m1.large": {Region: "ap-northeast-1", Type: "m1.large", Memory: kresource.MustParse("7680Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.243}, "m1.xlarge": {Region: "ap-northeast-1", Type: "m1.xlarge", Memory: kresource.MustParse("15360Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.486}, "m2.xlarge": {Region: "ap-northeast-1", Type: "m2.xlarge", Memory: kresource.MustParse("17510Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.287}, "m2.2xlarge": {Region: "ap-northeast-1", Type: "m2.2xlarge", Memory: kresource.MustParse("35020Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.575}, "m2.4xlarge": {Region: "ap-northeast-1", Type: "m2.4xlarge", Memory: kresource.MustParse("70041Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.15}, "m3.medium": {Region: "ap-northeast-1", Type: "m3.medium", Memory: kresource.MustParse("3840Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.096}, "m3.large": {Region: "ap-northeast-1", Type: "m3.large", Memory: kresource.MustParse("7680Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.193}, "m3.xlarge": {Region: "ap-northeast-1", Type: "m3.xlarge", Memory: kresource.MustParse("15360Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.385}, "m3.2xlarge": {Region: "ap-northeast-1", Type: "m3.2xlarge", Memory: kresource.MustParse("30720Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.77}, "m4.large": {Region: "ap-northeast-1", Type: "m4.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.129}, "m4.xlarge": {Region: "ap-northeast-1", Type: "m4.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.258}, "m4.2xlarge": {Region: "ap-northeast-1", Type: "m4.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.516}, "m4.4xlarge": {Region: "ap-northeast-1", Type: "m4.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.032}, "m4.10xlarge": {Region: "ap-northeast-1", Type: "m4.10xlarge", Memory: kresource.MustParse("163840Mi"), CPU: kresource.MustParse("40"), GPU: 0, Inf: 0, Price: 2.58}, "m4.16xlarge": {Region: "ap-northeast-1", Type: "m4.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.128}, "m5.large": {Region: "ap-northeast-1", Type: "m5.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.124}, "m5.xlarge": {Region: "ap-northeast-1", Type: "m5.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.248}, "m5.2xlarge": {Region: "ap-northeast-1", Type: "m5.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.496}, "m5.4xlarge": {Region: "ap-northeast-1", Type: "m5.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.992}, "m5.8xlarge": {Region: "ap-northeast-1", Type: "m5.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.984}, "m5.12xlarge": {Region: "ap-northeast-1", Type: "m5.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.976}, "m5.16xlarge": {Region: "ap-northeast-1", Type: "m5.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.968}, "m5.24xlarge": {Region: "ap-northeast-1", Type: "m5.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.952}, "m5.metal": {Region: "ap-northeast-1", Type: "m5.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.952}, "m5a.large": {Region: "ap-northeast-1", Type: "m5a.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.112}, "m5a.xlarge": {Region: "ap-northeast-1", Type: "m5a.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.224}, "m5a.2xlarge": {Region: "ap-northeast-1", Type: "m5a.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.448}, "m5a.4xlarge": {Region: "ap-northeast-1", Type: "m5a.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.896}, "m5a.8xlarge": {Region: "ap-northeast-1", Type: "m5a.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.792}, "m5a.12xlarge": {Region: "ap-northeast-1", Type: "m5a.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.688}, "m5a.16xlarge": {Region: "ap-northeast-1", Type: "m5a.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.584}, "m5a.24xlarge": {Region: "ap-northeast-1", Type: "m5a.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.376}, "m5ad.large": {Region: "ap-northeast-1", Type: "m5ad.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.134}, "m5ad.xlarge": {Region: "ap-northeast-1", Type: "m5ad.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.268}, "m5ad.2xlarge": {Region: "ap-northeast-1", Type: "m5ad.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.536}, "m5ad.4xlarge": {Region: "ap-northeast-1", Type: "m5ad.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.072}, "m5ad.8xlarge": {Region: "ap-northeast-1", Type: "m5ad.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.144}, "m5ad.12xlarge": {Region: "ap-northeast-1", Type: "m5ad.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.216}, "m5ad.16xlarge": {Region: "ap-northeast-1", Type: "m5ad.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.288}, "m5ad.24xlarge": {Region: "ap-northeast-1", Type: "m5ad.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.432}, "m5d.large": {Region: "ap-northeast-1", Type: "m5d.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.146}, "m5d.xlarge": {Region: "ap-northeast-1", Type: "m5d.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.292}, "m5d.2xlarge": {Region: "ap-northeast-1", Type: "m5d.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.584}, "m5d.4xlarge": {Region: "ap-northeast-1", Type: "m5d.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.168}, "m5d.8xlarge": {Region: "ap-northeast-1", Type: "m5d.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.336}, "m5d.12xlarge": {Region: "ap-northeast-1", Type: "m5d.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.504}, "m5d.16xlarge": {Region: "ap-northeast-1", Type: "m5d.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.672}, "m5d.24xlarge": {Region: "ap-northeast-1", Type: "m5d.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.008}, "m5d.metal": {Region: "ap-northeast-1", Type: "m5d.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.008}, "m5dn.large": {Region: "ap-northeast-1", Type: "m5dn.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.175}, "m5dn.xlarge": {Region: "ap-northeast-1", Type: "m5dn.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.35}, "m5dn.2xlarge": {Region: "ap-northeast-1", Type: "m5dn.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.7}, "m5dn.4xlarge": {Region: "ap-northeast-1", Type: "m5dn.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.4}, "m5dn.8xlarge": {Region: "ap-northeast-1", Type: "m5dn.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.8}, "m5dn.12xlarge": {Region: "ap-northeast-1", Type: "m5dn.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.2}, "m5dn.16xlarge": {Region: "ap-northeast-1", Type: "m5dn.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.6}, "m5dn.24xlarge": {Region: "ap-northeast-1", Type: "m5dn.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.4}, "m5dn.metal": {Region: "ap-northeast-1", Type: "m5dn.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.4}, "m5n.large": {Region: "ap-northeast-1", Type: "m5n.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.153}, "m5n.xlarge": {Region: "ap-northeast-1", Type: "m5n.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.306}, "m5n.2xlarge": {Region: "ap-northeast-1", Type: "m5n.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.612}, "m5n.4xlarge": {Region: "ap-northeast-1", Type: "m5n.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.224}, "m5n.8xlarge": {Region: "ap-northeast-1", Type: "m5n.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.448}, "m5n.12xlarge": {Region: "ap-northeast-1", Type: "m5n.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.672}, "m5n.16xlarge": {Region: "ap-northeast-1", Type: "m5n.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.896}, "m5n.24xlarge": {Region: "ap-northeast-1", Type: "m5n.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.344}, "m5n.metal": {Region: "ap-northeast-1", Type: "m5n.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.344}, "m5zn.large": {Region: "ap-northeast-1", Type: "m5zn.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.2134}, "m5zn.xlarge": {Region: "ap-northeast-1", Type: "m5zn.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.4267}, "m5zn.2xlarge": {Region: "ap-northeast-1", Type: "m5zn.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.8534}, "m5zn.3xlarge": {Region: "ap-northeast-1", Type: "m5zn.3xlarge", Memory: kresource.MustParse("49152Mi"), CPU: kresource.MustParse("12"), GPU: 0, Inf: 0, Price: 1.2801}, "m5zn.6xlarge": {Region: "ap-northeast-1", Type: "m5zn.6xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 0, Price: 2.5602}, "m5zn.12xlarge": {Region: "ap-northeast-1", Type: "m5zn.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 5.1204}, "m5zn.metal": {Region: "ap-northeast-1", Type: "m5zn.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 5.1204}, "m6g.medium": {Region: "ap-northeast-1", Type: "m6g.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0495}, "m6g.large": {Region: "ap-northeast-1", Type: "m6g.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.099}, "m6g.xlarge": {Region: "ap-northeast-1", Type: "m6g.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.198}, "m6g.2xlarge": {Region: "ap-northeast-1", Type: "m6g.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.396}, "m6g.4xlarge": {Region: "ap-northeast-1", Type: "m6g.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.792}, "m6g.8xlarge": {Region: "ap-northeast-1", Type: "m6g.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.584}, "m6g.12xlarge": {Region: "ap-northeast-1", Type: "m6g.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.376}, "m6g.16xlarge": {Region: "ap-northeast-1", Type: "m6g.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.168}, "m6g.metal": {Region: "ap-northeast-1", Type: "m6g.metal", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.168}, "m6gd.medium": {Region: "ap-northeast-1", Type: "m6gd.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0585}, "m6gd.large": {Region: "ap-northeast-1", Type: "m6gd.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.117}, "m6gd.xlarge": {Region: "ap-northeast-1", Type: "m6gd.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.234}, "m6gd.2xlarge": {Region: "ap-northeast-1", Type: "m6gd.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.468}, "m6gd.4xlarge": {Region: "ap-northeast-1", Type: "m6gd.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.936}, "m6gd.8xlarge": {Region: "ap-northeast-1", Type: "m6gd.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.872}, "m6gd.12xlarge": {Region: "ap-northeast-1", Type: "m6gd.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.808}, "m6gd.16xlarge": {Region: "ap-northeast-1", Type: "m6gd.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.744}, "m6gd.metal": {Region: "ap-northeast-1", Type: "m6gd.metal", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.744}, "m6i.large": {Region: "ap-northeast-1", Type: "m6i.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.124}, "m6i.xlarge": {Region: "ap-northeast-1", Type: "m6i.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.248}, "m6i.2xlarge": {Region: "ap-northeast-1", Type: "m6i.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.496}, "m6i.4xlarge": {Region: "ap-northeast-1", Type: "m6i.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.992}, "m6i.8xlarge": {Region: "ap-northeast-1", Type: "m6i.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.984}, "m6i.12xlarge": {Region: "ap-northeast-1", Type: "m6i.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.976}, "m6i.16xlarge": {Region: "ap-northeast-1", Type: "m6i.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.968}, "m6i.24xlarge": {Region: "ap-northeast-1", Type: "m6i.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.952}, "m6i.32xlarge": {Region: "ap-northeast-1", Type: "m6i.32xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 7.936}, "m6i.metal": {Region: "ap-northeast-1", Type: "m6i.metal", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 7.936}, "p2.xlarge": {Region: "ap-northeast-1", Type: "p2.xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("4"), GPU: 1, Inf: 0, Price: 1.542}, "p2.8xlarge": {Region: "ap-northeast-1", Type: "p2.8xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("32"), GPU: 8, Inf: 0, Price: 12.336}, "p2.16xlarge": {Region: "ap-northeast-1", Type: "p2.16xlarge", Memory: kresource.MustParse("749568Mi"), CPU: kresource.MustParse("64"), GPU: 16, Inf: 0, Price: 24.672}, "p3.2xlarge": {Region: "ap-northeast-1", Type: "p3.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 1, Inf: 0, Price: 4.194}, "p3.8xlarge": {Region: "ap-northeast-1", Type: "p3.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 4, Inf: 0, Price: 16.776}, "p3.16xlarge": {Region: "ap-northeast-1", Type: "p3.16xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("64"), GPU: 8, Inf: 0, Price: 33.552}, "p3dn.24xlarge": {Region: "ap-northeast-1", Type: "p3dn.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 8, Inf: 0, Price: 42.783}, "p4d.24xlarge": {Region: "ap-northeast-1", Type: "p4d.24xlarge", Memory: kresource.MustParse("1179648Mi"), CPU: kresource.MustParse("96"), GPU: 8, Inf: 0, Price: 44.92215}, "r3.large": {Region: "ap-northeast-1", Type: "r3.large", Memory: kresource.MustParse("15616Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.2}, "r3.xlarge": {Region: "ap-northeast-1", Type: "r3.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.399}, "r3.2xlarge": {Region: "ap-northeast-1", Type: "r3.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.798}, "r3.4xlarge": {Region: "ap-northeast-1", Type: "r3.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.596}, "r3.8xlarge": {Region: "ap-northeast-1", Type: "r3.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 3.192}, "r4.large": {Region: "ap-northeast-1", Type: "r4.large", Memory: kresource.MustParse("15616Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.16}, "r4.xlarge": {Region: "ap-northeast-1", Type: "r4.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.32}, "r4.2xlarge": {Region: "ap-northeast-1", Type: "r4.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.64}, "r4.4xlarge": {Region: "ap-northeast-1", Type: "r4.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.28}, "r4.8xlarge": {Region: "ap-northeast-1", Type: "r4.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.56}, "r4.16xlarge": {Region: "ap-northeast-1", Type: "r4.16xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.12}, "r5.large": {Region: "ap-northeast-1", Type: "r5.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.152}, "r5.xlarge": {Region: "ap-northeast-1", Type: "r5.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.304}, "r5.2xlarge": {Region: "ap-northeast-1", Type: "r5.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.608}, "r5.4xlarge": {Region: "ap-northeast-1", Type: "r5.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.216}, "r5.8xlarge": {Region: "ap-northeast-1", Type: "r5.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.432}, "r5.12xlarge": {Region: "ap-northeast-1", Type: "r5.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.648}, "r5.16xlarge": {Region: "ap-northeast-1", Type: "r5.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.864}, "r5.24xlarge": {Region: "ap-northeast-1", Type: "r5.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.296}, "r5.metal": {Region: "ap-northeast-1", Type: "r5.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.296}, "r5a.large": {Region: "ap-northeast-1", Type: "r5a.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.137}, "r5a.xlarge": {Region: "ap-northeast-1", Type: "r5a.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.274}, "r5a.2xlarge": {Region: "ap-northeast-1", Type: "r5a.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.548}, "r5a.4xlarge": {Region: "ap-northeast-1", Type: "r5a.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.096}, "r5a.8xlarge": {Region: "ap-northeast-1", Type: "r5a.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.192}, "r5a.12xlarge": {Region: "ap-northeast-1", Type: "r5a.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.288}, "r5a.16xlarge": {Region: "ap-northeast-1", Type: "r5a.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.384}, "r5a.24xlarge": {Region: "ap-northeast-1", Type: "r5a.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.576}, "r5ad.large": {Region: "ap-northeast-1", Type: "r5ad.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.159}, "r5ad.xlarge": {Region: "ap-northeast-1", Type: "r5ad.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.318}, "r5ad.2xlarge": {Region: "ap-northeast-1", Type: "r5ad.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.636}, "r5ad.4xlarge": {Region: "ap-northeast-1", Type: "r5ad.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.272}, "r5ad.8xlarge": {Region: "ap-northeast-1", Type: "r5ad.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.544}, "r5ad.12xlarge": {Region: "ap-northeast-1", Type: "r5ad.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.816}, "r5ad.16xlarge": {Region: "ap-northeast-1", Type: "r5ad.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.088}, "r5ad.24xlarge": {Region: "ap-northeast-1", Type: "r5ad.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.632}, "r5b.large": {Region: "ap-northeast-1", Type: "r5b.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.181}, "r5b.xlarge": {Region: "ap-northeast-1", Type: "r5b.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.362}, "r5b.2xlarge": {Region: "ap-northeast-1", Type: "r5b.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.724}, "r5b.4xlarge": {Region: "ap-northeast-1", Type: "r5b.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.448}, "r5b.8xlarge": {Region: "ap-northeast-1", Type: "r5b.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.896}, "r5b.12xlarge": {Region: "ap-northeast-1", Type: "r5b.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.344}, "r5b.16xlarge": {Region: "ap-northeast-1", Type: "r5b.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.792}, "r5b.24xlarge": {Region: "ap-northeast-1", Type: "r5b.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.688}, "r5b.metal": {Region: "ap-northeast-1", Type: "r5b.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.688}, "r5d.large": {Region: "ap-northeast-1", Type: "r5d.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.174}, "r5d.xlarge": {Region: "ap-northeast-1", Type: "r5d.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.348}, "r5d.2xlarge": {Region: "ap-northeast-1", Type: "r5d.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.696}, "r5d.4xlarge": {Region: "ap-northeast-1", Type: "r5d.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.392}, "r5d.8xlarge": {Region: "ap-northeast-1", Type: "r5d.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.784}, "r5d.12xlarge": {Region: "ap-northeast-1", Type: "r5d.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.176}, "r5d.16xlarge": {Region: "ap-northeast-1", Type: "r5d.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.568}, "r5d.24xlarge": {Region: "ap-northeast-1", Type: "r5d.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.352}, "r5d.metal": {Region: "ap-northeast-1", Type: "r5d.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.352}, "r5dn.large": {Region: "ap-northeast-1", Type: "r5dn.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.203}, "r5dn.xlarge": {Region: "ap-northeast-1", Type: "r5dn.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.406}, "r5dn.2xlarge": {Region: "ap-northeast-1", Type: "r5dn.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.812}, "r5dn.4xlarge": {Region: "ap-northeast-1", Type: "r5dn.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.624}, "r5dn.8xlarge": {Region: "ap-northeast-1", Type: "r5dn.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 3.248}, "r5dn.12xlarge": {Region: "ap-northeast-1", Type: "r5dn.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.872}, "r5dn.16xlarge": {Region: "ap-northeast-1", Type: "r5dn.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 6.496}, "r5dn.24xlarge": {Region: "ap-northeast-1", Type: "r5dn.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 9.744}, "r5dn.metal": {Region: "ap-northeast-1", Type: "r5dn.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 9.744}, "r5n.large": {Region: "ap-northeast-1", Type: "r5n.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.181}, "r5n.xlarge": {Region: "ap-northeast-1", Type: "r5n.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.362}, "r5n.2xlarge": {Region: "ap-northeast-1", Type: "r5n.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.724}, "r5n.4xlarge": {Region: "ap-northeast-1", Type: "r5n.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.448}, "r5n.8xlarge": {Region: "ap-northeast-1", Type: "r5n.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.896}, "r5n.12xlarge": {Region: "ap-northeast-1", Type: "r5n.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.344}, "r5n.16xlarge": {Region: "ap-northeast-1", Type: "r5n.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.792}, "r5n.24xlarge": {Region: "ap-northeast-1", Type: "r5n.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.688}, "r5n.metal": {Region: "ap-northeast-1", Type: "r5n.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.688}, "r6g.medium": {Region: "ap-northeast-1", Type: "r6g.medium", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0608}, "r6g.large": {Region: "ap-northeast-1", Type: "r6g.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.1216}, "r6g.xlarge": {Region: "ap-northeast-1", Type: "r6g.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.2432}, "r6g.2xlarge": {Region: "ap-northeast-1", Type: "r6g.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.4864}, "r6g.4xlarge": {Region: "ap-northeast-1", Type: "r6g.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.9728}, "r6g.8xlarge": {Region: "ap-northeast-1", Type: "r6g.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.9456}, "r6g.12xlarge": {Region: "ap-northeast-1", Type: "r6g.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.9184}, "r6g.16xlarge": {Region: "ap-northeast-1", Type: "r6g.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.8912}, "r6g.metal": {Region: "ap-northeast-1", Type: "r6g.metal", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.8912}, "r6gd.medium": {Region: "ap-northeast-1", Type: "r6gd.medium", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0695}, "r6gd.large": {Region: "ap-northeast-1", Type: "r6gd.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.139}, "r6gd.xlarge": {Region: "ap-northeast-1", Type: "r6gd.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.278}, "r6gd.2xlarge": {Region: "ap-northeast-1", Type: "r6gd.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.556}, "r6gd.4xlarge": {Region: "ap-northeast-1", Type: "r6gd.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.112}, "r6gd.8xlarge": {Region: "ap-northeast-1", Type: "r6gd.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.224}, "r6gd.12xlarge": {Region: "ap-northeast-1", Type: "r6gd.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.336}, "r6gd.16xlarge": {Region: "ap-northeast-1", Type: "r6gd.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.448}, "r6gd.metal": {Region: "ap-northeast-1", Type: "r6gd.metal", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.448}, "r6i.large": {Region: "ap-northeast-1", Type: "r6i.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.152}, "r6i.xlarge": {Region: "ap-northeast-1", Type: "r6i.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.304}, "r6i.2xlarge": {Region: "ap-northeast-1", Type: "r6i.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.608}, "r6i.4xlarge": {Region: "ap-northeast-1", Type: "r6i.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.216}, "r6i.8xlarge": {Region: "ap-northeast-1", Type: "r6i.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.432}, "r6i.12xlarge": {Region: "ap-northeast-1", Type: "r6i.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.648}, "r6i.16xlarge": {Region: "ap-northeast-1", Type: "r6i.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.864}, "r6i.24xlarge": {Region: "ap-northeast-1", Type: "r6i.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.296}, "r6i.32xlarge": {Region: "ap-northeast-1", Type: "r6i.32xlarge", Memory: kresource.MustParse("1048576Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 9.728}, "r6i.metal": {Region: "ap-northeast-1", Type: "r6i.metal", Memory: kresource.MustParse("1048576Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 9.728}, "t1.micro": {Region: "ap-northeast-1", Type: "t1.micro", Memory: kresource.MustParse("627Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.026}, "t2.nano": {Region: "ap-northeast-1", Type: "t2.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0076}, "t2.micro": {Region: "ap-northeast-1", Type: "t2.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0152}, "t2.small": {Region: "ap-northeast-1", Type: "t2.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0304}, "t2.medium": {Region: "ap-northeast-1", Type: "t2.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0608}, "t2.large": {Region: "ap-northeast-1", Type: "t2.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.1216}, "t2.xlarge": {Region: "ap-northeast-1", Type: "t2.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.2432}, "t2.2xlarge": {Region: "ap-northeast-1", Type: "t2.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.4864}, "t3.nano": {Region: "ap-northeast-1", Type: "t3.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0068}, "t3.micro": {Region: "ap-northeast-1", Type: "t3.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0136}, "t3.small": {Region: "ap-northeast-1", Type: "t3.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0272}, "t3.medium": {Region: "ap-northeast-1", Type: "t3.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0544}, "t3.large": {Region: "ap-northeast-1", Type: "t3.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.1088}, "t3.xlarge": {Region: "ap-northeast-1", Type: "t3.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.2176}, "t3.2xlarge": {Region: "ap-northeast-1", Type: "t3.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.4352}, "t3a.nano": {Region: "ap-northeast-1", Type: "t3a.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0061}, "t3a.micro": {Region: "ap-northeast-1", Type: "t3a.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0122}, "t3a.small": {Region: "ap-northeast-1", Type: "t3a.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0245}, "t3a.medium": {Region: "ap-northeast-1", Type: "t3a.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.049}, "t3a.large": {Region: "ap-northeast-1", Type: "t3a.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0979}, "t3a.xlarge": {Region: "ap-northeast-1", Type: "t3a.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1958}, "t3a.2xlarge": {Region: "ap-northeast-1", Type: "t3a.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3917}, "t4g.nano": {Region: "ap-northeast-1", Type: "t4g.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0054}, "t4g.micro": {Region: "ap-northeast-1", Type: "t4g.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0108}, "t4g.small": {Region: "ap-northeast-1", Type: "t4g.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0216}, "t4g.medium": {Region: "ap-northeast-1", Type: "t4g.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0432}, "t4g.large": {Region: "ap-northeast-1", Type: "t4g.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0864}, "t4g.xlarge": {Region: "ap-northeast-1", Type: "t4g.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1728}, "t4g.2xlarge": {Region: "ap-northeast-1", Type: "t4g.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3456}, "vt1.3xlarge": {Region: "ap-northeast-1", Type: "vt1.3xlarge", Memory: kresource.MustParse("24576Mi"), CPU: kresource.MustParse("12"), GPU: 0, Inf: 0, Price: 0.81824}, "vt1.6xlarge": {Region: "ap-northeast-1", Type: "vt1.6xlarge", Memory: kresource.MustParse("49152Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 0, Price: 1.63647}, "vt1.24xlarge": {Region: "ap-northeast-1", Type: "vt1.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.54588}, "x1.16xlarge": {Region: "ap-northeast-1", Type: "x1.16xlarge", Memory: kresource.MustParse("999424Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 9.671}, "x1.32xlarge": {Region: "ap-northeast-1", Type: "x1.32xlarge", Memory: kresource.MustParse("1998848Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 19.341}, "x1e.xlarge": {Region: "ap-northeast-1", Type: "x1e.xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 1.209}, "x1e.2xlarge": {Region: "ap-northeast-1", Type: "x1e.2xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 2.418}, "x1e.4xlarge": {Region: "ap-northeast-1", Type: "x1e.4xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 4.836}, "x1e.8xlarge": {Region: "ap-northeast-1", Type: "x1e.8xlarge", Memory: kresource.MustParse("999424Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 9.672}, "x1e.16xlarge": {Region: "ap-northeast-1", Type: "x1e.16xlarge", Memory: kresource.MustParse("1998848Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 19.344}, "x1e.32xlarge": {Region: "ap-northeast-1", Type: "x1e.32xlarge", Memory: kresource.MustParse("3997696Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 38.688}, "x2idn.16xlarge": {Region: "ap-northeast-1", Type: "x2idn.16xlarge", Memory: kresource.MustParse("1048576Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 9.6705}, "x2idn.24xlarge": {Region: "ap-northeast-1", Type: "x2idn.24xlarge", Memory: kresource.MustParse("1572864Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 14.50575}, "x2idn.32xlarge": {Region: "ap-northeast-1", Type: "x2idn.32xlarge", Memory: kresource.MustParse("2097152Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 19.341}, "x2iedn.xlarge": {Region: "ap-northeast-1", Type: "x2iedn.xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 1.20881}, "x2iedn.2xlarge": {Region: "ap-northeast-1", Type: "x2iedn.2xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 2.41763}, "x2iedn.4xlarge": {Region: "ap-northeast-1", Type: "x2iedn.4xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 4.83525}, "x2iedn.8xlarge": {Region: "ap-northeast-1", Type: "x2iedn.8xlarge", Memory: kresource.MustParse("1048576Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 9.6705}, "x2iedn.16xlarge": {Region: "ap-northeast-1", Type: "x2iedn.16xlarge", Memory: kresource.MustParse("2097152Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 19.341}, "x2iedn.24xlarge": {Region: "ap-northeast-1", Type: "x2iedn.24xlarge", Memory: kresource.MustParse("3145728Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 29.0115}, "x2iedn.32xlarge": {Region: "ap-northeast-1", Type: "x2iedn.32xlarge", Memory: kresource.MustParse("4194304Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 38.682}, "x2iezn.2xlarge": {Region: "ap-northeast-1", Type: "x2iezn.2xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 2.418}, "x2iezn.4xlarge": {Region: "ap-northeast-1", Type: "x2iezn.4xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 4.836}, "x2iezn.6xlarge": {Region: "ap-northeast-1", Type: "x2iezn.6xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 0, Price: 7.254}, "x2iezn.8xlarge": {Region: "ap-northeast-1", Type: "x2iezn.8xlarge", Memory: kresource.MustParse("1048576Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 9.672}, "x2iezn.12xlarge": {Region: "ap-northeast-1", Type: "x2iezn.12xlarge", Memory: kresource.MustParse("1572864Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 14.508}, "x2iezn.metal": {Region: "ap-northeast-1", Type: "x2iezn.metal", Memory: kresource.MustParse("1572864Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 14.508}, "z1d.large": {Region: "ap-northeast-1", Type: "z1d.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.227}, "z1d.xlarge": {Region: "ap-northeast-1", Type: "z1d.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.454}, "z1d.2xlarge": {Region: "ap-northeast-1", Type: "z1d.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.908}, "z1d.3xlarge": {Region: "ap-northeast-1", Type: "z1d.3xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("12"), GPU: 0, Inf: 0, Price: 1.362}, "z1d.6xlarge": {Region: "ap-northeast-1", Type: "z1d.6xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 0, Price: 2.724}, "z1d.12xlarge": {Region: "ap-northeast-1", Type: "z1d.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 5.448}, "z1d.metal": {Region: "ap-northeast-1", Type: "z1d.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 5.448}, }, "ap-northeast-2": { "c3.large": {Region: "ap-northeast-2", Type: "c3.large", Memory: kresource.MustParse("3840Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.115}, "c3.xlarge": {Region: "ap-northeast-2", Type: "c3.xlarge", Memory: kresource.MustParse("7680Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.23}, "c3.2xlarge": {Region: "ap-northeast-2", Type: "c3.2xlarge", Memory: kresource.MustParse("15360Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.46}, "c3.4xlarge": {Region: "ap-northeast-2", Type: "c3.4xlarge", Memory: kresource.MustParse("30720Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.919}, "c3.8xlarge": {Region: "ap-northeast-2", Type: "c3.8xlarge", Memory: kresource.MustParse("61440Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.839}, "c4.large": {Region: "ap-northeast-2", Type: "c4.large", Memory: kresource.MustParse("3840Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.114}, "c4.xlarge": {Region: "ap-northeast-2", Type: "c4.xlarge", Memory: kresource.MustParse("7680Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.227}, "c4.2xlarge": {Region: "ap-northeast-2", Type: "c4.2xlarge", Memory: kresource.MustParse("15360Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.454}, "c4.4xlarge": {Region: "ap-northeast-2", Type: "c4.4xlarge", Memory: kresource.MustParse("30720Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.907}, "c4.8xlarge": {Region: "ap-northeast-2", Type: "c4.8xlarge", Memory: kresource.MustParse("61440Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 1.815}, "c5.large": {Region: "ap-northeast-2", Type: "c5.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.096}, "c5.xlarge": {Region: "ap-northeast-2", Type: "c5.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.192}, "c5.2xlarge": {Region: "ap-northeast-2", Type: "c5.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.384}, "c5.4xlarge": {Region: "ap-northeast-2", Type: "c5.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.768}, "c5.9xlarge": {Region: "ap-northeast-2", Type: "c5.9xlarge", Memory: kresource.MustParse("73728Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 1.728}, "c5.12xlarge": {Region: "ap-northeast-2", Type: "c5.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.304}, "c5.18xlarge": {Region: "ap-northeast-2", Type: "c5.18xlarge", Memory: kresource.MustParse("147456Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 3.456}, "c5.24xlarge": {Region: "ap-northeast-2", Type: "c5.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.608}, "c5.metal": {Region: "ap-northeast-2", Type: "c5.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.608}, "c5a.large": {Region: "ap-northeast-2", Type: "c5a.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.086}, "c5a.xlarge": {Region: "ap-northeast-2", Type: "c5a.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.172}, "c5a.2xlarge": {Region: "ap-northeast-2", Type: "c5a.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.344}, "c5a.4xlarge": {Region: "ap-northeast-2", Type: "c5a.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.688}, "c5a.8xlarge": {Region: "ap-northeast-2", Type: "c5a.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.376}, "c5a.12xlarge": {Region: "ap-northeast-2", Type: "c5a.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.064}, "c5a.16xlarge": {Region: "ap-northeast-2", Type: "c5a.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.752}, "c5a.24xlarge": {Region: "ap-northeast-2", Type: "c5a.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.128}, "c5d.large": {Region: "ap-northeast-2", Type: "c5d.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.11}, "c5d.xlarge": {Region: "ap-northeast-2", Type: "c5d.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.22}, "c5d.2xlarge": {Region: "ap-northeast-2", Type: "c5d.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.44}, "c5d.4xlarge": {Region: "ap-northeast-2", Type: "c5d.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.88}, "c5d.9xlarge": {Region: "ap-northeast-2", Type: "c5d.9xlarge", Memory: kresource.MustParse("73728Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 1.98}, "c5d.12xlarge": {Region: "ap-northeast-2", Type: "c5d.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.64}, "c5d.18xlarge": {Region: "ap-northeast-2", Type: "c5d.18xlarge", Memory: kresource.MustParse("147456Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 3.96}, "c5d.24xlarge": {Region: "ap-northeast-2", Type: "c5d.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.28}, "c5d.metal": {Region: "ap-northeast-2", Type: "c5d.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.28}, "c5n.large": {Region: "ap-northeast-2", Type: "c5n.large", Memory: kresource.MustParse("5376Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.122}, "c5n.xlarge": {Region: "ap-northeast-2", Type: "c5n.xlarge", Memory: kresource.MustParse("10752Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.244}, "c5n.2xlarge": {Region: "ap-northeast-2", Type: "c5n.2xlarge", Memory: kresource.MustParse("21504Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.488}, "c5n.4xlarge": {Region: "ap-northeast-2", Type: "c5n.4xlarge", Memory: kresource.MustParse("43008Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.976}, "c5n.9xlarge": {Region: "ap-northeast-2", Type: "c5n.9xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 2.196}, "c5n.18xlarge": {Region: "ap-northeast-2", Type: "c5n.18xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 4.392}, "c5n.metal": {Region: "ap-northeast-2", Type: "c5n.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 4.392}, "c6g.medium": {Region: "ap-northeast-2", Type: "c6g.medium", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0385}, "c6g.large": {Region: "ap-northeast-2", Type: "c6g.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.077}, "c6g.xlarge": {Region: "ap-northeast-2", Type: "c6g.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.154}, "c6g.2xlarge": {Region: "ap-northeast-2", Type: "c6g.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.308}, "c6g.4xlarge": {Region: "ap-northeast-2", Type: "c6g.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.616}, "c6g.8xlarge": {Region: "ap-northeast-2", Type: "c6g.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.232}, "c6g.12xlarge": {Region: "ap-northeast-2", Type: "c6g.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 1.848}, "c6g.16xlarge": {Region: "ap-northeast-2", Type: "c6g.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.464}, "c6g.metal": {Region: "ap-northeast-2", Type: "c6g.metal", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.464}, "c6i.large": {Region: "ap-northeast-2", Type: "c6i.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.096}, "c6i.xlarge": {Region: "ap-northeast-2", Type: "c6i.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.192}, "c6i.2xlarge": {Region: "ap-northeast-2", Type: "c6i.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.384}, "c6i.4xlarge": {Region: "ap-northeast-2", Type: "c6i.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.768}, "c6i.8xlarge": {Region: "ap-northeast-2", Type: "c6i.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.536}, "c6i.12xlarge": {Region: "ap-northeast-2", Type: "c6i.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.304}, "c6i.16xlarge": {Region: "ap-northeast-2", Type: "c6i.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.072}, "c6i.24xlarge": {Region: "ap-northeast-2", Type: "c6i.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.608}, "c6i.32xlarge": {Region: "ap-northeast-2", Type: "c6i.32xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 6.144}, "c6i.metal": {Region: "ap-northeast-2", Type: "c6i.metal", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 6.144}, "d2.xlarge": {Region: "ap-northeast-2", Type: "d2.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.844}, "d2.2xlarge": {Region: "ap-northeast-2", Type: "d2.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.688}, "d2.4xlarge": {Region: "ap-northeast-2", Type: "d2.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 3.376}, "d2.8xlarge": {Region: "ap-northeast-2", Type: "d2.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 6.752}, "g2.2xlarge": {Region: "ap-northeast-2", Type: "g2.2xlarge", Memory: kresource.MustParse("15360Mi"), CPU: kresource.MustParse("8"), GPU: 1, Inf: 0, Price: 0.898}, "g2.8xlarge": {Region: "ap-northeast-2", Type: "g2.8xlarge", Memory: kresource.MustParse("61440Mi"), CPU: kresource.MustParse("32"), GPU: 4, Inf: 0, Price: 3.592}, "g3.4xlarge": {Region: "ap-northeast-2", Type: "g3.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 1, Inf: 0, Price: 1.42}, "g3.8xlarge": {Region: "ap-northeast-2", Type: "g3.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 2, Inf: 0, Price: 2.84}, "g3.16xlarge": {Region: "ap-northeast-2", Type: "g3.16xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("64"), GPU: 4, Inf: 0, Price: 5.68}, "g3s.xlarge": {Region: "ap-northeast-2", Type: "g3s.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 1, Inf: 0, Price: 0.934}, "g4dn.xlarge": {Region: "ap-northeast-2", Type: "g4dn.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 1, Inf: 0, Price: 0.647}, "g4dn.2xlarge": {Region: "ap-northeast-2", Type: "g4dn.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 1, Inf: 0, Price: 0.925}, "g4dn.4xlarge": {Region: "ap-northeast-2", Type: "g4dn.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 1, Inf: 0, Price: 1.481}, "g4dn.8xlarge": {Region: "ap-northeast-2", Type: "g4dn.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 1, Inf: 0, Price: 2.677}, "g4dn.12xlarge": {Region: "ap-northeast-2", Type: "g4dn.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 4, Inf: 0, Price: 4.812}, "g4dn.16xlarge": {Region: "ap-northeast-2", Type: "g4dn.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 1, Inf: 0, Price: 5.353}, "g4dn.metal": {Region: "ap-northeast-2", Type: "g4dn.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 8, Inf: 0, Price: 9.624}, "g5g.xlarge": {Region: "ap-northeast-2", Type: "g5g.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 1, Inf: 0, Price: 0.5166}, "g5g.2xlarge": {Region: "ap-northeast-2", Type: "g5g.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 1, Inf: 0, Price: 0.6839}, "g5g.4xlarge": {Region: "ap-northeast-2", Type: "g5g.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 1, Inf: 0, Price: 1.0185}, "g5g.8xlarge": {Region: "ap-northeast-2", Type: "g5g.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 1, Inf: 0, Price: 1.6876}, "g5g.16xlarge": {Region: "ap-northeast-2", Type: "g5g.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 2, Inf: 0, Price: 3.3752}, "g5g.metal": {Region: "ap-northeast-2", Type: "g5g.metal", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 2, Inf: 0, Price: 3.3752}, "i2.xlarge": {Region: "ap-northeast-2", Type: "i2.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 1.001}, "i2.2xlarge": {Region: "ap-northeast-2", Type: "i2.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 2.001}, "i2.4xlarge": {Region: "ap-northeast-2", Type: "i2.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 4.002}, "i2.8xlarge": {Region: "ap-northeast-2", Type: "i2.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 8.004}, "i3.large": {Region: "ap-northeast-2", Type: "i3.large", Memory: kresource.MustParse("15616Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.183}, "i3.xlarge": {Region: "ap-northeast-2", Type: "i3.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.366}, "i3.2xlarge": {Region: "ap-northeast-2", Type: "i3.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.732}, "i3.4xlarge": {Region: "ap-northeast-2", Type: "i3.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.464}, "i3.8xlarge": {Region: "ap-northeast-2", Type: "i3.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.928}, "i3.16xlarge": {Region: "ap-northeast-2", Type: "i3.16xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.856}, "i3.metal": {Region: "ap-northeast-2", Type: "i3.metal", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.856}, "i3en.large": {Region: "ap-northeast-2", Type: "i3en.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.266}, "i3en.xlarge": {Region: "ap-northeast-2", Type: "i3en.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.532}, "i3en.2xlarge": {Region: "ap-northeast-2", Type: "i3en.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.064}, "i3en.3xlarge": {Region: "ap-northeast-2", Type: "i3en.3xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("12"), GPU: 0, Inf: 0, Price: 1.596}, "i3en.6xlarge": {Region: "ap-northeast-2", Type: "i3en.6xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 0, Price: 3.192}, "i3en.12xlarge": {Region: "ap-northeast-2", Type: "i3en.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 6.384}, "i3en.24xlarge": {Region: "ap-northeast-2", Type: "i3en.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 12.768}, "i3en.metal": {Region: "ap-northeast-2", Type: "i3en.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 12.768}, "inf1.xlarge": {Region: "ap-northeast-2", Type: "inf1.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 1, Price: 0.281}, "inf1.2xlarge": {Region: "ap-northeast-2", Type: "inf1.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 1, Price: 0.446}, "inf1.6xlarge": {Region: "ap-northeast-2", Type: "inf1.6xlarge", Memory: kresource.MustParse("49152Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 4, Price: 1.453}, "inf1.24xlarge": {Region: "ap-northeast-2", Type: "inf1.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 16, Price: 5.812}, "m3.medium": {Region: "ap-northeast-2", Type: "m3.medium", Memory: kresource.MustParse("3840Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.091}, "m3.large": {Region: "ap-northeast-2", Type: "m3.large", Memory: kresource.MustParse("7680Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.183}, "m3.xlarge": {Region: "ap-northeast-2", Type: "m3.xlarge", Memory: kresource.MustParse("15360Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.366}, "m3.2xlarge": {Region: "ap-northeast-2", Type: "m3.2xlarge", Memory: kresource.MustParse("30720Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.732}, "m4.large": {Region: "ap-northeast-2", Type: "m4.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.123}, "m4.xlarge": {Region: "ap-northeast-2", Type: "m4.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.246}, "m4.2xlarge": {Region: "ap-northeast-2", Type: "m4.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.492}, "m4.4xlarge": {Region: "ap-northeast-2", Type: "m4.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.984}, "m4.10xlarge": {Region: "ap-northeast-2", Type: "m4.10xlarge", Memory: kresource.MustParse("163840Mi"), CPU: kresource.MustParse("40"), GPU: 0, Inf: 0, Price: 2.46}, "m4.16xlarge": {Region: "ap-northeast-2", Type: "m4.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.936}, "m5.large": {Region: "ap-northeast-2", Type: "m5.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.118}, "m5.xlarge": {Region: "ap-northeast-2", Type: "m5.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.236}, "m5.2xlarge": {Region: "ap-northeast-2", Type: "m5.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.472}, "m5.4xlarge": {Region: "ap-northeast-2", Type: "m5.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.944}, "m5.8xlarge": {Region: "ap-northeast-2", Type: "m5.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.888}, "m5.12xlarge": {Region: "ap-northeast-2", Type: "m5.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.832}, "m5.16xlarge": {Region: "ap-northeast-2", Type: "m5.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.776}, "m5.24xlarge": {Region: "ap-northeast-2", Type: "m5.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.664}, "m5.metal": {Region: "ap-northeast-2", Type: "m5.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.664}, "m5a.large": {Region: "ap-northeast-2", Type: "m5a.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.106}, "m5a.xlarge": {Region: "ap-northeast-2", Type: "m5a.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.212}, "m5a.2xlarge": {Region: "ap-northeast-2", Type: "m5a.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.424}, "m5a.4xlarge": {Region: "ap-northeast-2", Type: "m5a.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.848}, "m5a.8xlarge": {Region: "ap-northeast-2", Type: "m5a.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.696}, "m5a.12xlarge": {Region: "ap-northeast-2", Type: "m5a.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.544}, "m5a.16xlarge": {Region: "ap-northeast-2", Type: "m5a.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.392}, "m5a.24xlarge": {Region: "ap-northeast-2", Type: "m5a.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.088}, "m5ad.large": {Region: "ap-northeast-2", Type: "m5ad.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.127}, "m5ad.xlarge": {Region: "ap-northeast-2", Type: "m5ad.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.254}, "m5ad.2xlarge": {Region: "ap-northeast-2", Type: "m5ad.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.508}, "m5ad.4xlarge": {Region: "ap-northeast-2", Type: "m5ad.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.016}, "m5ad.8xlarge": {Region: "ap-northeast-2", Type: "m5ad.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.032}, "m5ad.12xlarge": {Region: "ap-northeast-2", Type: "m5ad.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.048}, "m5ad.16xlarge": {Region: "ap-northeast-2", Type: "m5ad.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.064}, "m5ad.24xlarge": {Region: "ap-northeast-2", Type: "m5ad.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.096}, "m5d.large": {Region: "ap-northeast-2", Type: "m5d.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.139}, "m5d.xlarge": {Region: "ap-northeast-2", Type: "m5d.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.278}, "m5d.2xlarge": {Region: "ap-northeast-2", Type: "m5d.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.556}, "m5d.4xlarge": {Region: "ap-northeast-2", Type: "m5d.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.112}, "m5d.8xlarge": {Region: "ap-northeast-2", Type: "m5d.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.224}, "m5d.12xlarge": {Region: "ap-northeast-2", Type: "m5d.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.336}, "m5d.16xlarge": {Region: "ap-northeast-2", Type: "m5d.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.448}, "m5d.24xlarge": {Region: "ap-northeast-2", Type: "m5d.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.672}, "m5d.metal": {Region: "ap-northeast-2", Type: "m5d.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.672}, "m5zn.large": {Region: "ap-northeast-2", Type: "m5zn.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.203}, "m5zn.xlarge": {Region: "ap-northeast-2", Type: "m5zn.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.406}, "m5zn.2xlarge": {Region: "ap-northeast-2", Type: "m5zn.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.812}, "m5zn.3xlarge": {Region: "ap-northeast-2", Type: "m5zn.3xlarge", Memory: kresource.MustParse("49152Mi"), CPU: kresource.MustParse("12"), GPU: 0, Inf: 0, Price: 1.218}, "m5zn.6xlarge": {Region: "ap-northeast-2", Type: "m5zn.6xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 0, Price: 2.436}, "m5zn.12xlarge": {Region: "ap-northeast-2", Type: "m5zn.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.872}, "m5zn.metal": {Region: "ap-northeast-2", Type: "m5zn.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.872}, "m6g.medium": {Region: "ap-northeast-2", Type: "m6g.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.047}, "m6g.large": {Region: "ap-northeast-2", Type: "m6g.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.094}, "m6g.xlarge": {Region: "ap-northeast-2", Type: "m6g.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.188}, "m6g.2xlarge": {Region: "ap-northeast-2", Type: "m6g.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.376}, "m6g.4xlarge": {Region: "ap-northeast-2", Type: "m6g.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.752}, "m6g.8xlarge": {Region: "ap-northeast-2", Type: "m6g.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.504}, "m6g.12xlarge": {Region: "ap-northeast-2", Type: "m6g.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.256}, "m6g.16xlarge": {Region: "ap-northeast-2", Type: "m6g.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.008}, "m6g.metal": {Region: "ap-northeast-2", Type: "m6g.metal", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.008}, "m6i.large": {Region: "ap-northeast-2", Type: "m6i.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.118}, "m6i.xlarge": {Region: "ap-northeast-2", Type: "m6i.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.236}, "m6i.2xlarge": {Region: "ap-northeast-2", Type: "m6i.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.472}, "m6i.4xlarge": {Region: "ap-northeast-2", Type: "m6i.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.944}, "m6i.8xlarge": {Region: "ap-northeast-2", Type: "m6i.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.888}, "m6i.12xlarge": {Region: "ap-northeast-2", Type: "m6i.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.832}, "m6i.16xlarge": {Region: "ap-northeast-2", Type: "m6i.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.776}, "m6i.24xlarge": {Region: "ap-northeast-2", Type: "m6i.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.664}, "m6i.32xlarge": {Region: "ap-northeast-2", Type: "m6i.32xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 7.552}, "m6i.metal": {Region: "ap-northeast-2", Type: "m6i.metal", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 7.552}, "p2.xlarge": {Region: "ap-northeast-2", Type: "p2.xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("4"), GPU: 1, Inf: 0, Price: 1.465}, "p2.8xlarge": {Region: "ap-northeast-2", Type: "p2.8xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("32"), GPU: 8, Inf: 0, Price: 11.72}, "p2.16xlarge": {Region: "ap-northeast-2", Type: "p2.16xlarge", Memory: kresource.MustParse("749568Mi"), CPU: kresource.MustParse("64"), GPU: 16, Inf: 0, Price: 23.44}, "p3.2xlarge": {Region: "ap-northeast-2", Type: "p3.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 1, Inf: 0, Price: 4.234}, "p3.8xlarge": {Region: "ap-northeast-2", Type: "p3.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 4, Inf: 0, Price: 16.936}, "p3.16xlarge": {Region: "ap-northeast-2", Type: "p3.16xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("64"), GPU: 8, Inf: 0, Price: 33.872}, "p4d.24xlarge": {Region: "ap-northeast-2", Type: "p4d.24xlarge", Memory: kresource.MustParse("1179648Mi"), CPU: kresource.MustParse("96"), GPU: 8, Inf: 0, Price: 45.38848}, "r3.large": {Region: "ap-northeast-2", Type: "r3.large", Memory: kresource.MustParse("15616Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.2}, "r3.xlarge": {Region: "ap-northeast-2", Type: "r3.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.399}, "r3.2xlarge": {Region: "ap-northeast-2", Type: "r3.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.798}, "r3.4xlarge": {Region: "ap-northeast-2", Type: "r3.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.596}, "r3.8xlarge": {Region: "ap-northeast-2", Type: "r3.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 3.192}, "r4.large": {Region: "ap-northeast-2", Type: "r4.large", Memory: kresource.MustParse("15616Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.16}, "r4.xlarge": {Region: "ap-northeast-2", Type: "r4.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.32}, "r4.2xlarge": {Region: "ap-northeast-2", Type: "r4.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.64}, "r4.4xlarge": {Region: "ap-northeast-2", Type: "r4.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.28}, "r4.8xlarge": {Region: "ap-northeast-2", Type: "r4.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.56}, "r4.16xlarge": {Region: "ap-northeast-2", Type: "r4.16xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.12}, "r5.large": {Region: "ap-northeast-2", Type: "r5.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.152}, "r5.xlarge": {Region: "ap-northeast-2", Type: "r5.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.304}, "r5.2xlarge": {Region: "ap-northeast-2", Type: "r5.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.608}, "r5.4xlarge": {Region: "ap-northeast-2", Type: "r5.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.216}, "r5.8xlarge": {Region: "ap-northeast-2", Type: "r5.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.432}, "r5.12xlarge": {Region: "ap-northeast-2", Type: "r5.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.648}, "r5.16xlarge": {Region: "ap-northeast-2", Type: "r5.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.864}, "r5.24xlarge": {Region: "ap-northeast-2", Type: "r5.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.296}, "r5.metal": {Region: "ap-northeast-2", Type: "r5.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.296}, "r5a.large": {Region: "ap-northeast-2", Type: "r5a.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.136}, "r5a.xlarge": {Region: "ap-northeast-2", Type: "r5a.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.272}, "r5a.2xlarge": {Region: "ap-northeast-2", Type: "r5a.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.544}, "r5a.4xlarge": {Region: "ap-northeast-2", Type: "r5a.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.088}, "r5a.8xlarge": {Region: "ap-northeast-2", Type: "r5a.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.176}, "r5a.12xlarge": {Region: "ap-northeast-2", Type: "r5a.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.264}, "r5a.16xlarge": {Region: "ap-northeast-2", Type: "r5a.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.352}, "r5a.24xlarge": {Region: "ap-northeast-2", Type: "r5a.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.528}, "r5ad.large": {Region: "ap-northeast-2", Type: "r5ad.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.158}, "r5ad.xlarge": {Region: "ap-northeast-2", Type: "r5ad.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.316}, "r5ad.2xlarge": {Region: "ap-northeast-2", Type: "r5ad.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.632}, "r5ad.4xlarge": {Region: "ap-northeast-2", Type: "r5ad.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.264}, "r5ad.8xlarge": {Region: "ap-northeast-2", Type: "r5ad.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.528}, "r5ad.12xlarge": {Region: "ap-northeast-2", Type: "r5ad.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.792}, "r5ad.16xlarge": {Region: "ap-northeast-2", Type: "r5ad.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.056}, "r5ad.24xlarge": {Region: "ap-northeast-2", Type: "r5ad.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.584}, "r5b.large": {Region: "ap-northeast-2", Type: "r5b.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.178}, "r5b.xlarge": {Region: "ap-northeast-2", Type: "r5b.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.356}, "r5b.2xlarge": {Region: "ap-northeast-2", Type: "r5b.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.712}, "r5b.4xlarge": {Region: "ap-northeast-2", Type: "r5b.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.424}, "r5b.8xlarge": {Region: "ap-northeast-2", Type: "r5b.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.848}, "r5b.12xlarge": {Region: "ap-northeast-2", Type: "r5b.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.272}, "r5b.16xlarge": {Region: "ap-northeast-2", Type: "r5b.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.696}, "r5b.24xlarge": {Region: "ap-northeast-2", Type: "r5b.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.544}, "r5b.metal": {Region: "ap-northeast-2", Type: "r5b.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.544}, "r5d.large": {Region: "ap-northeast-2", Type: "r5d.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.173}, "r5d.xlarge": {Region: "ap-northeast-2", Type: "r5d.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.346}, "r5d.2xlarge": {Region: "ap-northeast-2", Type: "r5d.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.692}, "r5d.4xlarge": {Region: "ap-northeast-2", Type: "r5d.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.384}, "r5d.8xlarge": {Region: "ap-northeast-2", Type: "r5d.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.768}, "r5d.12xlarge": {Region: "ap-northeast-2", Type: "r5d.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.152}, "r5d.16xlarge": {Region: "ap-northeast-2", Type: "r5d.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.536}, "r5d.24xlarge": {Region: "ap-northeast-2", Type: "r5d.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.304}, "r5d.metal": {Region: "ap-northeast-2", Type: "r5d.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.304}, "r5dn.large": {Region: "ap-northeast-2", Type: "r5dn.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.199}, "r5dn.xlarge": {Region: "ap-northeast-2", Type: "r5dn.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.398}, "r5dn.2xlarge": {Region: "ap-northeast-2", Type: "r5dn.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.796}, "r5dn.4xlarge": {Region: "ap-northeast-2", Type: "r5dn.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.592}, "r5dn.8xlarge": {Region: "ap-northeast-2", Type: "r5dn.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 3.184}, "r5dn.12xlarge": {Region: "ap-northeast-2", Type: "r5dn.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.776}, "r5dn.16xlarge": {Region: "ap-northeast-2", Type: "r5dn.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 6.368}, "r5dn.24xlarge": {Region: "ap-northeast-2", Type: "r5dn.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 9.552}, "r5dn.metal": {Region: "ap-northeast-2", Type: "r5dn.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 9.552}, "r5n.large": {Region: "ap-northeast-2", Type: "r5n.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.178}, "r5n.xlarge": {Region: "ap-northeast-2", Type: "r5n.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.356}, "r5n.2xlarge": {Region: "ap-northeast-2", Type: "r5n.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.712}, "r5n.4xlarge": {Region: "ap-northeast-2", Type: "r5n.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.424}, "r5n.8xlarge": {Region: "ap-northeast-2", Type: "r5n.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.848}, "r5n.12xlarge": {Region: "ap-northeast-2", Type: "r5n.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.272}, "r5n.16xlarge": {Region: "ap-northeast-2", Type: "r5n.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.696}, "r5n.24xlarge": {Region: "ap-northeast-2", Type: "r5n.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.544}, "r5n.metal": {Region: "ap-northeast-2", Type: "r5n.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.544}, "r6g.medium": {Region: "ap-northeast-2", Type: "r6g.medium", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.061}, "r6g.large": {Region: "ap-northeast-2", Type: "r6g.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.122}, "r6g.xlarge": {Region: "ap-northeast-2", Type: "r6g.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.244}, "r6g.2xlarge": {Region: "ap-northeast-2", Type: "r6g.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.488}, "r6g.4xlarge": {Region: "ap-northeast-2", Type: "r6g.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.976}, "r6g.8xlarge": {Region: "ap-northeast-2", Type: "r6g.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.952}, "r6g.12xlarge": {Region: "ap-northeast-2", Type: "r6g.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.928}, "r6g.16xlarge": {Region: "ap-northeast-2", Type: "r6g.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.904}, "r6g.metal": {Region: "ap-northeast-2", Type: "r6g.metal", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.904}, "r6i.large": {Region: "ap-northeast-2", Type: "r6i.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.152}, "r6i.xlarge": {Region: "ap-northeast-2", Type: "r6i.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.304}, "r6i.2xlarge": {Region: "ap-northeast-2", Type: "r6i.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.608}, "r6i.4xlarge": {Region: "ap-northeast-2", Type: "r6i.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.216}, "r6i.8xlarge": {Region: "ap-northeast-2", Type: "r6i.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.432}, "r6i.12xlarge": {Region: "ap-northeast-2", Type: "r6i.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.648}, "r6i.16xlarge": {Region: "ap-northeast-2", Type: "r6i.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.864}, "r6i.24xlarge": {Region: "ap-northeast-2", Type: "r6i.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.296}, "r6i.32xlarge": {Region: "ap-northeast-2", Type: "r6i.32xlarge", Memory: kresource.MustParse("1048576Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 9.728}, "r6i.metal": {Region: "ap-northeast-2", Type: "r6i.metal", Memory: kresource.MustParse("1048576Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 9.728}, "t2.nano": {Region: "ap-northeast-2", Type: "t2.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0072}, "t2.micro": {Region: "ap-northeast-2", Type: "t2.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0144}, "t2.small": {Region: "ap-northeast-2", Type: "t2.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0288}, "t2.medium": {Region: "ap-northeast-2", Type: "t2.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0576}, "t2.large": {Region: "ap-northeast-2", Type: "t2.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.1152}, "t2.xlarge": {Region: "ap-northeast-2", Type: "t2.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.2304}, "t2.2xlarge": {Region: "ap-northeast-2", Type: "t2.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.4608}, "t3.nano": {Region: "ap-northeast-2", Type: "t3.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0065}, "t3.micro": {Region: "ap-northeast-2", Type: "t3.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.013}, "t3.small": {Region: "ap-northeast-2", Type: "t3.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.026}, "t3.medium": {Region: "ap-northeast-2", Type: "t3.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.052}, "t3.large": {Region: "ap-northeast-2", Type: "t3.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.104}, "t3.xlarge": {Region: "ap-northeast-2", Type: "t3.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.208}, "t3.2xlarge": {Region: "ap-northeast-2", Type: "t3.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.416}, "t3a.nano": {Region: "ap-northeast-2", Type: "t3a.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0059}, "t3a.micro": {Region: "ap-northeast-2", Type: "t3a.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0117}, "t3a.small": {Region: "ap-northeast-2", Type: "t3a.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0234}, "t3a.medium": {Region: "ap-northeast-2", Type: "t3a.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0468}, "t3a.large": {Region: "ap-northeast-2", Type: "t3a.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0936}, "t3a.xlarge": {Region: "ap-northeast-2", Type: "t3a.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1872}, "t3a.2xlarge": {Region: "ap-northeast-2", Type: "t3a.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3744}, "t4g.nano": {Region: "ap-northeast-2", Type: "t4g.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0052}, "t4g.micro": {Region: "ap-northeast-2", Type: "t4g.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0104}, "t4g.small": {Region: "ap-northeast-2", Type: "t4g.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0208}, "t4g.medium": {Region: "ap-northeast-2", Type: "t4g.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0416}, "t4g.large": {Region: "ap-northeast-2", Type: "t4g.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0832}, "t4g.xlarge": {Region: "ap-northeast-2", Type: "t4g.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1664}, "t4g.2xlarge": {Region: "ap-northeast-2", Type: "t4g.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3328}, "u-6tb1.56xlarge": {Region: "ap-northeast-2", Type: "u-6tb1.56xlarge", Memory: kresource.MustParse("6291456Mi"), CPU: kresource.MustParse("224"), GPU: 0, Inf: 0, Price: 55.9796}, "u-6tb1.112xlarge": {Region: "ap-northeast-2", Type: "u-6tb1.112xlarge", Memory: kresource.MustParse("6291456Mi"), CPU: kresource.MustParse("448"), GPU: 0, Inf: 0, Price: 65.867}, "x1.16xlarge": {Region: "ap-northeast-2", Type: "x1.16xlarge", Memory: kresource.MustParse("999424Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 9.671}, "x1.32xlarge": {Region: "ap-northeast-2", Type: "x1.32xlarge", Memory: kresource.MustParse("1998848Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 19.341}, "x1e.xlarge": {Region: "ap-northeast-2", Type: "x1e.xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 1.209}, "x1e.2xlarge": {Region: "ap-northeast-2", Type: "x1e.2xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 2.418}, "x1e.4xlarge": {Region: "ap-northeast-2", Type: "x1e.4xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 4.836}, "x1e.8xlarge": {Region: "ap-northeast-2", Type: "x1e.8xlarge", Memory: kresource.MustParse("999424Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 9.672}, "x1e.16xlarge": {Region: "ap-northeast-2", Type: "x1e.16xlarge", Memory: kresource.MustParse("1998848Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 19.344}, "x1e.32xlarge": {Region: "ap-northeast-2", Type: "x1e.32xlarge", Memory: kresource.MustParse("3997696Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 38.688}, "z1d.large": {Region: "ap-northeast-2", Type: "z1d.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.225}, "z1d.xlarge": {Region: "ap-northeast-2", Type: "z1d.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.45}, "z1d.2xlarge": {Region: "ap-northeast-2", Type: "z1d.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.9}, "z1d.3xlarge": {Region: "ap-northeast-2", Type: "z1d.3xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("12"), GPU: 0, Inf: 0, Price: 1.35}, "z1d.6xlarge": {Region: "ap-northeast-2", Type: "z1d.6xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 0, Price: 2.7}, "z1d.12xlarge": {Region: "ap-northeast-2", Type: "z1d.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 5.4}, "z1d.metal": {Region: "ap-northeast-2", Type: "z1d.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 5.4}, }, "ap-northeast-3": { "c3.large": {Region: "ap-northeast-3", Type: "c3.large", Memory: kresource.MustParse("3840Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.128}, "c3.xlarge": {Region: "ap-northeast-3", Type: "c3.xlarge", Memory: kresource.MustParse("7680Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.256}, "c3.2xlarge": {Region: "ap-northeast-3", Type: "c3.2xlarge", Memory: kresource.MustParse("15360Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.512}, "c3.4xlarge": {Region: "ap-northeast-3", Type: "c3.4xlarge", Memory: kresource.MustParse("30720Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.024}, "c3.8xlarge": {Region: "ap-northeast-3", Type: "c3.8xlarge", Memory: kresource.MustParse("61440Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.048}, "c4.large": {Region: "ap-northeast-3", Type: "c4.large", Memory: kresource.MustParse("3840Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.126}, "c4.xlarge": {Region: "ap-northeast-3", Type: "c4.xlarge", Memory: kresource.MustParse("7680Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.252}, "c4.2xlarge": {Region: "ap-northeast-3", Type: "c4.2xlarge", Memory: kresource.MustParse("15360Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.504}, "c4.4xlarge": {Region: "ap-northeast-3", Type: "c4.4xlarge", Memory: kresource.MustParse("30720Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.008}, "c4.8xlarge": {Region: "ap-northeast-3", Type: "c4.8xlarge", Memory: kresource.MustParse("61440Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 2.016}, "c5.large": {Region: "ap-northeast-3", Type: "c5.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.107}, "c5.xlarge": {Region: "ap-northeast-3", Type: "c5.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.214}, "c5.2xlarge": {Region: "ap-northeast-3", Type: "c5.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.428}, "c5.4xlarge": {Region: "ap-northeast-3", Type: "c5.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.856}, "c5.9xlarge": {Region: "ap-northeast-3", Type: "c5.9xlarge", Memory: kresource.MustParse("73728Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 1.926}, "c5.12xlarge": {Region: "ap-northeast-3", Type: "c5.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.568}, "c5.18xlarge": {Region: "ap-northeast-3", Type: "c5.18xlarge", Memory: kresource.MustParse("147456Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 3.852}, "c5.24xlarge": {Region: "ap-northeast-3", Type: "c5.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.136}, "c5.metal": {Region: "ap-northeast-3", Type: "c5.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.136}, "c5d.large": {Region: "ap-northeast-3", Type: "c5d.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.122}, "c5d.xlarge": {Region: "ap-northeast-3", Type: "c5d.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.244}, "c5d.2xlarge": {Region: "ap-northeast-3", Type: "c5d.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.488}, "c5d.4xlarge": {Region: "ap-northeast-3", Type: "c5d.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.976}, "c5d.9xlarge": {Region: "ap-northeast-3", Type: "c5d.9xlarge", Memory: kresource.MustParse("73728Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 2.196}, "c5d.12xlarge": {Region: "ap-northeast-3", Type: "c5d.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.928}, "c5d.18xlarge": {Region: "ap-northeast-3", Type: "c5d.18xlarge", Memory: kresource.MustParse("147456Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 4.392}, "c5d.24xlarge": {Region: "ap-northeast-3", Type: "c5d.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.856}, "c5d.metal": {Region: "ap-northeast-3", Type: "c5d.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.856}, "d2.xlarge": {Region: "ap-northeast-3", Type: "d2.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.844}, "d2.2xlarge": {Region: "ap-northeast-3", Type: "d2.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.688}, "d2.4xlarge": {Region: "ap-northeast-3", Type: "d2.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 3.376}, "d2.8xlarge": {Region: "ap-northeast-3", Type: "d2.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 6.752}, "g4dn.xlarge": {Region: "ap-northeast-3", Type: "g4dn.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 1, Inf: 0, Price: 0.71}, "g4dn.2xlarge": {Region: "ap-northeast-3", Type: "g4dn.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 1, Inf: 0, Price: 1.015}, "g4dn.4xlarge": {Region: "ap-northeast-3", Type: "g4dn.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 1, Inf: 0, Price: 1.625}, "g4dn.8xlarge": {Region: "ap-northeast-3", Type: "g4dn.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 1, Inf: 0, Price: 2.938}, "g4dn.12xlarge": {Region: "ap-northeast-3", Type: "g4dn.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 4, Inf: 0, Price: 5.281}, "g4dn.16xlarge": {Region: "ap-northeast-3", Type: "g4dn.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 1, Inf: 0, Price: 5.875}, "g4dn.metal": {Region: "ap-northeast-3", Type: "g4dn.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 8, Inf: 0, Price: 10.562}, "i3.large": {Region: "ap-northeast-3", Type: "i3.large", Memory: kresource.MustParse("15616Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.183}, "i3.xlarge": {Region: "ap-northeast-3", Type: "i3.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.366}, "i3.2xlarge": {Region: "ap-northeast-3", Type: "i3.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.732}, "i3.4xlarge": {Region: "ap-northeast-3", Type: "i3.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.464}, "i3.8xlarge": {Region: "ap-northeast-3", Type: "i3.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.928}, "i3.16xlarge": {Region: "ap-northeast-3", Type: "i3.16xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.856}, "i3.metal": {Region: "ap-northeast-3", Type: "i3.metal", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.856}, "i3en.large": {Region: "ap-northeast-3", Type: "i3en.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.266}, "i3en.xlarge": {Region: "ap-northeast-3", Type: "i3en.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.532}, "i3en.2xlarge": {Region: "ap-northeast-3", Type: "i3en.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.064}, "i3en.3xlarge": {Region: "ap-northeast-3", Type: "i3en.3xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("12"), GPU: 0, Inf: 0, Price: 1.596}, "i3en.6xlarge": {Region: "ap-northeast-3", Type: "i3en.6xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 0, Price: 3.192}, "i3en.12xlarge": {Region: "ap-northeast-3", Type: "i3en.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 6.384}, "i3en.24xlarge": {Region: "ap-northeast-3", Type: "i3en.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 12.768}, "i3en.metal": {Region: "ap-northeast-3", Type: "i3en.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 12.768}, "m3.medium": {Region: "ap-northeast-3", Type: "m3.medium", Memory: kresource.MustParse("3840Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.096}, "m3.large": {Region: "ap-northeast-3", Type: "m3.large", Memory: kresource.MustParse("7680Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.192}, "m3.xlarge": {Region: "ap-northeast-3", Type: "m3.xlarge", Memory: kresource.MustParse("15360Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.384}, "m3.2xlarge": {Region: "ap-northeast-3", Type: "m3.2xlarge", Memory: kresource.MustParse("30720Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.768}, "m4.large": {Region: "ap-northeast-3", Type: "m4.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.129}, "m4.xlarge": {Region: "ap-northeast-3", Type: "m4.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.258}, "m4.2xlarge": {Region: "ap-northeast-3", Type: "m4.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.516}, "m4.4xlarge": {Region: "ap-northeast-3", Type: "m4.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.032}, "m4.10xlarge": {Region: "ap-northeast-3", Type: "m4.10xlarge", Memory: kresource.MustParse("163840Mi"), CPU: kresource.MustParse("40"), GPU: 0, Inf: 0, Price: 2.58}, "m4.16xlarge": {Region: "ap-northeast-3", Type: "m4.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.128}, "m5.large": {Region: "ap-northeast-3", Type: "m5.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.124}, "m5.xlarge": {Region: "ap-northeast-3", Type: "m5.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.248}, "m5.2xlarge": {Region: "ap-northeast-3", Type: "m5.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.496}, "m5.4xlarge": {Region: "ap-northeast-3", Type: "m5.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.992}, "m5.8xlarge": {Region: "ap-northeast-3", Type: "m5.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.984}, "m5.12xlarge": {Region: "ap-northeast-3", Type: "m5.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.976}, "m5.16xlarge": {Region: "ap-northeast-3", Type: "m5.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.968}, "m5.24xlarge": {Region: "ap-northeast-3", Type: "m5.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.952}, "m5.metal": {Region: "ap-northeast-3", Type: "m5.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.952}, "m5d.large": {Region: "ap-northeast-3", Type: "m5d.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.146}, "m5d.xlarge": {Region: "ap-northeast-3", Type: "m5d.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.292}, "m5d.2xlarge": {Region: "ap-northeast-3", Type: "m5d.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.584}, "m5d.4xlarge": {Region: "ap-northeast-3", Type: "m5d.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.168}, "m5d.8xlarge": {Region: "ap-northeast-3", Type: "m5d.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.336}, "m5d.12xlarge": {Region: "ap-northeast-3", Type: "m5d.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.504}, "m5d.16xlarge": {Region: "ap-northeast-3", Type: "m5d.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.672}, "m5d.24xlarge": {Region: "ap-northeast-3", Type: "m5d.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.008}, "m5d.metal": {Region: "ap-northeast-3", Type: "m5d.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.008}, "r3.large": {Region: "ap-northeast-3", Type: "r3.large", Memory: kresource.MustParse("15616Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.2}, "r3.xlarge": {Region: "ap-northeast-3", Type: "r3.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.4}, "r3.2xlarge": {Region: "ap-northeast-3", Type: "r3.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.8}, "r3.4xlarge": {Region: "ap-northeast-3", Type: "r3.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.6}, "r3.8xlarge": {Region: "ap-northeast-3", Type: "r3.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 3.2}, "r4.large": {Region: "ap-northeast-3", Type: "r4.large", Memory: kresource.MustParse("15616Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.16}, "r4.xlarge": {Region: "ap-northeast-3", Type: "r4.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.32}, "r4.2xlarge": {Region: "ap-northeast-3", Type: "r4.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.64}, "r4.4xlarge": {Region: "ap-northeast-3", Type: "r4.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.28}, "r4.8xlarge": {Region: "ap-northeast-3", Type: "r4.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.56}, "r4.16xlarge": {Region: "ap-northeast-3", Type: "r4.16xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.12}, "r5.large": {Region: "ap-northeast-3", Type: "r5.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.152}, "r5.xlarge": {Region: "ap-northeast-3", Type: "r5.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.304}, "r5.2xlarge": {Region: "ap-northeast-3", Type: "r5.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.608}, "r5.4xlarge": {Region: "ap-northeast-3", Type: "r5.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.216}, "r5.8xlarge": {Region: "ap-northeast-3", Type: "r5.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.432}, "r5.12xlarge": {Region: "ap-northeast-3", Type: "r5.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.648}, "r5.16xlarge": {Region: "ap-northeast-3", Type: "r5.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.864}, "r5.24xlarge": {Region: "ap-northeast-3", Type: "r5.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.296}, "r5.metal": {Region: "ap-northeast-3", Type: "r5.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.296}, "r5d.large": {Region: "ap-northeast-3", Type: "r5d.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.174}, "r5d.xlarge": {Region: "ap-northeast-3", Type: "r5d.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.348}, "r5d.2xlarge": {Region: "ap-northeast-3", Type: "r5d.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.696}, "r5d.4xlarge": {Region: "ap-northeast-3", Type: "r5d.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.392}, "r5d.8xlarge": {Region: "ap-northeast-3", Type: "r5d.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.784}, "r5d.12xlarge": {Region: "ap-northeast-3", Type: "r5d.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.176}, "r5d.16xlarge": {Region: "ap-northeast-3", Type: "r5d.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.568}, "r5d.24xlarge": {Region: "ap-northeast-3", Type: "r5d.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.352}, "r5d.metal": {Region: "ap-northeast-3", Type: "r5d.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.352}, "t2.nano": {Region: "ap-northeast-3", Type: "t2.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0076}, "t2.micro": {Region: "ap-northeast-3", Type: "t2.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0152}, "t2.small": {Region: "ap-northeast-3", Type: "t2.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0304}, "t2.medium": {Region: "ap-northeast-3", Type: "t2.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0608}, "t2.large": {Region: "ap-northeast-3", Type: "t2.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.1216}, "t2.xlarge": {Region: "ap-northeast-3", Type: "t2.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.2432}, "t2.2xlarge": {Region: "ap-northeast-3", Type: "t2.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.4864}, "t3.nano": {Region: "ap-northeast-3", Type: "t3.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0068}, "t3.micro": {Region: "ap-northeast-3", Type: "t3.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0136}, "t3.small": {Region: "ap-northeast-3", Type: "t3.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0272}, "t3.medium": {Region: "ap-northeast-3", Type: "t3.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0544}, "t3.large": {Region: "ap-northeast-3", Type: "t3.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.1088}, "t3.xlarge": {Region: "ap-northeast-3", Type: "t3.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.2176}, "t3.2xlarge": {Region: "ap-northeast-3", Type: "t3.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.4352}, "x1.16xlarge": {Region: "ap-northeast-3", Type: "x1.16xlarge", Memory: kresource.MustParse("999424Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 9.671}, "x1.32xlarge": {Region: "ap-northeast-3", Type: "x1.32xlarge", Memory: kresource.MustParse("1998848Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 19.342}, "x1e.xlarge": {Region: "ap-northeast-3", Type: "x1e.xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 1.209}, "x1e.2xlarge": {Region: "ap-northeast-3", Type: "x1e.2xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 2.418}, "x1e.4xlarge": {Region: "ap-northeast-3", Type: "x1e.4xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 4.836}, "x1e.8xlarge": {Region: "ap-northeast-3", Type: "x1e.8xlarge", Memory: kresource.MustParse("999424Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 9.672}, "x1e.16xlarge": {Region: "ap-northeast-3", Type: "x1e.16xlarge", Memory: kresource.MustParse("1998848Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 19.344}, "x1e.32xlarge": {Region: "ap-northeast-3", Type: "x1e.32xlarge", Memory: kresource.MustParse("3997696Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 38.688}, }, "ap-south-1": { "a1.medium": {Region: "ap-south-1", Type: "a1.medium", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0255}, "a1.large": {Region: "ap-south-1", Type: "a1.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.051}, "a1.xlarge": {Region: "ap-south-1", Type: "a1.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.102}, "a1.2xlarge": {Region: "ap-south-1", Type: "a1.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.204}, "a1.4xlarge": {Region: "ap-south-1", Type: "a1.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.408}, "a1.metal": {Region: "ap-south-1", Type: "a1.metal", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.408}, "c4.large": {Region: "ap-south-1", Type: "c4.large", Memory: kresource.MustParse("3840Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.1}, "c4.xlarge": {Region: "ap-south-1", Type: "c4.xlarge", Memory: kresource.MustParse("7680Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.2}, "c4.2xlarge": {Region: "ap-south-1", Type: "c4.2xlarge", Memory: kresource.MustParse("15360Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.4}, "c4.4xlarge": {Region: "ap-south-1", Type: "c4.4xlarge", Memory: kresource.MustParse("30720Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.8}, "c4.8xlarge": {Region: "ap-south-1", Type: "c4.8xlarge", Memory: kresource.MustParse("61440Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 1.6}, "c5.large": {Region: "ap-south-1", Type: "c5.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.085}, "c5.xlarge": {Region: "ap-south-1", Type: "c5.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.17}, "c5.2xlarge": {Region: "ap-south-1", Type: "c5.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.34}, "c5.4xlarge": {Region: "ap-south-1", Type: "c5.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.68}, "c5.9xlarge": {Region: "ap-south-1", Type: "c5.9xlarge", Memory: kresource.MustParse("73728Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 1.53}, "c5.12xlarge": {Region: "ap-south-1", Type: "c5.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.04}, "c5.18xlarge": {Region: "ap-south-1", Type: "c5.18xlarge", Memory: kresource.MustParse("147456Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 3.06}, "c5.24xlarge": {Region: "ap-south-1", Type: "c5.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.08}, "c5.metal": {Region: "ap-south-1", Type: "c5.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.08}, "c5a.large": {Region: "ap-south-1", Type: "c5a.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.047}, "c5a.xlarge": {Region: "ap-south-1", Type: "c5a.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.094}, "c5a.2xlarge": {Region: "ap-south-1", Type: "c5a.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.188}, "c5a.4xlarge": {Region: "ap-south-1", Type: "c5a.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.376}, "c5a.8xlarge": {Region: "ap-south-1", Type: "c5a.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 0.752}, "c5a.12xlarge": {Region: "ap-south-1", Type: "c5a.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 1.128}, "c5a.16xlarge": {Region: "ap-south-1", Type: "c5a.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 1.504}, "c5a.24xlarge": {Region: "ap-south-1", Type: "c5a.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 2.256}, "c5d.large": {Region: "ap-south-1", Type: "c5d.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.099}, "c5d.xlarge": {Region: "ap-south-1", Type: "c5d.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.198}, "c5d.2xlarge": {Region: "ap-south-1", Type: "c5d.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.396}, "c5d.4xlarge": {Region: "ap-south-1", Type: "c5d.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.792}, "c5d.9xlarge": {Region: "ap-south-1", Type: "c5d.9xlarge", Memory: kresource.MustParse("73728Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 1.782}, "c5d.12xlarge": {Region: "ap-south-1", Type: "c5d.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.376}, "c5d.18xlarge": {Region: "ap-south-1", Type: "c5d.18xlarge", Memory: kresource.MustParse("147456Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 3.564}, "c5d.24xlarge": {Region: "ap-south-1", Type: "c5d.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.752}, "c5d.metal": {Region: "ap-south-1", Type: "c5d.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.752}, "c5n.large": {Region: "ap-south-1", Type: "c5n.large", Memory: kresource.MustParse("5376Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.108}, "c5n.xlarge": {Region: "ap-south-1", Type: "c5n.xlarge", Memory: kresource.MustParse("10752Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.216}, "c5n.2xlarge": {Region: "ap-south-1", Type: "c5n.2xlarge", Memory: kresource.MustParse("21504Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.432}, "c5n.4xlarge": {Region: "ap-south-1", Type: "c5n.4xlarge", Memory: kresource.MustParse("43008Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.864}, "c5n.9xlarge": {Region: "ap-south-1", Type: "c5n.9xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 1.944}, "c5n.18xlarge": {Region: "ap-south-1", Type: "c5n.18xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 3.888}, "c5n.metal": {Region: "ap-south-1", Type: "c5n.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 3.888}, "c6g.medium": {Region: "ap-south-1", Type: "c6g.medium", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0213}, "c6g.large": {Region: "ap-south-1", Type: "c6g.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0426}, "c6g.xlarge": {Region: "ap-south-1", Type: "c6g.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.0852}, "c6g.2xlarge": {Region: "ap-south-1", Type: "c6g.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.1704}, "c6g.4xlarge": {Region: "ap-south-1", Type: "c6g.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.3408}, "c6g.8xlarge": {Region: "ap-south-1", Type: "c6g.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 0.6816}, "c6g.12xlarge": {Region: "ap-south-1", Type: "c6g.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 1.0224}, "c6g.16xlarge": {Region: "ap-south-1", Type: "c6g.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 1.3632}, "c6g.metal": {Region: "ap-south-1", Type: "c6g.metal", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 1.3632}, "c6gd.medium": {Region: "ap-south-1", Type: "c6gd.medium", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0245}, "c6gd.large": {Region: "ap-south-1", Type: "c6gd.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.049}, "c6gd.xlarge": {Region: "ap-south-1", Type: "c6gd.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.098}, "c6gd.2xlarge": {Region: "ap-south-1", Type: "c6gd.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.196}, "c6gd.4xlarge": {Region: "ap-south-1", Type: "c6gd.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.392}, "c6gd.8xlarge": {Region: "ap-south-1", Type: "c6gd.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 0.784}, "c6gd.12xlarge": {Region: "ap-south-1", Type: "c6gd.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 1.176}, "c6gd.16xlarge": {Region: "ap-south-1", Type: "c6gd.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 1.568}, "c6gd.metal": {Region: "ap-south-1", Type: "c6gd.metal", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 1.568}, "c6gn.medium": {Region: "ap-south-1", Type: "c6gn.medium", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.04325}, "c6gn.large": {Region: "ap-south-1", Type: "c6gn.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0865}, "c6gn.xlarge": {Region: "ap-south-1", Type: "c6gn.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.173}, "c6gn.2xlarge": {Region: "ap-south-1", Type: "c6gn.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.346}, "c6gn.4xlarge": {Region: "ap-south-1", Type: "c6gn.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.692}, "c6gn.8xlarge": {Region: "ap-south-1", Type: "c6gn.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.384}, "c6gn.12xlarge": {Region: "ap-south-1", Type: "c6gn.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.076}, "c6gn.16xlarge": {Region: "ap-south-1", Type: "c6gn.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.768}, "c6i.large": {Region: "ap-south-1", Type: "c6i.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.085}, "c6i.xlarge": {Region: "ap-south-1", Type: "c6i.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.17}, "c6i.2xlarge": {Region: "ap-south-1", Type: "c6i.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.34}, "c6i.4xlarge": {Region: "ap-south-1", Type: "c6i.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.68}, "c6i.8xlarge": {Region: "ap-south-1", Type: "c6i.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.36}, "c6i.12xlarge": {Region: "ap-south-1", Type: "c6i.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.04}, "c6i.16xlarge": {Region: "ap-south-1", Type: "c6i.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.72}, "c6i.24xlarge": {Region: "ap-south-1", Type: "c6i.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.08}, "c6i.32xlarge": {Region: "ap-south-1", Type: "c6i.32xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 5.44}, "c6i.metal": {Region: "ap-south-1", Type: "c6i.metal", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 5.44}, "d2.xlarge": {Region: "ap-south-1", Type: "d2.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.827}, "d2.2xlarge": {Region: "ap-south-1", Type: "d2.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.653}, "d2.4xlarge": {Region: "ap-south-1", Type: "d2.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 3.306}, "d2.8xlarge": {Region: "ap-south-1", Type: "d2.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 6.612}, "d3.xlarge": {Region: "ap-south-1", Type: "d3.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.524}, "d3.2xlarge": {Region: "ap-south-1", Type: "d3.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.047}, "d3.4xlarge": {Region: "ap-south-1", Type: "d3.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 2.095}, "d3.8xlarge": {Region: "ap-south-1", Type: "d3.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 4.18904}, "g4dn.xlarge": {Region: "ap-south-1", Type: "g4dn.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 1, Inf: 0, Price: 0.579}, "g4dn.2xlarge": {Region: "ap-south-1", Type: "g4dn.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 1, Inf: 0, Price: 0.828}, "g4dn.4xlarge": {Region: "ap-south-1", Type: "g4dn.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 1, Inf: 0, Price: 1.325}, "g4dn.8xlarge": {Region: "ap-south-1", Type: "g4dn.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 1, Inf: 0, Price: 2.395}, "g4dn.12xlarge": {Region: "ap-south-1", Type: "g4dn.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 4, Inf: 0, Price: 4.306}, "g4dn.16xlarge": {Region: "ap-south-1", Type: "g4dn.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 1, Inf: 0, Price: 4.791}, "g4dn.metal": {Region: "ap-south-1", Type: "g4dn.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 8, Inf: 0, Price: 8.612}, "i2.xlarge": {Region: "ap-south-1", Type: "i2.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.967}, "i2.2xlarge": {Region: "ap-south-1", Type: "i2.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.933}, "i2.4xlarge": {Region: "ap-south-1", Type: "i2.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 3.867}, "i2.8xlarge": {Region: "ap-south-1", Type: "i2.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 7.733}, "i3.large": {Region: "ap-south-1", Type: "i3.large", Memory: kresource.MustParse("15616Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.177}, "i3.xlarge": {Region: "ap-south-1", Type: "i3.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.354}, "i3.2xlarge": {Region: "ap-south-1", Type: "i3.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.708}, "i3.4xlarge": {Region: "ap-south-1", Type: "i3.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.416}, "i3.8xlarge": {Region: "ap-south-1", Type: "i3.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.832}, "i3.16xlarge": {Region: "ap-south-1", Type: "i3.16xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.664}, "i3.metal": {Region: "ap-south-1", Type: "i3.metal", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.664}, "i3en.large": {Region: "ap-south-1", Type: "i3en.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.257}, "i3en.xlarge": {Region: "ap-south-1", Type: "i3en.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.514}, "i3en.2xlarge": {Region: "ap-south-1", Type: "i3en.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.028}, "i3en.3xlarge": {Region: "ap-south-1", Type: "i3en.3xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("12"), GPU: 0, Inf: 0, Price: 1.542}, "i3en.6xlarge": {Region: "ap-south-1", Type: "i3en.6xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 0, Price: 3.084}, "i3en.12xlarge": {Region: "ap-south-1", Type: "i3en.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 6.168}, "i3en.24xlarge": {Region: "ap-south-1", Type: "i3en.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 12.336}, "i3en.metal": {Region: "ap-south-1", Type: "i3en.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 12.336}, "inf1.xlarge": {Region: "ap-south-1", Type: "inf1.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 1, Price: 0.24}, "inf1.2xlarge": {Region: "ap-south-1", Type: "inf1.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 1, Price: 0.381}, "inf1.6xlarge": {Region: "ap-south-1", Type: "inf1.6xlarge", Memory: kresource.MustParse("49152Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 4, Price: 1.241}, "inf1.24xlarge": {Region: "ap-south-1", Type: "inf1.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 16, Price: 4.965}, "m4.large": {Region: "ap-south-1", Type: "m4.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.105}, "m4.xlarge": {Region: "ap-south-1", Type: "m4.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.21}, "m4.2xlarge": {Region: "ap-south-1", Type: "m4.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.42}, "m4.4xlarge": {Region: "ap-south-1", Type: "m4.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.84}, "m4.10xlarge": {Region: "ap-south-1", Type: "m4.10xlarge", Memory: kresource.MustParse("163840Mi"), CPU: kresource.MustParse("40"), GPU: 0, Inf: 0, Price: 2.1}, "m4.16xlarge": {Region: "ap-south-1", Type: "m4.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.36}, "m5.large": {Region: "ap-south-1", Type: "m5.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.101}, "m5.xlarge": {Region: "ap-south-1", Type: "m5.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.202}, "m5.2xlarge": {Region: "ap-south-1", Type: "m5.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.404}, "m5.4xlarge": {Region: "ap-south-1", Type: "m5.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.808}, "m5.8xlarge": {Region: "ap-south-1", Type: "m5.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.616}, "m5.12xlarge": {Region: "ap-south-1", Type: "m5.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.424}, "m5.16xlarge": {Region: "ap-south-1", Type: "m5.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.232}, "m5.24xlarge": {Region: "ap-south-1", Type: "m5.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.848}, "m5.metal": {Region: "ap-south-1", Type: "m5.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.848}, "m5a.large": {Region: "ap-south-1", Type: "m5a.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.056}, "m5a.xlarge": {Region: "ap-south-1", Type: "m5a.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.111}, "m5a.2xlarge": {Region: "ap-south-1", Type: "m5a.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.222}, "m5a.4xlarge": {Region: "ap-south-1", Type: "m5a.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.444}, "m5a.8xlarge": {Region: "ap-south-1", Type: "m5a.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 0.889}, "m5a.12xlarge": {Region: "ap-south-1", Type: "m5a.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 1.333}, "m5a.16xlarge": {Region: "ap-south-1", Type: "m5a.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 1.778}, "m5a.24xlarge": {Region: "ap-south-1", Type: "m5a.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 2.666}, "m5ad.large": {Region: "ap-south-1", Type: "m5ad.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.067}, "m5ad.xlarge": {Region: "ap-south-1", Type: "m5ad.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.134}, "m5ad.2xlarge": {Region: "ap-south-1", Type: "m5ad.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.268}, "m5ad.4xlarge": {Region: "ap-south-1", Type: "m5ad.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.537}, "m5ad.8xlarge": {Region: "ap-south-1", Type: "m5ad.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.074}, "m5ad.12xlarge": {Region: "ap-south-1", Type: "m5ad.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 1.61}, "m5ad.16xlarge": {Region: "ap-south-1", Type: "m5ad.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.147}, "m5ad.24xlarge": {Region: "ap-south-1", Type: "m5ad.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 3.221}, "m5d.large": {Region: "ap-south-1", Type: "m5d.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.122}, "m5d.xlarge": {Region: "ap-south-1", Type: "m5d.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.244}, "m5d.2xlarge": {Region: "ap-south-1", Type: "m5d.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.488}, "m5d.4xlarge": {Region: "ap-south-1", Type: "m5d.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.976}, "m5d.8xlarge": {Region: "ap-south-1", Type: "m5d.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.952}, "m5d.12xlarge": {Region: "ap-south-1", Type: "m5d.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.928}, "m5d.16xlarge": {Region: "ap-south-1", Type: "m5d.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.904}, "m5d.24xlarge": {Region: "ap-south-1", Type: "m5d.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.856}, "m5d.metal": {Region: "ap-south-1", Type: "m5d.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.856}, "m6a.large": {Region: "ap-south-1", Type: "m6a.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.05555}, "m6a.xlarge": {Region: "ap-south-1", Type: "m6a.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1111}, "m6a.2xlarge": {Region: "ap-south-1", Type: "m6a.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.2222}, "m6a.4xlarge": {Region: "ap-south-1", Type: "m6a.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.4444}, "m6a.8xlarge": {Region: "ap-south-1", Type: "m6a.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 0.8888}, "m6a.12xlarge": {Region: "ap-south-1", Type: "m6a.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 1.3332}, "m6a.16xlarge": {Region: "ap-south-1", Type: "m6a.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 1.7776}, "m6a.24xlarge": {Region: "ap-south-1", Type: "m6a.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 2.6664}, "m6a.32xlarge": {Region: "ap-south-1", Type: "m6a.32xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 3.5552}, "m6a.48xlarge": {Region: "ap-south-1", Type: "m6a.48xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("192"), GPU: 0, Inf: 0, Price: 5.3328}, "m6a.metal": {Region: "ap-south-1", Type: "m6a.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("192"), GPU: 0, Inf: 0, Price: 5.3328}, "m6g.medium": {Region: "ap-south-1", Type: "m6g.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0253}, "m6g.large": {Region: "ap-south-1", Type: "m6g.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0506}, "m6g.xlarge": {Region: "ap-south-1", Type: "m6g.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1012}, "m6g.2xlarge": {Region: "ap-south-1", Type: "m6g.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.2024}, "m6g.4xlarge": {Region: "ap-south-1", Type: "m6g.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.4048}, "m6g.8xlarge": {Region: "ap-south-1", Type: "m6g.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 0.8096}, "m6g.12xlarge": {Region: "ap-south-1", Type: "m6g.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 1.2144}, "m6g.16xlarge": {Region: "ap-south-1", Type: "m6g.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 1.6192}, "m6g.metal": {Region: "ap-south-1", Type: "m6g.metal", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 1.6192}, "m6gd.medium": {Region: "ap-south-1", Type: "m6gd.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0302}, "m6gd.large": {Region: "ap-south-1", Type: "m6gd.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0604}, "m6gd.xlarge": {Region: "ap-south-1", Type: "m6gd.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1208}, "m6gd.2xlarge": {Region: "ap-south-1", Type: "m6gd.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.2416}, "m6gd.4xlarge": {Region: "ap-south-1", Type: "m6gd.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.4832}, "m6gd.8xlarge": {Region: "ap-south-1", Type: "m6gd.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 0.9664}, "m6gd.12xlarge": {Region: "ap-south-1", Type: "m6gd.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 1.4496}, "m6gd.16xlarge": {Region: "ap-south-1", Type: "m6gd.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 1.9328}, "m6gd.metal": {Region: "ap-south-1", Type: "m6gd.metal", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 1.9328}, "m6i.large": {Region: "ap-south-1", Type: "m6i.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.101}, "m6i.xlarge": {Region: "ap-south-1", Type: "m6i.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.202}, "m6i.2xlarge": {Region: "ap-south-1", Type: "m6i.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.404}, "m6i.4xlarge": {Region: "ap-south-1", Type: "m6i.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.808}, "m6i.8xlarge": {Region: "ap-south-1", Type: "m6i.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.616}, "m6i.12xlarge": {Region: "ap-south-1", Type: "m6i.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.424}, "m6i.16xlarge": {Region: "ap-south-1", Type: "m6i.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.232}, "m6i.24xlarge": {Region: "ap-south-1", Type: "m6i.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.848}, "m6i.32xlarge": {Region: "ap-south-1", Type: "m6i.32xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 6.464}, "m6i.metal": {Region: "ap-south-1", Type: "m6i.metal", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 6.464}, "p2.xlarge": {Region: "ap-south-1", Type: "p2.xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("4"), GPU: 1, Inf: 0, Price: 1.718}, "p2.8xlarge": {Region: "ap-south-1", Type: "p2.8xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("32"), GPU: 8, Inf: 0, Price: 13.744}, "p2.16xlarge": {Region: "ap-south-1", Type: "p2.16xlarge", Memory: kresource.MustParse("749568Mi"), CPU: kresource.MustParse("64"), GPU: 16, Inf: 0, Price: 27.488}, "r3.large": {Region: "ap-south-1", Type: "r3.large", Memory: kresource.MustParse("15616Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.19}, "r3.xlarge": {Region: "ap-south-1", Type: "r3.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.379}, "r3.2xlarge": {Region: "ap-south-1", Type: "r3.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.758}, "r3.4xlarge": {Region: "ap-south-1", Type: "r3.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.516}, "r3.8xlarge": {Region: "ap-south-1", Type: "r3.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 3.032}, "r4.large": {Region: "ap-south-1", Type: "r4.large", Memory: kresource.MustParse("15616Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.137}, "r4.xlarge": {Region: "ap-south-1", Type: "r4.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.274}, "r4.2xlarge": {Region: "ap-south-1", Type: "r4.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.548}, "r4.4xlarge": {Region: "ap-south-1", Type: "r4.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.096}, "r4.8xlarge": {Region: "ap-south-1", Type: "r4.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.192}, "r4.16xlarge": {Region: "ap-south-1", Type: "r4.16xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.384}, "r5.large": {Region: "ap-south-1", Type: "r5.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.13}, "r5.xlarge": {Region: "ap-south-1", Type: "r5.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.26}, "r5.2xlarge": {Region: "ap-south-1", Type: "r5.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.52}, "r5.4xlarge": {Region: "ap-south-1", Type: "r5.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.04}, "r5.8xlarge": {Region: "ap-south-1", Type: "r5.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.08}, "r5.12xlarge": {Region: "ap-south-1", Type: "r5.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.12}, "r5.16xlarge": {Region: "ap-south-1", Type: "r5.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.16}, "r5.24xlarge": {Region: "ap-south-1", Type: "r5.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.24}, "r5.metal": {Region: "ap-south-1", Type: "r5.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.24}, "r5a.large": {Region: "ap-south-1", Type: "r5a.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.072}, "r5a.xlarge": {Region: "ap-south-1", Type: "r5a.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.143}, "r5a.2xlarge": {Region: "ap-south-1", Type: "r5a.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.286}, "r5a.4xlarge": {Region: "ap-south-1", Type: "r5a.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.572}, "r5a.8xlarge": {Region: "ap-south-1", Type: "r5a.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.144}, "r5a.12xlarge": {Region: "ap-south-1", Type: "r5a.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 1.716}, "r5a.16xlarge": {Region: "ap-south-1", Type: "r5a.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.288}, "r5a.24xlarge": {Region: "ap-south-1", Type: "r5a.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 3.432}, "r5ad.large": {Region: "ap-south-1", Type: "r5ad.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.083}, "r5ad.xlarge": {Region: "ap-south-1", Type: "r5ad.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.166}, "r5ad.2xlarge": {Region: "ap-south-1", Type: "r5ad.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.332}, "r5ad.4xlarge": {Region: "ap-south-1", Type: "r5ad.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.664}, "r5ad.8xlarge": {Region: "ap-south-1", Type: "r5ad.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.329}, "r5ad.12xlarge": {Region: "ap-south-1", Type: "r5ad.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 1.993}, "r5ad.16xlarge": {Region: "ap-south-1", Type: "r5ad.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.658}, "r5ad.24xlarge": {Region: "ap-south-1", Type: "r5ad.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 3.986}, "r5d.large": {Region: "ap-south-1", Type: "r5d.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.151}, "r5d.xlarge": {Region: "ap-south-1", Type: "r5d.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.302}, "r5d.2xlarge": {Region: "ap-south-1", Type: "r5d.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.604}, "r5d.4xlarge": {Region: "ap-south-1", Type: "r5d.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.208}, "r5d.8xlarge": {Region: "ap-south-1", Type: "r5d.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.416}, "r5d.12xlarge": {Region: "ap-south-1", Type: "r5d.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.624}, "r5d.16xlarge": {Region: "ap-south-1", Type: "r5d.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.832}, "r5d.24xlarge": {Region: "ap-south-1", Type: "r5d.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.248}, "r5d.metal": {Region: "ap-south-1", Type: "r5d.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.248}, "r5n.large": {Region: "ap-south-1", Type: "r5n.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.153}, "r5n.xlarge": {Region: "ap-south-1", Type: "r5n.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.306}, "r5n.2xlarge": {Region: "ap-south-1", Type: "r5n.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.612}, "r5n.4xlarge": {Region: "ap-south-1", Type: "r5n.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.224}, "r5n.8xlarge": {Region: "ap-south-1", Type: "r5n.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.448}, "r5n.12xlarge": {Region: "ap-south-1", Type: "r5n.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.672}, "r5n.16xlarge": {Region: "ap-south-1", Type: "r5n.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.896}, "r5n.24xlarge": {Region: "ap-south-1", Type: "r5n.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.344}, "r5n.metal": {Region: "ap-south-1", Type: "r5n.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.344}, "r6g.medium": {Region: "ap-south-1", Type: "r6g.medium", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0325}, "r6g.large": {Region: "ap-south-1", Type: "r6g.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.065}, "r6g.xlarge": {Region: "ap-south-1", Type: "r6g.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.13}, "r6g.2xlarge": {Region: "ap-south-1", Type: "r6g.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.26}, "r6g.4xlarge": {Region: "ap-south-1", Type: "r6g.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.52}, "r6g.8xlarge": {Region: "ap-south-1", Type: "r6g.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.04}, "r6g.12xlarge": {Region: "ap-south-1", Type: "r6g.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 1.56}, "r6g.16xlarge": {Region: "ap-south-1", Type: "r6g.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.08}, "r6g.metal": {Region: "ap-south-1", Type: "r6g.metal", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.08}, "r6gd.medium": {Region: "ap-south-1", Type: "r6gd.medium", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0374}, "r6gd.large": {Region: "ap-south-1", Type: "r6gd.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0747}, "r6gd.xlarge": {Region: "ap-south-1", Type: "r6gd.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1495}, "r6gd.2xlarge": {Region: "ap-south-1", Type: "r6gd.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.299}, "r6gd.4xlarge": {Region: "ap-south-1", Type: "r6gd.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.5979}, "r6gd.8xlarge": {Region: "ap-south-1", Type: "r6gd.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.1958}, "r6gd.12xlarge": {Region: "ap-south-1", Type: "r6gd.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 1.7938}, "r6gd.16xlarge": {Region: "ap-south-1", Type: "r6gd.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.3917}, "r6gd.metal": {Region: "ap-south-1", Type: "r6gd.metal", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.3917}, "r6i.large": {Region: "ap-south-1", Type: "r6i.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.13}, "r6i.xlarge": {Region: "ap-south-1", Type: "r6i.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.26}, "r6i.2xlarge": {Region: "ap-south-1", Type: "r6i.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.52}, "r6i.4xlarge": {Region: "ap-south-1", Type: "r6i.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.04}, "r6i.8xlarge": {Region: "ap-south-1", Type: "r6i.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.08}, "r6i.12xlarge": {Region: "ap-south-1", Type: "r6i.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.12}, "r6i.16xlarge": {Region: "ap-south-1", Type: "r6i.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.16}, "r6i.24xlarge": {Region: "ap-south-1", Type: "r6i.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.24}, "r6i.32xlarge": {Region: "ap-south-1", Type: "r6i.32xlarge", Memory: kresource.MustParse("1048576Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 8.32}, "r6i.metal": {Region: "ap-south-1", Type: "r6i.metal", Memory: kresource.MustParse("1048576Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 8.32}, "t2.nano": {Region: "ap-south-1", Type: "t2.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0062}, "t2.micro": {Region: "ap-south-1", Type: "t2.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0124}, "t2.small": {Region: "ap-south-1", Type: "t2.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0248}, "t2.medium": {Region: "ap-south-1", Type: "t2.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0496}, "t2.large": {Region: "ap-south-1", Type: "t2.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0992}, "t2.xlarge": {Region: "ap-south-1", Type: "t2.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1984}, "t2.2xlarge": {Region: "ap-south-1", Type: "t2.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3968}, "t3.nano": {Region: "ap-south-1", Type: "t3.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0056}, "t3.micro": {Region: "ap-south-1", Type: "t3.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0112}, "t3.small": {Region: "ap-south-1", Type: "t3.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0224}, "t3.medium": {Region: "ap-south-1", Type: "t3.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0448}, "t3.large": {Region: "ap-south-1", Type: "t3.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0896}, "t3.xlarge": {Region: "ap-south-1", Type: "t3.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1792}, "t3.2xlarge": {Region: "ap-south-1", Type: "t3.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3584}, "t3a.nano": {Region: "ap-south-1", Type: "t3a.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0031}, "t3a.micro": {Region: "ap-south-1", Type: "t3a.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0062}, "t3a.small": {Region: "ap-south-1", Type: "t3a.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0123}, "t3a.medium": {Region: "ap-south-1", Type: "t3a.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0246}, "t3a.large": {Region: "ap-south-1", Type: "t3a.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0493}, "t3a.xlarge": {Region: "ap-south-1", Type: "t3a.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.0986}, "t3a.2xlarge": {Region: "ap-south-1", Type: "t3a.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.1971}, "t4g.nano": {Region: "ap-south-1", Type: "t4g.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0028}, "t4g.micro": {Region: "ap-south-1", Type: "t4g.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0056}, "t4g.small": {Region: "ap-south-1", Type: "t4g.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0112}, "t4g.medium": {Region: "ap-south-1", Type: "t4g.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0224}, "t4g.large": {Region: "ap-south-1", Type: "t4g.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0448}, "t4g.xlarge": {Region: "ap-south-1", Type: "t4g.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.0896}, "t4g.2xlarge": {Region: "ap-south-1", Type: "t4g.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.1792}, "u-6tb1.56xlarge": {Region: "ap-south-1", Type: "u-6tb1.56xlarge", Memory: kresource.MustParse("6291456Mi"), CPU: kresource.MustParse("224"), GPU: 0, Inf: 0, Price: 47.87676}, "u-6tb1.112xlarge": {Region: "ap-south-1", Type: "u-6tb1.112xlarge", Memory: kresource.MustParse("6291456Mi"), CPU: kresource.MustParse("448"), GPU: 0, Inf: 0, Price: 56.333}, "x1.16xlarge": {Region: "ap-south-1", Type: "x1.16xlarge", Memory: kresource.MustParse("999424Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 6.881}, "x1.32xlarge": {Region: "ap-south-1", Type: "x1.32xlarge", Memory: kresource.MustParse("1998848Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 13.762}, "x1e.xlarge": {Region: "ap-south-1", Type: "x1e.xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.86}, "x1e.2xlarge": {Region: "ap-south-1", Type: "x1e.2xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.72}, "x1e.4xlarge": {Region: "ap-south-1", Type: "x1e.4xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 3.44}, "x1e.8xlarge": {Region: "ap-south-1", Type: "x1e.8xlarge", Memory: kresource.MustParse("999424Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 6.88}, "x1e.16xlarge": {Region: "ap-south-1", Type: "x1e.16xlarge", Memory: kresource.MustParse("1998848Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 13.76}, "x1e.32xlarge": {Region: "ap-south-1", Type: "x1e.32xlarge", Memory: kresource.MustParse("3997696Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 27.52}, "x2idn.16xlarge": {Region: "ap-south-1", Type: "x2idn.16xlarge", Memory: kresource.MustParse("1048576Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 6.881}, "x2idn.24xlarge": {Region: "ap-south-1", Type: "x2idn.24xlarge", Memory: kresource.MustParse("1572864Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 10.3215}, "x2idn.32xlarge": {Region: "ap-south-1", Type: "x2idn.32xlarge", Memory: kresource.MustParse("2097152Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 13.762}, "z1d.large": {Region: "ap-south-1", Type: "z1d.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.196}, "z1d.xlarge": {Region: "ap-south-1", Type: "z1d.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.392}, "z1d.2xlarge": {Region: "ap-south-1", Type: "z1d.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.784}, "z1d.3xlarge": {Region: "ap-south-1", Type: "z1d.3xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("12"), GPU: 0, Inf: 0, Price: 1.176}, "z1d.6xlarge": {Region: "ap-south-1", Type: "z1d.6xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 0, Price: 2.352}, "z1d.12xlarge": {Region: "ap-south-1", Type: "z1d.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.704}, "z1d.metal": {Region: "ap-south-1", Type: "z1d.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.704}, }, "ap-southeast-1": { "a1.medium": {Region: "ap-southeast-1", Type: "a1.medium", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0294}, "a1.large": {Region: "ap-southeast-1", Type: "a1.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0588}, "a1.xlarge": {Region: "ap-southeast-1", Type: "a1.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1176}, "a1.2xlarge": {Region: "ap-southeast-1", Type: "a1.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.2352}, "a1.4xlarge": {Region: "ap-southeast-1", Type: "a1.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.4704}, "a1.metal": {Region: "ap-southeast-1", Type: "a1.metal", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.47}, "c1.medium": {Region: "ap-southeast-1", Type: "c1.medium", Memory: kresource.MustParse("1740Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.164}, "c1.xlarge": {Region: "ap-southeast-1", Type: "c1.xlarge", Memory: kresource.MustParse("7168Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.655}, "c3.large": {Region: "ap-southeast-1", Type: "c3.large", Memory: kresource.MustParse("3840Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.132}, "c3.xlarge": {Region: "ap-southeast-1", Type: "c3.xlarge", Memory: kresource.MustParse("7680Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.265}, "c3.2xlarge": {Region: "ap-southeast-1", Type: "c3.2xlarge", Memory: kresource.MustParse("15360Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.529}, "c3.4xlarge": {Region: "ap-southeast-1", Type: "c3.4xlarge", Memory: kresource.MustParse("30720Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.058}, "c3.8xlarge": {Region: "ap-southeast-1", Type: "c3.8xlarge", Memory: kresource.MustParse("61440Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.117}, "c4.large": {Region: "ap-southeast-1", Type: "c4.large", Memory: kresource.MustParse("3840Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.115}, "c4.xlarge": {Region: "ap-southeast-1", Type: "c4.xlarge", Memory: kresource.MustParse("7680Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.231}, "c4.2xlarge": {Region: "ap-southeast-1", Type: "c4.2xlarge", Memory: kresource.MustParse("15360Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.462}, "c4.4xlarge": {Region: "ap-southeast-1", Type: "c4.4xlarge", Memory: kresource.MustParse("30720Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.924}, "c4.8xlarge": {Region: "ap-southeast-1", Type: "c4.8xlarge", Memory: kresource.MustParse("61440Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 1.848}, "c5.large": {Region: "ap-southeast-1", Type: "c5.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.098}, "c5.xlarge": {Region: "ap-southeast-1", Type: "c5.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.196}, "c5.2xlarge": {Region: "ap-southeast-1", Type: "c5.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.392}, "c5.4xlarge": {Region: "ap-southeast-1", Type: "c5.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.784}, "c5.9xlarge": {Region: "ap-southeast-1", Type: "c5.9xlarge", Memory: kresource.MustParse("73728Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 1.764}, "c5.12xlarge": {Region: "ap-southeast-1", Type: "c5.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.352}, "c5.18xlarge": {Region: "ap-southeast-1", Type: "c5.18xlarge", Memory: kresource.MustParse("147456Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 3.528}, "c5.24xlarge": {Region: "ap-southeast-1", Type: "c5.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.704}, "c5.metal": {Region: "ap-southeast-1", Type: "c5.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.704}, "c5a.large": {Region: "ap-southeast-1", Type: "c5a.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.088}, "c5a.xlarge": {Region: "ap-southeast-1", Type: "c5a.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.176}, "c5a.2xlarge": {Region: "ap-southeast-1", Type: "c5a.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.352}, "c5a.4xlarge": {Region: "ap-southeast-1", Type: "c5a.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.704}, "c5a.8xlarge": {Region: "ap-southeast-1", Type: "c5a.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.408}, "c5a.12xlarge": {Region: "ap-southeast-1", Type: "c5a.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.112}, "c5a.16xlarge": {Region: "ap-southeast-1", Type: "c5a.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.816}, "c5a.24xlarge": {Region: "ap-southeast-1", Type: "c5a.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.224}, "c5ad.large": {Region: "ap-southeast-1", Type: "c5ad.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.101}, "c5ad.xlarge": {Region: "ap-southeast-1", Type: "c5ad.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.202}, "c5ad.2xlarge": {Region: "ap-southeast-1", Type: "c5ad.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.404}, "c5ad.4xlarge": {Region: "ap-southeast-1", Type: "c5ad.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.808}, "c5ad.8xlarge": {Region: "ap-southeast-1", Type: "c5ad.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.616}, "c5ad.12xlarge": {Region: "ap-southeast-1", Type: "c5ad.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.424}, "c5ad.16xlarge": {Region: "ap-southeast-1", Type: "c5ad.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.232}, "c5ad.24xlarge": {Region: "ap-southeast-1", Type: "c5ad.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.848}, "c5d.large": {Region: "ap-southeast-1", Type: "c5d.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.112}, "c5d.xlarge": {Region: "ap-southeast-1", Type: "c5d.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.224}, "c5d.2xlarge": {Region: "ap-southeast-1", Type: "c5d.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.448}, "c5d.4xlarge": {Region: "ap-southeast-1", Type: "c5d.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.896}, "c5d.9xlarge": {Region: "ap-southeast-1", Type: "c5d.9xlarge", Memory: kresource.MustParse("73728Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 2.016}, "c5d.12xlarge": {Region: "ap-southeast-1", Type: "c5d.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.688}, "c5d.18xlarge": {Region: "ap-southeast-1", Type: "c5d.18xlarge", Memory: kresource.MustParse("147456Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 4.032}, "c5d.24xlarge": {Region: "ap-southeast-1", Type: "c5d.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.376}, "c5d.metal": {Region: "ap-southeast-1", Type: "c5d.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.376}, "c5n.large": {Region: "ap-southeast-1", Type: "c5n.large", Memory: kresource.MustParse("5376Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.124}, "c5n.xlarge": {Region: "ap-southeast-1", Type: "c5n.xlarge", Memory: kresource.MustParse("10752Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.248}, "c5n.2xlarge": {Region: "ap-southeast-1", Type: "c5n.2xlarge", Memory: kresource.MustParse("21504Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.496}, "c5n.4xlarge": {Region: "ap-southeast-1", Type: "c5n.4xlarge", Memory: kresource.MustParse("43008Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.992}, "c5n.9xlarge": {Region: "ap-southeast-1", Type: "c5n.9xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 2.232}, "c5n.18xlarge": {Region: "ap-southeast-1", Type: "c5n.18xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 4.464}, "c5n.metal": {Region: "ap-southeast-1", Type: "c5n.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 4.464}, "c6g.medium": {Region: "ap-southeast-1", Type: "c6g.medium", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0392}, "c6g.large": {Region: "ap-southeast-1", Type: "c6g.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0784}, "c6g.xlarge": {Region: "ap-southeast-1", Type: "c6g.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1568}, "c6g.2xlarge": {Region: "ap-southeast-1", Type: "c6g.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3136}, "c6g.4xlarge": {Region: "ap-southeast-1", Type: "c6g.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.6272}, "c6g.8xlarge": {Region: "ap-southeast-1", Type: "c6g.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.2544}, "c6g.12xlarge": {Region: "ap-southeast-1", Type: "c6g.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 1.8816}, "c6g.16xlarge": {Region: "ap-southeast-1", Type: "c6g.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.5088}, "c6g.metal": {Region: "ap-southeast-1", Type: "c6g.metal", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.5088}, "c6gd.medium": {Region: "ap-southeast-1", Type: "c6gd.medium", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.045}, "c6gd.large": {Region: "ap-southeast-1", Type: "c6gd.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.09}, "c6gd.xlarge": {Region: "ap-southeast-1", Type: "c6gd.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.18}, "c6gd.2xlarge": {Region: "ap-southeast-1", Type: "c6gd.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.36}, "c6gd.4xlarge": {Region: "ap-southeast-1", Type: "c6gd.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.72}, "c6gd.8xlarge": {Region: "ap-southeast-1", Type: "c6gd.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.44}, "c6gd.12xlarge": {Region: "ap-southeast-1", Type: "c6gd.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.16}, "c6gd.16xlarge": {Region: "ap-southeast-1", Type: "c6gd.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.88}, "c6gd.metal": {Region: "ap-southeast-1", Type: "c6gd.metal", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.88}, "c6gn.medium": {Region: "ap-southeast-1", Type: "c6gn.medium", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0495}, "c6gn.large": {Region: "ap-southeast-1", Type: "c6gn.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.099}, "c6gn.xlarge": {Region: "ap-southeast-1", Type: "c6gn.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.198}, "c6gn.2xlarge": {Region: "ap-southeast-1", Type: "c6gn.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.396}, "c6gn.4xlarge": {Region: "ap-southeast-1", Type: "c6gn.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.792}, "c6gn.8xlarge": {Region: "ap-southeast-1", Type: "c6gn.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.584}, "c6gn.12xlarge": {Region: "ap-southeast-1", Type: "c6gn.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.376}, "c6gn.16xlarge": {Region: "ap-southeast-1", Type: "c6gn.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.168}, "c6i.large": {Region: "ap-southeast-1", Type: "c6i.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.098}, "c6i.xlarge": {Region: "ap-southeast-1", Type: "c6i.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.196}, "c6i.2xlarge": {Region: "ap-southeast-1", Type: "c6i.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.392}, "c6i.4xlarge": {Region: "ap-southeast-1", Type: "c6i.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.784}, "c6i.8xlarge": {Region: "ap-southeast-1", Type: "c6i.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.568}, "c6i.12xlarge": {Region: "ap-southeast-1", Type: "c6i.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.352}, "c6i.16xlarge": {Region: "ap-southeast-1", Type: "c6i.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.136}, "c6i.24xlarge": {Region: "ap-southeast-1", Type: "c6i.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.704}, "c6i.32xlarge": {Region: "ap-southeast-1", Type: "c6i.32xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 6.272}, "c6i.metal": {Region: "ap-southeast-1", Type: "c6i.metal", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 6.272}, "d2.xlarge": {Region: "ap-southeast-1", Type: "d2.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.87}, "d2.2xlarge": {Region: "ap-southeast-1", Type: "d2.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.74}, "d2.4xlarge": {Region: "ap-southeast-1", Type: "d2.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 3.48}, "d2.8xlarge": {Region: "ap-southeast-1", Type: "d2.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 6.96}, "d3.xlarge": {Region: "ap-southeast-1", Type: "d3.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.626}, "d3.2xlarge": {Region: "ap-southeast-1", Type: "d3.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.252}, "d3.4xlarge": {Region: "ap-southeast-1", Type: "d3.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 2.504}, "d3.8xlarge": {Region: "ap-southeast-1", Type: "d3.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 5.00808}, "g2.2xlarge": {Region: "ap-southeast-1", Type: "g2.2xlarge", Memory: kresource.MustParse("15360Mi"), CPU: kresource.MustParse("8"), GPU: 1, Inf: 0, Price: 1.0}, "g2.8xlarge": {Region: "ap-southeast-1", Type: "g2.8xlarge", Memory: kresource.MustParse("61440Mi"), CPU: kresource.MustParse("32"), GPU: 4, Inf: 0, Price: 4.0}, "g3.4xlarge": {Region: "ap-southeast-1", Type: "g3.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 1, Inf: 0, Price: 1.67}, "g3.8xlarge": {Region: "ap-southeast-1", Type: "g3.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 2, Inf: 0, Price: 3.34}, "g3.16xlarge": {Region: "ap-southeast-1", Type: "g3.16xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("64"), GPU: 4, Inf: 0, Price: 6.68}, "g4dn.xlarge": {Region: "ap-southeast-1", Type: "g4dn.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 1, Inf: 0, Price: 0.736}, "g4dn.2xlarge": {Region: "ap-southeast-1", Type: "g4dn.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 1, Inf: 0, Price: 1.052}, "g4dn.4xlarge": {Region: "ap-southeast-1", Type: "g4dn.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 1, Inf: 0, Price: 1.685}, "g4dn.8xlarge": {Region: "ap-southeast-1", Type: "g4dn.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 1, Inf: 0, Price: 3.045}, "g4dn.12xlarge": {Region: "ap-southeast-1", Type: "g4dn.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 4, Inf: 0, Price: 5.474}, "g4dn.16xlarge": {Region: "ap-southeast-1", Type: "g4dn.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 1, Inf: 0, Price: 6.089}, "g4dn.metal": {Region: "ap-southeast-1", Type: "g4dn.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 8, Inf: 0, Price: 10.948}, "g5g.xlarge": {Region: "ap-southeast-1", Type: "g5g.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 1, Inf: 0, Price: 0.5877}, "g5g.2xlarge": {Region: "ap-southeast-1", Type: "g5g.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 1, Inf: 0, Price: 0.778}, "g5g.4xlarge": {Region: "ap-southeast-1", Type: "g5g.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 1, Inf: 0, Price: 1.1586}, "g5g.8xlarge": {Region: "ap-southeast-1", Type: "g5g.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 1, Inf: 0, Price: 1.9198}, "g5g.16xlarge": {Region: "ap-southeast-1", Type: "g5g.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 2, Inf: 0, Price: 3.8395}, "g5g.metal": {Region: "ap-southeast-1", Type: "g5g.metal", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 2, Inf: 0, Price: 3.8395}, "hs1.8xlarge": {Region: "ap-southeast-1", Type: "hs1.8xlarge", Memory: kresource.MustParse("119808Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 5.57}, "i2.xlarge": {Region: "ap-southeast-1", Type: "i2.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 1.018}, "i2.2xlarge": {Region: "ap-southeast-1", Type: "i2.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 2.035}, "i2.4xlarge": {Region: "ap-southeast-1", Type: "i2.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 4.07}, "i2.8xlarge": {Region: "ap-southeast-1", Type: "i2.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 8.14}, "i3.large": {Region: "ap-southeast-1", Type: "i3.large", Memory: kresource.MustParse("15616Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.187}, "i3.xlarge": {Region: "ap-southeast-1", Type: "i3.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.374}, "i3.2xlarge": {Region: "ap-southeast-1", Type: "i3.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.748}, "i3.4xlarge": {Region: "ap-southeast-1", Type: "i3.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.496}, "i3.8xlarge": {Region: "ap-southeast-1", Type: "i3.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.992}, "i3.16xlarge": {Region: "ap-southeast-1", Type: "i3.16xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.984}, "i3.metal": {Region: "ap-southeast-1", Type: "i3.metal", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.984}, "i3en.large": {Region: "ap-southeast-1", Type: "i3en.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.271}, "i3en.xlarge": {Region: "ap-southeast-1", Type: "i3en.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.542}, "i3en.2xlarge": {Region: "ap-southeast-1", Type: "i3en.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.084}, "i3en.3xlarge": {Region: "ap-southeast-1", Type: "i3en.3xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("12"), GPU: 0, Inf: 0, Price: 1.626}, "i3en.6xlarge": {Region: "ap-southeast-1", Type: "i3en.6xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 0, Price: 3.252}, "i3en.12xlarge": {Region: "ap-southeast-1", Type: "i3en.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 6.504}, "i3en.24xlarge": {Region: "ap-southeast-1", Type: "i3en.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 13.008}, "i3en.metal": {Region: "ap-southeast-1", Type: "i3en.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 13.008}, "inf1.xlarge": {Region: "ap-southeast-1", Type: "inf1.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 1, Price: 0.308}, "inf1.2xlarge": {Region: "ap-southeast-1", Type: "inf1.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 1, Price: 0.489}, "inf1.6xlarge": {Region: "ap-southeast-1", Type: "inf1.6xlarge", Memory: kresource.MustParse("49152Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 4, Price: 1.594}, "inf1.24xlarge": {Region: "ap-southeast-1", Type: "inf1.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 16, Price: 6.376}, "m1.small": {Region: "ap-southeast-1", Type: "m1.small", Memory: kresource.MustParse("1740Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.058}, "m1.medium": {Region: "ap-southeast-1", Type: "m1.medium", Memory: kresource.MustParse("3840Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.117}, "m1.large": {Region: "ap-southeast-1", Type: "m1.large", Memory: kresource.MustParse("7680Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.233}, "m1.xlarge": {Region: "ap-southeast-1", Type: "m1.xlarge", Memory: kresource.MustParse("15360Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.467}, "m2.xlarge": {Region: "ap-southeast-1", Type: "m2.xlarge", Memory: kresource.MustParse("17510Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.296}, "m2.2xlarge": {Region: "ap-southeast-1", Type: "m2.2xlarge", Memory: kresource.MustParse("35020Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.592}, "m2.4xlarge": {Region: "ap-southeast-1", Type: "m2.4xlarge", Memory: kresource.MustParse("70041Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.183}, "m3.medium": {Region: "ap-southeast-1", Type: "m3.medium", Memory: kresource.MustParse("3840Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.098}, "m3.large": {Region: "ap-southeast-1", Type: "m3.large", Memory: kresource.MustParse("7680Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.196}, "m3.xlarge": {Region: "ap-southeast-1", Type: "m3.xlarge", Memory: kresource.MustParse("15360Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.392}, "m3.2xlarge": {Region: "ap-southeast-1", Type: "m3.2xlarge", Memory: kresource.MustParse("30720Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.784}, "m4.large": {Region: "ap-southeast-1", Type: "m4.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.125}, "m4.xlarge": {Region: "ap-southeast-1", Type: "m4.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.25}, "m4.2xlarge": {Region: "ap-southeast-1", Type: "m4.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.5}, "m4.4xlarge": {Region: "ap-southeast-1", Type: "m4.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.0}, "m4.10xlarge": {Region: "ap-southeast-1", Type: "m4.10xlarge", Memory: kresource.MustParse("163840Mi"), CPU: kresource.MustParse("40"), GPU: 0, Inf: 0, Price: 2.5}, "m4.16xlarge": {Region: "ap-southeast-1", Type: "m4.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.0}, "m5.large": {Region: "ap-southeast-1", Type: "m5.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.12}, "m5.xlarge": {Region: "ap-southeast-1", Type: "m5.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.24}, "m5.2xlarge": {Region: "ap-southeast-1", Type: "m5.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.48}, "m5.4xlarge": {Region: "ap-southeast-1", Type: "m5.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.96}, "m5.8xlarge": {Region: "ap-southeast-1", Type: "m5.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.92}, "m5.12xlarge": {Region: "ap-southeast-1", Type: "m5.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.88}, "m5.16xlarge": {Region: "ap-southeast-1", Type: "m5.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.84}, "m5.24xlarge": {Region: "ap-southeast-1", Type: "m5.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.76}, "m5.metal": {Region: "ap-southeast-1", Type: "m5.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.76}, "m5a.large": {Region: "ap-southeast-1", Type: "m5a.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.108}, "m5a.xlarge": {Region: "ap-southeast-1", Type: "m5a.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.216}, "m5a.2xlarge": {Region: "ap-southeast-1", Type: "m5a.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.432}, "m5a.4xlarge": {Region: "ap-southeast-1", Type: "m5a.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.864}, "m5a.8xlarge": {Region: "ap-southeast-1", Type: "m5a.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.728}, "m5a.12xlarge": {Region: "ap-southeast-1", Type: "m5a.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.592}, "m5a.16xlarge": {Region: "ap-southeast-1", Type: "m5a.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.456}, "m5a.24xlarge": {Region: "ap-southeast-1", Type: "m5a.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.184}, "m5ad.large": {Region: "ap-southeast-1", Type: "m5ad.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.129}, "m5ad.xlarge": {Region: "ap-southeast-1", Type: "m5ad.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.258}, "m5ad.2xlarge": {Region: "ap-southeast-1", Type: "m5ad.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.516}, "m5ad.4xlarge": {Region: "ap-southeast-1", Type: "m5ad.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.032}, "m5ad.8xlarge": {Region: "ap-southeast-1", Type: "m5ad.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.064}, "m5ad.12xlarge": {Region: "ap-southeast-1", Type: "m5ad.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.096}, "m5ad.16xlarge": {Region: "ap-southeast-1", Type: "m5ad.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.128}, "m5ad.24xlarge": {Region: "ap-southeast-1", Type: "m5ad.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.192}, "m5d.large": {Region: "ap-southeast-1", Type: "m5d.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.141}, "m5d.xlarge": {Region: "ap-southeast-1", Type: "m5d.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.282}, "m5d.2xlarge": {Region: "ap-southeast-1", Type: "m5d.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.564}, "m5d.4xlarge": {Region: "ap-southeast-1", Type: "m5d.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.128}, "m5d.8xlarge": {Region: "ap-southeast-1", Type: "m5d.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.256}, "m5d.12xlarge": {Region: "ap-southeast-1", Type: "m5d.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.384}, "m5d.16xlarge": {Region: "ap-southeast-1", Type: "m5d.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.512}, "m5d.24xlarge": {Region: "ap-southeast-1", Type: "m5d.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.768}, "m5d.metal": {Region: "ap-southeast-1", Type: "m5d.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.768}, "m5dn.large": {Region: "ap-southeast-1", Type: "m5dn.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.167}, "m5dn.xlarge": {Region: "ap-southeast-1", Type: "m5dn.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.334}, "m5dn.2xlarge": {Region: "ap-southeast-1", Type: "m5dn.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.668}, "m5dn.4xlarge": {Region: "ap-southeast-1", Type: "m5dn.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.336}, "m5dn.8xlarge": {Region: "ap-southeast-1", Type: "m5dn.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.672}, "m5dn.12xlarge": {Region: "ap-southeast-1", Type: "m5dn.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.008}, "m5dn.16xlarge": {Region: "ap-southeast-1", Type: "m5dn.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.344}, "m5dn.24xlarge": {Region: "ap-southeast-1", Type: "m5dn.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.016}, "m5dn.metal": {Region: "ap-southeast-1", Type: "m5dn.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.016}, "m5n.large": {Region: "ap-southeast-1", Type: "m5n.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.146}, "m5n.xlarge": {Region: "ap-southeast-1", Type: "m5n.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.292}, "m5n.2xlarge": {Region: "ap-southeast-1", Type: "m5n.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.584}, "m5n.4xlarge": {Region: "ap-southeast-1", Type: "m5n.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.168}, "m5n.8xlarge": {Region: "ap-southeast-1", Type: "m5n.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.336}, "m5n.12xlarge": {Region: "ap-southeast-1", Type: "m5n.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.504}, "m5n.16xlarge": {Region: "ap-southeast-1", Type: "m5n.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.672}, "m5n.24xlarge": {Region: "ap-southeast-1", Type: "m5n.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.008}, "m5n.metal": {Region: "ap-southeast-1", Type: "m5n.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.008}, "m5zn.large": {Region: "ap-southeast-1", Type: "m5zn.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.2065}, "m5zn.xlarge": {Region: "ap-southeast-1", Type: "m5zn.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.413}, "m5zn.2xlarge": {Region: "ap-southeast-1", Type: "m5zn.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.826}, "m5zn.3xlarge": {Region: "ap-southeast-1", Type: "m5zn.3xlarge", Memory: kresource.MustParse("49152Mi"), CPU: kresource.MustParse("12"), GPU: 0, Inf: 0, Price: 1.239}, "m5zn.6xlarge": {Region: "ap-southeast-1", Type: "m5zn.6xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 0, Price: 2.478}, "m5zn.12xlarge": {Region: "ap-southeast-1", Type: "m5zn.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.956}, "m5zn.metal": {Region: "ap-southeast-1", Type: "m5zn.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.956}, "m6g.medium": {Region: "ap-southeast-1", Type: "m6g.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.048}, "m6g.large": {Region: "ap-southeast-1", Type: "m6g.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.096}, "m6g.xlarge": {Region: "ap-southeast-1", Type: "m6g.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.192}, "m6g.2xlarge": {Region: "ap-southeast-1", Type: "m6g.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.384}, "m6g.4xlarge": {Region: "ap-southeast-1", Type: "m6g.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.768}, "m6g.8xlarge": {Region: "ap-southeast-1", Type: "m6g.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.536}, "m6g.12xlarge": {Region: "ap-southeast-1", Type: "m6g.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.304}, "m6g.16xlarge": {Region: "ap-southeast-1", Type: "m6g.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.072}, "m6g.metal": {Region: "ap-southeast-1", Type: "m6g.metal", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.072}, "m6gd.medium": {Region: "ap-southeast-1", Type: "m6gd.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0565}, "m6gd.large": {Region: "ap-southeast-1", Type: "m6gd.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.113}, "m6gd.xlarge": {Region: "ap-southeast-1", Type: "m6gd.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.226}, "m6gd.2xlarge": {Region: "ap-southeast-1", Type: "m6gd.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.452}, "m6gd.4xlarge": {Region: "ap-southeast-1", Type: "m6gd.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.904}, "m6gd.8xlarge": {Region: "ap-southeast-1", Type: "m6gd.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.808}, "m6gd.12xlarge": {Region: "ap-southeast-1", Type: "m6gd.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.712}, "m6gd.16xlarge": {Region: "ap-southeast-1", Type: "m6gd.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.616}, "m6gd.metal": {Region: "ap-southeast-1", Type: "m6gd.metal", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.616}, "m6i.large": {Region: "ap-southeast-1", Type: "m6i.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.12}, "m6i.xlarge": {Region: "ap-southeast-1", Type: "m6i.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.24}, "m6i.2xlarge": {Region: "ap-southeast-1", Type: "m6i.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.48}, "m6i.4xlarge": {Region: "ap-southeast-1", Type: "m6i.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.96}, "m6i.8xlarge": {Region: "ap-southeast-1", Type: "m6i.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.92}, "m6i.12xlarge": {Region: "ap-southeast-1", Type: "m6i.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.88}, "m6i.16xlarge": {Region: "ap-southeast-1", Type: "m6i.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.84}, "m6i.24xlarge": {Region: "ap-southeast-1", Type: "m6i.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.76}, "m6i.32xlarge": {Region: "ap-southeast-1", Type: "m6i.32xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 7.68}, "m6i.metal": {Region: "ap-southeast-1", Type: "m6i.metal", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 7.68}, "p2.xlarge": {Region: "ap-southeast-1", Type: "p2.xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("4"), GPU: 1, Inf: 0, Price: 1.718}, "p2.8xlarge": {Region: "ap-southeast-1", Type: "p2.8xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("32"), GPU: 8, Inf: 0, Price: 13.744}, "p2.16xlarge": {Region: "ap-southeast-1", Type: "p2.16xlarge", Memory: kresource.MustParse("749568Mi"), CPU: kresource.MustParse("64"), GPU: 16, Inf: 0, Price: 27.488}, "p3.2xlarge": {Region: "ap-southeast-1", Type: "p3.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 1, Inf: 0, Price: 4.234}, "p3.8xlarge": {Region: "ap-southeast-1", Type: "p3.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 4, Inf: 0, Price: 16.936}, "p3.16xlarge": {Region: "ap-southeast-1", Type: "p3.16xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("64"), GPU: 8, Inf: 0, Price: 33.872}, "r3.large": {Region: "ap-southeast-1", Type: "r3.large", Memory: kresource.MustParse("15616Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.2}, "r3.xlarge": {Region: "ap-southeast-1", Type: "r3.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.399}, "r3.2xlarge": {Region: "ap-southeast-1", Type: "r3.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.798}, "r3.4xlarge": {Region: "ap-southeast-1", Type: "r3.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.596}, "r3.8xlarge": {Region: "ap-southeast-1", Type: "r3.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 3.192}, "r4.large": {Region: "ap-southeast-1", Type: "r4.large", Memory: kresource.MustParse("15616Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.16}, "r4.xlarge": {Region: "ap-southeast-1", Type: "r4.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.32}, "r4.2xlarge": {Region: "ap-southeast-1", Type: "r4.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.64}, "r4.4xlarge": {Region: "ap-southeast-1", Type: "r4.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.28}, "r4.8xlarge": {Region: "ap-southeast-1", Type: "r4.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.56}, "r4.16xlarge": {Region: "ap-southeast-1", Type: "r4.16xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.12}, "r5.large": {Region: "ap-southeast-1", Type: "r5.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.152}, "r5.xlarge": {Region: "ap-southeast-1", Type: "r5.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.304}, "r5.2xlarge": {Region: "ap-southeast-1", Type: "r5.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.608}, "r5.4xlarge": {Region: "ap-southeast-1", Type: "r5.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.216}, "r5.8xlarge": {Region: "ap-southeast-1", Type: "r5.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.432}, "r5.12xlarge": {Region: "ap-southeast-1", Type: "r5.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.648}, "r5.16xlarge": {Region: "ap-southeast-1", Type: "r5.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.864}, "r5.24xlarge": {Region: "ap-southeast-1", Type: "r5.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.296}, "r5.metal": {Region: "ap-southeast-1", Type: "r5.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.296}, "r5a.large": {Region: "ap-southeast-1", Type: "r5a.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.136}, "r5a.xlarge": {Region: "ap-southeast-1", Type: "r5a.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.272}, "r5a.2xlarge": {Region: "ap-southeast-1", Type: "r5a.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.544}, "r5a.4xlarge": {Region: "ap-southeast-1", Type: "r5a.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.088}, "r5a.8xlarge": {Region: "ap-southeast-1", Type: "r5a.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.176}, "r5a.12xlarge": {Region: "ap-southeast-1", Type: "r5a.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.264}, "r5a.16xlarge": {Region: "ap-southeast-1", Type: "r5a.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.352}, "r5a.24xlarge": {Region: "ap-southeast-1", Type: "r5a.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.528}, "r5ad.large": {Region: "ap-southeast-1", Type: "r5ad.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.159}, "r5ad.xlarge": {Region: "ap-southeast-1", Type: "r5ad.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.318}, "r5ad.2xlarge": {Region: "ap-southeast-1", Type: "r5ad.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.636}, "r5ad.4xlarge": {Region: "ap-southeast-1", Type: "r5ad.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.272}, "r5ad.8xlarge": {Region: "ap-southeast-1", Type: "r5ad.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.544}, "r5ad.12xlarge": {Region: "ap-southeast-1", Type: "r5ad.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.816}, "r5ad.16xlarge": {Region: "ap-southeast-1", Type: "r5ad.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.088}, "r5ad.24xlarge": {Region: "ap-southeast-1", Type: "r5ad.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.632}, "r5b.large": {Region: "ap-southeast-1", Type: "r5b.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.178}, "r5b.xlarge": {Region: "ap-southeast-1", Type: "r5b.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.356}, "r5b.2xlarge": {Region: "ap-southeast-1", Type: "r5b.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.712}, "r5b.4xlarge": {Region: "ap-southeast-1", Type: "r5b.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.424}, "r5b.8xlarge": {Region: "ap-southeast-1", Type: "r5b.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.848}, "r5b.12xlarge": {Region: "ap-southeast-1", Type: "r5b.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.272}, "r5b.16xlarge": {Region: "ap-southeast-1", Type: "r5b.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.696}, "r5b.24xlarge": {Region: "ap-southeast-1", Type: "r5b.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.544}, "r5b.metal": {Region: "ap-southeast-1", Type: "r5b.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.544}, "r5d.large": {Region: "ap-southeast-1", Type: "r5d.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.174}, "r5d.xlarge": {Region: "ap-southeast-1", Type: "r5d.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.348}, "r5d.2xlarge": {Region: "ap-southeast-1", Type: "r5d.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.696}, "r5d.4xlarge": {Region: "ap-southeast-1", Type: "r5d.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.392}, "r5d.8xlarge": {Region: "ap-southeast-1", Type: "r5d.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.784}, "r5d.12xlarge": {Region: "ap-southeast-1", Type: "r5d.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.176}, "r5d.16xlarge": {Region: "ap-southeast-1", Type: "r5d.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.568}, "r5d.24xlarge": {Region: "ap-southeast-1", Type: "r5d.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.352}, "r5d.metal": {Region: "ap-southeast-1", Type: "r5d.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.352}, "r5dn.large": {Region: "ap-southeast-1", Type: "r5dn.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.2}, "r5dn.xlarge": {Region: "ap-southeast-1", Type: "r5dn.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.4}, "r5dn.2xlarge": {Region: "ap-southeast-1", Type: "r5dn.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.8}, "r5dn.4xlarge": {Region: "ap-southeast-1", Type: "r5dn.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.6}, "r5dn.8xlarge": {Region: "ap-southeast-1", Type: "r5dn.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 3.2}, "r5dn.12xlarge": {Region: "ap-southeast-1", Type: "r5dn.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.8}, "r5dn.16xlarge": {Region: "ap-southeast-1", Type: "r5dn.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 6.4}, "r5dn.24xlarge": {Region: "ap-southeast-1", Type: "r5dn.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 9.6}, "r5dn.metal": {Region: "ap-southeast-1", Type: "r5dn.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 9.6}, "r5n.large": {Region: "ap-southeast-1", Type: "r5n.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.178}, "r5n.xlarge": {Region: "ap-southeast-1", Type: "r5n.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.356}, "r5n.2xlarge": {Region: "ap-southeast-1", Type: "r5n.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.712}, "r5n.4xlarge": {Region: "ap-southeast-1", Type: "r5n.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.424}, "r5n.8xlarge": {Region: "ap-southeast-1", Type: "r5n.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.848}, "r5n.12xlarge": {Region: "ap-southeast-1", Type: "r5n.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.272}, "r5n.16xlarge": {Region: "ap-southeast-1", Type: "r5n.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.696}, "r5n.24xlarge": {Region: "ap-southeast-1", Type: "r5n.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.544}, "r5n.metal": {Region: "ap-southeast-1", Type: "r5n.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.544}, "r6g.medium": {Region: "ap-southeast-1", Type: "r6g.medium", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0608}, "r6g.large": {Region: "ap-southeast-1", Type: "r6g.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.1216}, "r6g.xlarge": {Region: "ap-southeast-1", Type: "r6g.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.2432}, "r6g.2xlarge": {Region: "ap-southeast-1", Type: "r6g.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.4864}, "r6g.4xlarge": {Region: "ap-southeast-1", Type: "r6g.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.9728}, "r6g.8xlarge": {Region: "ap-southeast-1", Type: "r6g.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.9456}, "r6g.12xlarge": {Region: "ap-southeast-1", Type: "r6g.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.9184}, "r6g.16xlarge": {Region: "ap-southeast-1", Type: "r6g.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.8912}, "r6g.metal": {Region: "ap-southeast-1", Type: "r6g.metal", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.8912}, "r6gd.medium": {Region: "ap-southeast-1", Type: "r6gd.medium", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0695}, "r6gd.large": {Region: "ap-southeast-1", Type: "r6gd.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.139}, "r6gd.xlarge": {Region: "ap-southeast-1", Type: "r6gd.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.278}, "r6gd.2xlarge": {Region: "ap-southeast-1", Type: "r6gd.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.556}, "r6gd.4xlarge": {Region: "ap-southeast-1", Type: "r6gd.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.112}, "r6gd.8xlarge": {Region: "ap-southeast-1", Type: "r6gd.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.224}, "r6gd.12xlarge": {Region: "ap-southeast-1", Type: "r6gd.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.336}, "r6gd.16xlarge": {Region: "ap-southeast-1", Type: "r6gd.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.448}, "r6gd.metal": {Region: "ap-southeast-1", Type: "r6gd.metal", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.448}, "r6i.large": {Region: "ap-southeast-1", Type: "r6i.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.152}, "r6i.xlarge": {Region: "ap-southeast-1", Type: "r6i.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.304}, "r6i.2xlarge": {Region: "ap-southeast-1", Type: "r6i.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.608}, "r6i.4xlarge": {Region: "ap-southeast-1", Type: "r6i.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.216}, "r6i.8xlarge": {Region: "ap-southeast-1", Type: "r6i.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.432}, "r6i.12xlarge": {Region: "ap-southeast-1", Type: "r6i.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.648}, "r6i.16xlarge": {Region: "ap-southeast-1", Type: "r6i.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.864}, "r6i.24xlarge": {Region: "ap-southeast-1", Type: "r6i.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.296}, "r6i.32xlarge": {Region: "ap-southeast-1", Type: "r6i.32xlarge", Memory: kresource.MustParse("1048576Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 9.728}, "r6i.metal": {Region: "ap-southeast-1", Type: "r6i.metal", Memory: kresource.MustParse("1048576Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 9.728}, "t1.micro": {Region: "ap-southeast-1", Type: "t1.micro", Memory: kresource.MustParse("627Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.02}, "t2.nano": {Region: "ap-southeast-1", Type: "t2.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0073}, "t2.micro": {Region: "ap-southeast-1", Type: "t2.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0146}, "t2.small": {Region: "ap-southeast-1", Type: "t2.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0292}, "t2.medium": {Region: "ap-southeast-1", Type: "t2.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0584}, "t2.large": {Region: "ap-southeast-1", Type: "t2.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.1168}, "t2.xlarge": {Region: "ap-southeast-1", Type: "t2.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.2336}, "t2.2xlarge": {Region: "ap-southeast-1", Type: "t2.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.4672}, "t3.nano": {Region: "ap-southeast-1", Type: "t3.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0066}, "t3.micro": {Region: "ap-southeast-1", Type: "t3.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0132}, "t3.small": {Region: "ap-southeast-1", Type: "t3.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0264}, "t3.medium": {Region: "ap-southeast-1", Type: "t3.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0528}, "t3.large": {Region: "ap-southeast-1", Type: "t3.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.1056}, "t3.xlarge": {Region: "ap-southeast-1", Type: "t3.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.2112}, "t3.2xlarge": {Region: "ap-southeast-1", Type: "t3.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.4224}, "t3a.nano": {Region: "ap-southeast-1", Type: "t3a.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0059}, "t3a.micro": {Region: "ap-southeast-1", Type: "t3a.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0118}, "t3a.small": {Region: "ap-southeast-1", Type: "t3a.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0236}, "t3a.medium": {Region: "ap-southeast-1", Type: "t3a.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0472}, "t3a.large": {Region: "ap-southeast-1", Type: "t3a.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0944}, "t3a.xlarge": {Region: "ap-southeast-1", Type: "t3a.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1888}, "t3a.2xlarge": {Region: "ap-southeast-1", Type: "t3a.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3776}, "t4g.nano": {Region: "ap-southeast-1", Type: "t4g.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0053}, "t4g.micro": {Region: "ap-southeast-1", Type: "t4g.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0106}, "t4g.small": {Region: "ap-southeast-1", Type: "t4g.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0212}, "t4g.medium": {Region: "ap-southeast-1", Type: "t4g.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0424}, "t4g.large": {Region: "ap-southeast-1", Type: "t4g.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0848}, "t4g.xlarge": {Region: "ap-southeast-1", Type: "t4g.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1696}, "t4g.2xlarge": {Region: "ap-southeast-1", Type: "t4g.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3392}, "u-12tb1.112xlarge": {Region: "ap-southeast-1", Type: "u-12tb1.112xlarge", Memory: kresource.MustParse("12582912Mi"), CPU: kresource.MustParse("448"), GPU: 0, Inf: 0, Price: 131.733}, "u-6tb1.56xlarge": {Region: "ap-southeast-1", Type: "u-6tb1.56xlarge", Memory: kresource.MustParse("6291456Mi"), CPU: kresource.MustParse("224"), GPU: 0, Inf: 0, Price: 55.9796}, "u-6tb1.112xlarge": {Region: "ap-southeast-1", Type: "u-6tb1.112xlarge", Memory: kresource.MustParse("6291456Mi"), CPU: kresource.MustParse("448"), GPU: 0, Inf: 0, Price: 65.867}, "u-9tb1.112xlarge": {Region: "ap-southeast-1", Type: "u-9tb1.112xlarge", Memory: kresource.MustParse("9437184Mi"), CPU: kresource.MustParse("448"), GPU: 0, Inf: 0, Price: 98.8}, "x1.16xlarge": {Region: "ap-southeast-1", Type: "x1.16xlarge", Memory: kresource.MustParse("999424Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 9.671}, "x1.32xlarge": {Region: "ap-southeast-1", Type: "x1.32xlarge", Memory: kresource.MustParse("1998848Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 19.341}, "x1e.xlarge": {Region: "ap-southeast-1", Type: "x1e.xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 1.209}, "x1e.2xlarge": {Region: "ap-southeast-1", Type: "x1e.2xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 2.418}, "x1e.4xlarge": {Region: "ap-southeast-1", Type: "x1e.4xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 4.836}, "x1e.8xlarge": {Region: "ap-southeast-1", Type: "x1e.8xlarge", Memory: kresource.MustParse("999424Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 9.672}, "x1e.16xlarge": {Region: "ap-southeast-1", Type: "x1e.16xlarge", Memory: kresource.MustParse("1998848Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 19.344}, "x1e.32xlarge": {Region: "ap-southeast-1", Type: "x1e.32xlarge", Memory: kresource.MustParse("3997696Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 38.688}, "x2idn.16xlarge": {Region: "ap-southeast-1", Type: "x2idn.16xlarge", Memory: kresource.MustParse("1048576Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 9.6705}, "x2idn.24xlarge": {Region: "ap-southeast-1", Type: "x2idn.24xlarge", Memory: kresource.MustParse("1572864Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 14.50575}, "x2idn.32xlarge": {Region: "ap-southeast-1", Type: "x2idn.32xlarge", Memory: kresource.MustParse("2097152Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 19.341}, "x2iedn.xlarge": {Region: "ap-southeast-1", Type: "x2iedn.xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 1.20881}, "x2iedn.2xlarge": {Region: "ap-southeast-1", Type: "x2iedn.2xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 2.41763}, "x2iedn.4xlarge": {Region: "ap-southeast-1", Type: "x2iedn.4xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 4.83525}, "x2iedn.8xlarge": {Region: "ap-southeast-1", Type: "x2iedn.8xlarge", Memory: kresource.MustParse("1048576Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 9.6705}, "x2iedn.16xlarge": {Region: "ap-southeast-1", Type: "x2iedn.16xlarge", Memory: kresource.MustParse("2097152Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 19.341}, "x2iedn.24xlarge": {Region: "ap-southeast-1", Type: "x2iedn.24xlarge", Memory: kresource.MustParse("3145728Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 29.0115}, "x2iedn.32xlarge": {Region: "ap-southeast-1", Type: "x2iedn.32xlarge", Memory: kresource.MustParse("4194304Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 38.682}, "z1d.large": {Region: "ap-southeast-1", Type: "z1d.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.226}, "z1d.xlarge": {Region: "ap-southeast-1", Type: "z1d.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.452}, "z1d.2xlarge": {Region: "ap-southeast-1", Type: "z1d.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.904}, "z1d.3xlarge": {Region: "ap-southeast-1", Type: "z1d.3xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("12"), GPU: 0, Inf: 0, Price: 1.356}, "z1d.6xlarge": {Region: "ap-southeast-1", Type: "z1d.6xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 0, Price: 2.712}, "z1d.12xlarge": {Region: "ap-southeast-1", Type: "z1d.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 5.424}, "z1d.metal": {Region: "ap-southeast-1", Type: "z1d.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 5.424}, }, "ap-southeast-2": { "a1.medium": {Region: "ap-southeast-2", Type: "a1.medium", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0333}, "a1.large": {Region: "ap-southeast-2", Type: "a1.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0666}, "a1.xlarge": {Region: "ap-southeast-2", Type: "a1.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1332}, "a1.2xlarge": {Region: "ap-southeast-2", Type: "a1.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.2664}, "a1.4xlarge": {Region: "ap-southeast-2", Type: "a1.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.5328}, "a1.metal": {Region: "ap-southeast-2", Type: "a1.metal", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.533}, "c1.medium": {Region: "ap-southeast-2", Type: "c1.medium", Memory: kresource.MustParse("1740Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.164}, "c1.xlarge": {Region: "ap-southeast-2", Type: "c1.xlarge", Memory: kresource.MustParse("7168Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.655}, "c3.large": {Region: "ap-southeast-2", Type: "c3.large", Memory: kresource.MustParse("3840Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.132}, "c3.xlarge": {Region: "ap-southeast-2", Type: "c3.xlarge", Memory: kresource.MustParse("7680Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.265}, "c3.2xlarge": {Region: "ap-southeast-2", Type: "c3.2xlarge", Memory: kresource.MustParse("15360Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.529}, "c3.4xlarge": {Region: "ap-southeast-2", Type: "c3.4xlarge", Memory: kresource.MustParse("30720Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.058}, "c3.8xlarge": {Region: "ap-southeast-2", Type: "c3.8xlarge", Memory: kresource.MustParse("61440Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.117}, "c4.large": {Region: "ap-southeast-2", Type: "c4.large", Memory: kresource.MustParse("3840Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.13}, "c4.xlarge": {Region: "ap-southeast-2", Type: "c4.xlarge", Memory: kresource.MustParse("7680Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.261}, "c4.2xlarge": {Region: "ap-southeast-2", Type: "c4.2xlarge", Memory: kresource.MustParse("15360Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.522}, "c4.4xlarge": {Region: "ap-southeast-2", Type: "c4.4xlarge", Memory: kresource.MustParse("30720Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.042}, "c4.8xlarge": {Region: "ap-southeast-2", Type: "c4.8xlarge", Memory: kresource.MustParse("61440Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 2.085}, "c5.large": {Region: "ap-southeast-2", Type: "c5.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.111}, "c5.xlarge": {Region: "ap-southeast-2", Type: "c5.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.222}, "c5.2xlarge": {Region: "ap-southeast-2", Type: "c5.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.444}, "c5.4xlarge": {Region: "ap-southeast-2", Type: "c5.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.888}, "c5.9xlarge": {Region: "ap-southeast-2", Type: "c5.9xlarge", Memory: kresource.MustParse("73728Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 1.998}, "c5.12xlarge": {Region: "ap-southeast-2", Type: "c5.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.664}, "c5.18xlarge": {Region: "ap-southeast-2", Type: "c5.18xlarge", Memory: kresource.MustParse("147456Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 3.996}, "c5.24xlarge": {Region: "ap-southeast-2", Type: "c5.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.328}, "c5.metal": {Region: "ap-southeast-2", Type: "c5.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.328}, "c5a.large": {Region: "ap-southeast-2", Type: "c5a.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.1}, "c5a.xlarge": {Region: "ap-southeast-2", Type: "c5a.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.2}, "c5a.2xlarge": {Region: "ap-southeast-2", Type: "c5a.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.4}, "c5a.4xlarge": {Region: "ap-southeast-2", Type: "c5a.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.8}, "c5a.8xlarge": {Region: "ap-southeast-2", Type: "c5a.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.6}, "c5a.12xlarge": {Region: "ap-southeast-2", Type: "c5a.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.4}, "c5a.16xlarge": {Region: "ap-southeast-2", Type: "c5a.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.2}, "c5a.24xlarge": {Region: "ap-southeast-2", Type: "c5a.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.8}, "c5ad.large": {Region: "ap-southeast-2", Type: "c5ad.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.113}, "c5ad.xlarge": {Region: "ap-southeast-2", Type: "c5ad.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.226}, "c5ad.2xlarge": {Region: "ap-southeast-2", Type: "c5ad.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.452}, "c5ad.4xlarge": {Region: "ap-southeast-2", Type: "c5ad.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.904}, "c5ad.8xlarge": {Region: "ap-southeast-2", Type: "c5ad.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.808}, "c5ad.12xlarge": {Region: "ap-southeast-2", Type: "c5ad.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.712}, "c5ad.16xlarge": {Region: "ap-southeast-2", Type: "c5ad.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.616}, "c5ad.24xlarge": {Region: "ap-southeast-2", Type: "c5ad.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.424}, "c5d.large": {Region: "ap-southeast-2", Type: "c5d.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.126}, "c5d.xlarge": {Region: "ap-southeast-2", Type: "c5d.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.252}, "c5d.2xlarge": {Region: "ap-southeast-2", Type: "c5d.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.504}, "c5d.4xlarge": {Region: "ap-southeast-2", Type: "c5d.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.008}, "c5d.9xlarge": {Region: "ap-southeast-2", Type: "c5d.9xlarge", Memory: kresource.MustParse("73728Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 2.268}, "c5d.12xlarge": {Region: "ap-southeast-2", Type: "c5d.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.024}, "c5d.18xlarge": {Region: "ap-southeast-2", Type: "c5d.18xlarge", Memory: kresource.MustParse("147456Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 4.536}, "c5d.24xlarge": {Region: "ap-southeast-2", Type: "c5d.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.048}, "c5d.metal": {Region: "ap-southeast-2", Type: "c5d.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.048}, "c5n.large": {Region: "ap-southeast-2", Type: "c5n.large", Memory: kresource.MustParse("5376Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.141}, "c5n.xlarge": {Region: "ap-southeast-2", Type: "c5n.xlarge", Memory: kresource.MustParse("10752Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.282}, "c5n.2xlarge": {Region: "ap-southeast-2", Type: "c5n.2xlarge", Memory: kresource.MustParse("21504Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.564}, "c5n.4xlarge": {Region: "ap-southeast-2", Type: "c5n.4xlarge", Memory: kresource.MustParse("43008Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.128}, "c5n.9xlarge": {Region: "ap-southeast-2", Type: "c5n.9xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 2.538}, "c5n.18xlarge": {Region: "ap-southeast-2", Type: "c5n.18xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 5.076}, "c5n.metal": {Region: "ap-southeast-2", Type: "c5n.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 5.076}, "c6g.medium": {Region: "ap-southeast-2", Type: "c6g.medium", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0444}, "c6g.large": {Region: "ap-southeast-2", Type: "c6g.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0888}, "c6g.xlarge": {Region: "ap-southeast-2", Type: "c6g.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1776}, "c6g.2xlarge": {Region: "ap-southeast-2", Type: "c6g.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3552}, "c6g.4xlarge": {Region: "ap-southeast-2", Type: "c6g.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.7104}, "c6g.8xlarge": {Region: "ap-southeast-2", Type: "c6g.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.4208}, "c6g.12xlarge": {Region: "ap-southeast-2", Type: "c6g.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.1312}, "c6g.16xlarge": {Region: "ap-southeast-2", Type: "c6g.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.8416}, "c6g.metal": {Region: "ap-southeast-2", Type: "c6g.metal", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.8416}, "c6gd.medium": {Region: "ap-southeast-2", Type: "c6gd.medium", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0505}, "c6gd.large": {Region: "ap-southeast-2", Type: "c6gd.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.101}, "c6gd.xlarge": {Region: "ap-southeast-2", Type: "c6gd.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.202}, "c6gd.2xlarge": {Region: "ap-southeast-2", Type: "c6gd.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.404}, "c6gd.4xlarge": {Region: "ap-southeast-2", Type: "c6gd.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.808}, "c6gd.8xlarge": {Region: "ap-southeast-2", Type: "c6gd.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.616}, "c6gd.12xlarge": {Region: "ap-southeast-2", Type: "c6gd.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.424}, "c6gd.16xlarge": {Region: "ap-southeast-2", Type: "c6gd.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.232}, "c6gd.metal": {Region: "ap-southeast-2", Type: "c6gd.metal", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.232}, "c6gn.medium": {Region: "ap-southeast-2", Type: "c6gn.medium", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0565}, "c6gn.large": {Region: "ap-southeast-2", Type: "c6gn.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.113}, "c6gn.xlarge": {Region: "ap-southeast-2", Type: "c6gn.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.226}, "c6gn.2xlarge": {Region: "ap-southeast-2", Type: "c6gn.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.452}, "c6gn.4xlarge": {Region: "ap-southeast-2", Type: "c6gn.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.904}, "c6gn.8xlarge": {Region: "ap-southeast-2", Type: "c6gn.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.808}, "c6gn.12xlarge": {Region: "ap-southeast-2", Type: "c6gn.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.712}, "c6gn.16xlarge": {Region: "ap-southeast-2", Type: "c6gn.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.616}, "c6i.large": {Region: "ap-southeast-2", Type: "c6i.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.111}, "c6i.xlarge": {Region: "ap-southeast-2", Type: "c6i.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.222}, "c6i.2xlarge": {Region: "ap-southeast-2", Type: "c6i.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.444}, "c6i.4xlarge": {Region: "ap-southeast-2", Type: "c6i.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.888}, "c6i.8xlarge": {Region: "ap-southeast-2", Type: "c6i.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.776}, "c6i.12xlarge": {Region: "ap-southeast-2", Type: "c6i.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.664}, "c6i.16xlarge": {Region: "ap-southeast-2", Type: "c6i.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.552}, "c6i.24xlarge": {Region: "ap-southeast-2", Type: "c6i.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.328}, "c6i.32xlarge": {Region: "ap-southeast-2", Type: "c6i.32xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 7.104}, "c6i.metal": {Region: "ap-southeast-2", Type: "c6i.metal", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 7.104}, "d2.xlarge": {Region: "ap-southeast-2", Type: "d2.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.87}, "d2.2xlarge": {Region: "ap-southeast-2", Type: "d2.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.74}, "d2.4xlarge": {Region: "ap-southeast-2", Type: "d2.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 3.48}, "d2.8xlarge": {Region: "ap-southeast-2", Type: "d2.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 6.96}, "d3.xlarge": {Region: "ap-southeast-2", Type: "d3.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.626}, "d3.2xlarge": {Region: "ap-southeast-2", Type: "d3.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.252}, "d3.4xlarge": {Region: "ap-southeast-2", Type: "d3.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 2.504}, "d3.8xlarge": {Region: "ap-southeast-2", Type: "d3.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 5.00808}, "f1.2xlarge": {Region: "ap-southeast-2", Type: "f1.2xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.981}, "f1.4xlarge": {Region: "ap-southeast-2", Type: "f1.4xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 3.962}, "f1.16xlarge": {Region: "ap-southeast-2", Type: "f1.16xlarge", Memory: kresource.MustParse("999424Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 15.848}, "g2.2xlarge": {Region: "ap-southeast-2", Type: "g2.2xlarge", Memory: kresource.MustParse("15360Mi"), CPU: kresource.MustParse("8"), GPU: 1, Inf: 0, Price: 0.898}, "g2.8xlarge": {Region: "ap-southeast-2", Type: "g2.8xlarge", Memory: kresource.MustParse("61440Mi"), CPU: kresource.MustParse("32"), GPU: 4, Inf: 0, Price: 3.592}, "g3.4xlarge": {Region: "ap-southeast-2", Type: "g3.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 1, Inf: 0, Price: 1.754}, "g3.8xlarge": {Region: "ap-southeast-2", Type: "g3.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 2, Inf: 0, Price: 3.508}, "g3.16xlarge": {Region: "ap-southeast-2", Type: "g3.16xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("64"), GPU: 4, Inf: 0, Price: 7.016}, "g3s.xlarge": {Region: "ap-southeast-2", Type: "g3s.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 1, Inf: 0, Price: 1.154}, "g4dn.xlarge": {Region: "ap-southeast-2", Type: "g4dn.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 1, Inf: 0, Price: 0.684}, "g4dn.2xlarge": {Region: "ap-southeast-2", Type: "g4dn.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 1, Inf: 0, Price: 0.978}, "g4dn.4xlarge": {Region: "ap-southeast-2", Type: "g4dn.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 1, Inf: 0, Price: 1.566}, "g4dn.8xlarge": {Region: "ap-southeast-2", Type: "g4dn.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 1, Inf: 0, Price: 2.83}, "g4dn.12xlarge": {Region: "ap-southeast-2", Type: "g4dn.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 4, Inf: 0, Price: 5.087}, "g4dn.16xlarge": {Region: "ap-southeast-2", Type: "g4dn.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 1, Inf: 0, Price: 5.659}, "g4dn.metal": {Region: "ap-southeast-2", Type: "g4dn.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 8, Inf: 0, Price: 10.174}, "hs1.8xlarge": {Region: "ap-southeast-2", Type: "hs1.8xlarge", Memory: kresource.MustParse("119808Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 5.57}, "i2.xlarge": {Region: "ap-southeast-2", Type: "i2.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 1.018}, "i2.2xlarge": {Region: "ap-southeast-2", Type: "i2.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 2.035}, "i2.4xlarge": {Region: "ap-southeast-2", Type: "i2.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 4.07}, "i2.8xlarge": {Region: "ap-southeast-2", Type: "i2.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 8.14}, "i3.large": {Region: "ap-southeast-2", Type: "i3.large", Memory: kresource.MustParse("15616Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.187}, "i3.xlarge": {Region: "ap-southeast-2", Type: "i3.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.374}, "i3.2xlarge": {Region: "ap-southeast-2", Type: "i3.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.748}, "i3.4xlarge": {Region: "ap-southeast-2", Type: "i3.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.496}, "i3.8xlarge": {Region: "ap-southeast-2", Type: "i3.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.992}, "i3.16xlarge": {Region: "ap-southeast-2", Type: "i3.16xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.984}, "i3.metal": {Region: "ap-southeast-2", Type: "i3.metal", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.984}, "i3en.large": {Region: "ap-southeast-2", Type: "i3en.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.271}, "i3en.xlarge": {Region: "ap-southeast-2", Type: "i3en.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.542}, "i3en.2xlarge": {Region: "ap-southeast-2", Type: "i3en.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.084}, "i3en.3xlarge": {Region: "ap-southeast-2", Type: "i3en.3xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("12"), GPU: 0, Inf: 0, Price: 1.626}, "i3en.6xlarge": {Region: "ap-southeast-2", Type: "i3en.6xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 0, Price: 3.252}, "i3en.12xlarge": {Region: "ap-southeast-2", Type: "i3en.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 6.504}, "i3en.24xlarge": {Region: "ap-southeast-2", Type: "i3en.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 13.008}, "i3en.metal": {Region: "ap-southeast-2", Type: "i3en.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 13.008}, "inf1.xlarge": {Region: "ap-southeast-2", Type: "inf1.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 1, Price: 0.285}, "inf1.2xlarge": {Region: "ap-southeast-2", Type: "inf1.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 1, Price: 0.453}, "inf1.6xlarge": {Region: "ap-southeast-2", Type: "inf1.6xlarge", Memory: kresource.MustParse("49152Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 4, Price: 1.475}, "inf1.24xlarge": {Region: "ap-southeast-2", Type: "inf1.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 16, Price: 5.902}, "m1.small": {Region: "ap-southeast-2", Type: "m1.small", Memory: kresource.MustParse("1740Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.058}, "m1.medium": {Region: "ap-southeast-2", Type: "m1.medium", Memory: kresource.MustParse("3840Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.117}, "m1.large": {Region: "ap-southeast-2", Type: "m1.large", Memory: kresource.MustParse("7680Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.233}, "m1.xlarge": {Region: "ap-southeast-2", Type: "m1.xlarge", Memory: kresource.MustParse("15360Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.467}, "m2.xlarge": {Region: "ap-southeast-2", Type: "m2.xlarge", Memory: kresource.MustParse("17510Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.296}, "m2.2xlarge": {Region: "ap-southeast-2", Type: "m2.2xlarge", Memory: kresource.MustParse("35020Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.592}, "m2.4xlarge": {Region: "ap-southeast-2", Type: "m2.4xlarge", Memory: kresource.MustParse("70041Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.183}, "m3.medium": {Region: "ap-southeast-2", Type: "m3.medium", Memory: kresource.MustParse("3840Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.093}, "m3.large": {Region: "ap-southeast-2", Type: "m3.large", Memory: kresource.MustParse("7680Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.186}, "m3.xlarge": {Region: "ap-southeast-2", Type: "m3.xlarge", Memory: kresource.MustParse("15360Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.372}, "m3.2xlarge": {Region: "ap-southeast-2", Type: "m3.2xlarge", Memory: kresource.MustParse("30720Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.745}, "m4.large": {Region: "ap-southeast-2", Type: "m4.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.125}, "m4.xlarge": {Region: "ap-southeast-2", Type: "m4.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.25}, "m4.2xlarge": {Region: "ap-southeast-2", Type: "m4.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.5}, "m4.4xlarge": {Region: "ap-southeast-2", Type: "m4.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.0}, "m4.10xlarge": {Region: "ap-southeast-2", Type: "m4.10xlarge", Memory: kresource.MustParse("163840Mi"), CPU: kresource.MustParse("40"), GPU: 0, Inf: 0, Price: 2.5}, "m4.16xlarge": {Region: "ap-southeast-2", Type: "m4.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.0}, "m5.large": {Region: "ap-southeast-2", Type: "m5.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.12}, "m5.xlarge": {Region: "ap-southeast-2", Type: "m5.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.24}, "m5.2xlarge": {Region: "ap-southeast-2", Type: "m5.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.48}, "m5.4xlarge": {Region: "ap-southeast-2", Type: "m5.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.96}, "m5.8xlarge": {Region: "ap-southeast-2", Type: "m5.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.92}, "m5.12xlarge": {Region: "ap-southeast-2", Type: "m5.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.88}, "m5.16xlarge": {Region: "ap-southeast-2", Type: "m5.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.84}, "m5.24xlarge": {Region: "ap-southeast-2", Type: "m5.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.76}, "m5.metal": {Region: "ap-southeast-2", Type: "m5.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.76}, "m5a.large": {Region: "ap-southeast-2", Type: "m5a.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.108}, "m5a.xlarge": {Region: "ap-southeast-2", Type: "m5a.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.216}, "m5a.2xlarge": {Region: "ap-southeast-2", Type: "m5a.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.432}, "m5a.4xlarge": {Region: "ap-southeast-2", Type: "m5a.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.864}, "m5a.8xlarge": {Region: "ap-southeast-2", Type: "m5a.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.728}, "m5a.12xlarge": {Region: "ap-southeast-2", Type: "m5a.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.592}, "m5a.16xlarge": {Region: "ap-southeast-2", Type: "m5a.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.456}, "m5a.24xlarge": {Region: "ap-southeast-2", Type: "m5a.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.184}, "m5ad.large": {Region: "ap-southeast-2", Type: "m5ad.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.13}, "m5ad.xlarge": {Region: "ap-southeast-2", Type: "m5ad.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.26}, "m5ad.2xlarge": {Region: "ap-southeast-2", Type: "m5ad.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.52}, "m5ad.4xlarge": {Region: "ap-southeast-2", Type: "m5ad.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.04}, "m5ad.8xlarge": {Region: "ap-southeast-2", Type: "m5ad.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.08}, "m5ad.12xlarge": {Region: "ap-southeast-2", Type: "m5ad.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.12}, "m5ad.16xlarge": {Region: "ap-southeast-2", Type: "m5ad.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.16}, "m5ad.24xlarge": {Region: "ap-southeast-2", Type: "m5ad.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.24}, "m5d.large": {Region: "ap-southeast-2", Type: "m5d.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.142}, "m5d.xlarge": {Region: "ap-southeast-2", Type: "m5d.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.284}, "m5d.2xlarge": {Region: "ap-southeast-2", Type: "m5d.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.568}, "m5d.4xlarge": {Region: "ap-southeast-2", Type: "m5d.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.136}, "m5d.8xlarge": {Region: "ap-southeast-2", Type: "m5d.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.272}, "m5d.12xlarge": {Region: "ap-southeast-2", Type: "m5d.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.408}, "m5d.16xlarge": {Region: "ap-southeast-2", Type: "m5d.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.544}, "m5d.24xlarge": {Region: "ap-southeast-2", Type: "m5d.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.816}, "m5d.metal": {Region: "ap-southeast-2", Type: "m5d.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.816}, "m5zn.large": {Region: "ap-southeast-2", Type: "m5zn.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.2065}, "m5zn.xlarge": {Region: "ap-southeast-2", Type: "m5zn.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.413}, "m5zn.2xlarge": {Region: "ap-southeast-2", Type: "m5zn.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.826}, "m5zn.3xlarge": {Region: "ap-southeast-2", Type: "m5zn.3xlarge", Memory: kresource.MustParse("49152Mi"), CPU: kresource.MustParse("12"), GPU: 0, Inf: 0, Price: 1.239}, "m5zn.6xlarge": {Region: "ap-southeast-2", Type: "m5zn.6xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 0, Price: 2.478}, "m5zn.12xlarge": {Region: "ap-southeast-2", Type: "m5zn.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.956}, "m5zn.metal": {Region: "ap-southeast-2", Type: "m5zn.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.956}, "m6g.medium": {Region: "ap-southeast-2", Type: "m6g.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.048}, "m6g.large": {Region: "ap-southeast-2", Type: "m6g.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.096}, "m6g.xlarge": {Region: "ap-southeast-2", Type: "m6g.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.192}, "m6g.2xlarge": {Region: "ap-southeast-2", Type: "m6g.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.384}, "m6g.4xlarge": {Region: "ap-southeast-2", Type: "m6g.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.768}, "m6g.8xlarge": {Region: "ap-southeast-2", Type: "m6g.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.536}, "m6g.12xlarge": {Region: "ap-southeast-2", Type: "m6g.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.304}, "m6g.16xlarge": {Region: "ap-southeast-2", Type: "m6g.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.072}, "m6g.metal": {Region: "ap-southeast-2", Type: "m6g.metal", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.072}, "m6gd.medium": {Region: "ap-southeast-2", Type: "m6gd.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.057}, "m6gd.large": {Region: "ap-southeast-2", Type: "m6gd.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.114}, "m6gd.xlarge": {Region: "ap-southeast-2", Type: "m6gd.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.228}, "m6gd.2xlarge": {Region: "ap-southeast-2", Type: "m6gd.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.456}, "m6gd.4xlarge": {Region: "ap-southeast-2", Type: "m6gd.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.912}, "m6gd.8xlarge": {Region: "ap-southeast-2", Type: "m6gd.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.824}, "m6gd.12xlarge": {Region: "ap-southeast-2", Type: "m6gd.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.736}, "m6gd.16xlarge": {Region: "ap-southeast-2", Type: "m6gd.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.648}, "m6gd.metal": {Region: "ap-southeast-2", Type: "m6gd.metal", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.648}, "m6i.large": {Region: "ap-southeast-2", Type: "m6i.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.12}, "m6i.xlarge": {Region: "ap-southeast-2", Type: "m6i.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.24}, "m6i.2xlarge": {Region: "ap-southeast-2", Type: "m6i.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.48}, "m6i.4xlarge": {Region: "ap-southeast-2", Type: "m6i.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.96}, "m6i.8xlarge": {Region: "ap-southeast-2", Type: "m6i.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.92}, "m6i.12xlarge": {Region: "ap-southeast-2", Type: "m6i.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.88}, "m6i.16xlarge": {Region: "ap-southeast-2", Type: "m6i.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.84}, "m6i.24xlarge": {Region: "ap-southeast-2", Type: "m6i.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.76}, "m6i.32xlarge": {Region: "ap-southeast-2", Type: "m6i.32xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 7.68}, "m6i.metal": {Region: "ap-southeast-2", Type: "m6i.metal", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 7.68}, "p2.xlarge": {Region: "ap-southeast-2", Type: "p2.xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("4"), GPU: 1, Inf: 0, Price: 1.542}, "p2.8xlarge": {Region: "ap-southeast-2", Type: "p2.8xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("32"), GPU: 8, Inf: 0, Price: 12.336}, "p2.16xlarge": {Region: "ap-southeast-2", Type: "p2.16xlarge", Memory: kresource.MustParse("749568Mi"), CPU: kresource.MustParse("64"), GPU: 16, Inf: 0, Price: 24.672}, "p3.2xlarge": {Region: "ap-southeast-2", Type: "p3.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 1, Inf: 0, Price: 4.234}, "p3.8xlarge": {Region: "ap-southeast-2", Type: "p3.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 4, Inf: 0, Price: 16.936}, "p3.16xlarge": {Region: "ap-southeast-2", Type: "p3.16xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("64"), GPU: 8, Inf: 0, Price: 33.872}, "r3.large": {Region: "ap-southeast-2", Type: "r3.large", Memory: kresource.MustParse("15616Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.2}, "r3.xlarge": {Region: "ap-southeast-2", Type: "r3.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.399}, "r3.2xlarge": {Region: "ap-southeast-2", Type: "r3.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.798}, "r3.4xlarge": {Region: "ap-southeast-2", Type: "r3.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.596}, "r3.8xlarge": {Region: "ap-southeast-2", Type: "r3.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 3.192}, "r4.large": {Region: "ap-southeast-2", Type: "r4.large", Memory: kresource.MustParse("15616Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.1596}, "r4.xlarge": {Region: "ap-southeast-2", Type: "r4.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.3192}, "r4.2xlarge": {Region: "ap-southeast-2", Type: "r4.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.6384}, "r4.4xlarge": {Region: "ap-southeast-2", Type: "r4.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.2768}, "r4.8xlarge": {Region: "ap-southeast-2", Type: "r4.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.5536}, "r4.16xlarge": {Region: "ap-southeast-2", Type: "r4.16xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.1072}, "r5.large": {Region: "ap-southeast-2", Type: "r5.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.151}, "r5.xlarge": {Region: "ap-southeast-2", Type: "r5.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.302}, "r5.2xlarge": {Region: "ap-southeast-2", Type: "r5.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.604}, "r5.4xlarge": {Region: "ap-southeast-2", Type: "r5.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.208}, "r5.8xlarge": {Region: "ap-southeast-2", Type: "r5.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.416}, "r5.12xlarge": {Region: "ap-southeast-2", Type: "r5.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.624}, "r5.16xlarge": {Region: "ap-southeast-2", Type: "r5.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.832}, "r5.24xlarge": {Region: "ap-southeast-2", Type: "r5.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.248}, "r5.metal": {Region: "ap-southeast-2", Type: "r5.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.248}, "r5a.large": {Region: "ap-southeast-2", Type: "r5a.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.136}, "r5a.xlarge": {Region: "ap-southeast-2", Type: "r5a.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.272}, "r5a.2xlarge": {Region: "ap-southeast-2", Type: "r5a.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.544}, "r5a.4xlarge": {Region: "ap-southeast-2", Type: "r5a.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.088}, "r5a.8xlarge": {Region: "ap-southeast-2", Type: "r5a.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.176}, "r5a.12xlarge": {Region: "ap-southeast-2", Type: "r5a.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.264}, "r5a.16xlarge": {Region: "ap-southeast-2", Type: "r5a.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.352}, "r5a.24xlarge": {Region: "ap-southeast-2", Type: "r5a.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.528}, "r5ad.large": {Region: "ap-southeast-2", Type: "r5ad.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.159}, "r5ad.xlarge": {Region: "ap-southeast-2", Type: "r5ad.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.318}, "r5ad.2xlarge": {Region: "ap-southeast-2", Type: "r5ad.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.636}, "r5ad.4xlarge": {Region: "ap-southeast-2", Type: "r5ad.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.272}, "r5ad.8xlarge": {Region: "ap-southeast-2", Type: "r5ad.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.544}, "r5ad.12xlarge": {Region: "ap-southeast-2", Type: "r5ad.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.816}, "r5ad.16xlarge": {Region: "ap-southeast-2", Type: "r5ad.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.088}, "r5ad.24xlarge": {Region: "ap-southeast-2", Type: "r5ad.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.632}, "r5d.large": {Region: "ap-southeast-2", Type: "r5d.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.174}, "r5d.xlarge": {Region: "ap-southeast-2", Type: "r5d.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.348}, "r5d.2xlarge": {Region: "ap-southeast-2", Type: "r5d.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.696}, "r5d.4xlarge": {Region: "ap-southeast-2", Type: "r5d.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.392}, "r5d.8xlarge": {Region: "ap-southeast-2", Type: "r5d.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.784}, "r5d.12xlarge": {Region: "ap-southeast-2", Type: "r5d.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.176}, "r5d.16xlarge": {Region: "ap-southeast-2", Type: "r5d.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.568}, "r5d.24xlarge": {Region: "ap-southeast-2", Type: "r5d.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.352}, "r5d.metal": {Region: "ap-southeast-2", Type: "r5d.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.352}, "r5dn.large": {Region: "ap-southeast-2", Type: "r5dn.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.204}, "r5dn.xlarge": {Region: "ap-southeast-2", Type: "r5dn.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.408}, "r5dn.2xlarge": {Region: "ap-southeast-2", Type: "r5dn.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.816}, "r5dn.4xlarge": {Region: "ap-southeast-2", Type: "r5dn.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.632}, "r5dn.8xlarge": {Region: "ap-southeast-2", Type: "r5dn.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 3.264}, "r5dn.12xlarge": {Region: "ap-southeast-2", Type: "r5dn.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.896}, "r5dn.16xlarge": {Region: "ap-southeast-2", Type: "r5dn.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 6.528}, "r5dn.24xlarge": {Region: "ap-southeast-2", Type: "r5dn.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 9.792}, "r5dn.metal": {Region: "ap-southeast-2", Type: "r5dn.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 9.792}, "r5n.large": {Region: "ap-southeast-2", Type: "r5n.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.181}, "r5n.xlarge": {Region: "ap-southeast-2", Type: "r5n.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.362}, "r5n.2xlarge": {Region: "ap-southeast-2", Type: "r5n.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.724}, "r5n.4xlarge": {Region: "ap-southeast-2", Type: "r5n.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.448}, "r5n.8xlarge": {Region: "ap-southeast-2", Type: "r5n.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.896}, "r5n.12xlarge": {Region: "ap-southeast-2", Type: "r5n.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.344}, "r5n.16xlarge": {Region: "ap-southeast-2", Type: "r5n.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.792}, "r5n.24xlarge": {Region: "ap-southeast-2", Type: "r5n.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.688}, "r5n.metal": {Region: "ap-southeast-2", Type: "r5n.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.688}, "r6g.medium": {Region: "ap-southeast-2", Type: "r6g.medium", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0604}, "r6g.large": {Region: "ap-southeast-2", Type: "r6g.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.1208}, "r6g.xlarge": {Region: "ap-southeast-2", Type: "r6g.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.2416}, "r6g.2xlarge": {Region: "ap-southeast-2", Type: "r6g.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.4832}, "r6g.4xlarge": {Region: "ap-southeast-2", Type: "r6g.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.9664}, "r6g.8xlarge": {Region: "ap-southeast-2", Type: "r6g.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.9328}, "r6g.12xlarge": {Region: "ap-southeast-2", Type: "r6g.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.8992}, "r6g.16xlarge": {Region: "ap-southeast-2", Type: "r6g.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.8656}, "r6g.metal": {Region: "ap-southeast-2", Type: "r6g.metal", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.8656}, "r6gd.medium": {Region: "ap-southeast-2", Type: "r6gd.medium", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0695}, "r6gd.large": {Region: "ap-southeast-2", Type: "r6gd.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.139}, "r6gd.xlarge": {Region: "ap-southeast-2", Type: "r6gd.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.278}, "r6gd.2xlarge": {Region: "ap-southeast-2", Type: "r6gd.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.556}, "r6gd.4xlarge": {Region: "ap-southeast-2", Type: "r6gd.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.112}, "r6gd.8xlarge": {Region: "ap-southeast-2", Type: "r6gd.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.224}, "r6gd.12xlarge": {Region: "ap-southeast-2", Type: "r6gd.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.336}, "r6gd.16xlarge": {Region: "ap-southeast-2", Type: "r6gd.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.448}, "r6gd.metal": {Region: "ap-southeast-2", Type: "r6gd.metal", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.448}, "r6i.large": {Region: "ap-southeast-2", Type: "r6i.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.151}, "r6i.xlarge": {Region: "ap-southeast-2", Type: "r6i.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.302}, "r6i.2xlarge": {Region: "ap-southeast-2", Type: "r6i.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.604}, "r6i.4xlarge": {Region: "ap-southeast-2", Type: "r6i.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.208}, "r6i.8xlarge": {Region: "ap-southeast-2", Type: "r6i.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.416}, "r6i.12xlarge": {Region: "ap-southeast-2", Type: "r6i.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.624}, "r6i.16xlarge": {Region: "ap-southeast-2", Type: "r6i.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.832}, "r6i.24xlarge": {Region: "ap-southeast-2", Type: "r6i.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.248}, "r6i.32xlarge": {Region: "ap-southeast-2", Type: "r6i.32xlarge", Memory: kresource.MustParse("1048576Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 9.664}, "r6i.metal": {Region: "ap-southeast-2", Type: "r6i.metal", Memory: kresource.MustParse("1048576Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 9.664}, "t1.micro": {Region: "ap-southeast-2", Type: "t1.micro", Memory: kresource.MustParse("627Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.02}, "t2.nano": {Region: "ap-southeast-2", Type: "t2.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0073}, "t2.micro": {Region: "ap-southeast-2", Type: "t2.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0146}, "t2.small": {Region: "ap-southeast-2", Type: "t2.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0292}, "t2.medium": {Region: "ap-southeast-2", Type: "t2.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0584}, "t2.large": {Region: "ap-southeast-2", Type: "t2.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.1168}, "t2.xlarge": {Region: "ap-southeast-2", Type: "t2.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.2336}, "t2.2xlarge": {Region: "ap-southeast-2", Type: "t2.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.4672}, "t3.nano": {Region: "ap-southeast-2", Type: "t3.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0066}, "t3.micro": {Region: "ap-southeast-2", Type: "t3.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0132}, "t3.small": {Region: "ap-southeast-2", Type: "t3.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0264}, "t3.medium": {Region: "ap-southeast-2", Type: "t3.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0528}, "t3.large": {Region: "ap-southeast-2", Type: "t3.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.1056}, "t3.xlarge": {Region: "ap-southeast-2", Type: "t3.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.2112}, "t3.2xlarge": {Region: "ap-southeast-2", Type: "t3.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.4224}, "t3a.nano": {Region: "ap-southeast-2", Type: "t3a.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0059}, "t3a.micro": {Region: "ap-southeast-2", Type: "t3a.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0119}, "t3a.small": {Region: "ap-southeast-2", Type: "t3a.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0238}, "t3a.medium": {Region: "ap-southeast-2", Type: "t3a.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0475}, "t3a.large": {Region: "ap-southeast-2", Type: "t3a.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.095}, "t3a.xlarge": {Region: "ap-southeast-2", Type: "t3a.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1901}, "t3a.2xlarge": {Region: "ap-southeast-2", Type: "t3a.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3802}, "t4g.nano": {Region: "ap-southeast-2", Type: "t4g.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0053}, "t4g.micro": {Region: "ap-southeast-2", Type: "t4g.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0106}, "t4g.small": {Region: "ap-southeast-2", Type: "t4g.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0212}, "t4g.medium": {Region: "ap-southeast-2", Type: "t4g.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0424}, "t4g.large": {Region: "ap-southeast-2", Type: "t4g.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0848}, "t4g.xlarge": {Region: "ap-southeast-2", Type: "t4g.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1696}, "t4g.2xlarge": {Region: "ap-southeast-2", Type: "t4g.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3392}, "u-6tb1.56xlarge": {Region: "ap-southeast-2", Type: "u-6tb1.56xlarge", Memory: kresource.MustParse("6291456Mi"), CPU: kresource.MustParse("224"), GPU: 0, Inf: 0, Price: 55.61075}, "u-6tb1.112xlarge": {Region: "ap-southeast-2", Type: "u-6tb1.112xlarge", Memory: kresource.MustParse("6291456Mi"), CPU: kresource.MustParse("448"), GPU: 0, Inf: 0, Price: 65.433}, "x1.16xlarge": {Region: "ap-southeast-2", Type: "x1.16xlarge", Memory: kresource.MustParse("999424Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 9.671}, "x1.32xlarge": {Region: "ap-southeast-2", Type: "x1.32xlarge", Memory: kresource.MustParse("1998848Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 19.341}, "x1e.xlarge": {Region: "ap-southeast-2", Type: "x1e.xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 1.209}, "x1e.2xlarge": {Region: "ap-southeast-2", Type: "x1e.2xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 2.418}, "x1e.4xlarge": {Region: "ap-southeast-2", Type: "x1e.4xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 4.836}, "x1e.8xlarge": {Region: "ap-southeast-2", Type: "x1e.8xlarge", Memory: kresource.MustParse("999424Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 9.672}, "x1e.16xlarge": {Region: "ap-southeast-2", Type: "x1e.16xlarge", Memory: kresource.MustParse("1998848Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 19.344}, "x1e.32xlarge": {Region: "ap-southeast-2", Type: "x1e.32xlarge", Memory: kresource.MustParse("3997696Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 38.688}, "z1d.large": {Region: "ap-southeast-2", Type: "z1d.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.226}, "z1d.xlarge": {Region: "ap-southeast-2", Type: "z1d.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.452}, "z1d.2xlarge": {Region: "ap-southeast-2", Type: "z1d.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.904}, "z1d.3xlarge": {Region: "ap-southeast-2", Type: "z1d.3xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("12"), GPU: 0, Inf: 0, Price: 1.356}, "z1d.6xlarge": {Region: "ap-southeast-2", Type: "z1d.6xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 0, Price: 2.712}, "z1d.12xlarge": {Region: "ap-southeast-2", Type: "z1d.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 5.424}, "z1d.metal": {Region: "ap-southeast-2", Type: "z1d.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 5.424}, }, "ca-central-1": { "c4.large": {Region: "ca-central-1", Type: "c4.large", Memory: kresource.MustParse("3840Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.11}, "c4.xlarge": {Region: "ca-central-1", Type: "c4.xlarge", Memory: kresource.MustParse("7680Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.218}, "c4.2xlarge": {Region: "ca-central-1", Type: "c4.2xlarge", Memory: kresource.MustParse("15360Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.438}, "c4.4xlarge": {Region: "ca-central-1", Type: "c4.4xlarge", Memory: kresource.MustParse("30720Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.876}, "c4.8xlarge": {Region: "ca-central-1", Type: "c4.8xlarge", Memory: kresource.MustParse("61440Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 1.75}, "c5.large": {Region: "ca-central-1", Type: "c5.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.093}, "c5.xlarge": {Region: "ca-central-1", Type: "c5.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.186}, "c5.2xlarge": {Region: "ca-central-1", Type: "c5.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.372}, "c5.4xlarge": {Region: "ca-central-1", Type: "c5.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.744}, "c5.9xlarge": {Region: "ca-central-1", Type: "c5.9xlarge", Memory: kresource.MustParse("73728Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 1.674}, "c5.12xlarge": {Region: "ca-central-1", Type: "c5.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.232}, "c5.18xlarge": {Region: "ca-central-1", Type: "c5.18xlarge", Memory: kresource.MustParse("147456Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 3.348}, "c5.24xlarge": {Region: "ca-central-1", Type: "c5.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.464}, "c5.metal": {Region: "ca-central-1", Type: "c5.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.464}, "c5a.large": {Region: "ca-central-1", Type: "c5a.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.084}, "c5a.xlarge": {Region: "ca-central-1", Type: "c5a.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.168}, "c5a.2xlarge": {Region: "ca-central-1", Type: "c5a.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.336}, "c5a.4xlarge": {Region: "ca-central-1", Type: "c5a.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.672}, "c5a.8xlarge": {Region: "ca-central-1", Type: "c5a.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.344}, "c5a.12xlarge": {Region: "ca-central-1", Type: "c5a.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.016}, "c5a.16xlarge": {Region: "ca-central-1", Type: "c5a.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.688}, "c5a.24xlarge": {Region: "ca-central-1", Type: "c5a.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.032}, "c5d.large": {Region: "ca-central-1", Type: "c5d.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.106}, "c5d.xlarge": {Region: "ca-central-1", Type: "c5d.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.212}, "c5d.2xlarge": {Region: "ca-central-1", Type: "c5d.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.424}, "c5d.4xlarge": {Region: "ca-central-1", Type: "c5d.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.848}, "c5d.9xlarge": {Region: "ca-central-1", Type: "c5d.9xlarge", Memory: kresource.MustParse("73728Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 1.908}, "c5d.12xlarge": {Region: "ca-central-1", Type: "c5d.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.544}, "c5d.18xlarge": {Region: "ca-central-1", Type: "c5d.18xlarge", Memory: kresource.MustParse("147456Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 3.816}, "c5d.24xlarge": {Region: "ca-central-1", Type: "c5d.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.088}, "c5d.metal": {Region: "ca-central-1", Type: "c5d.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.088}, "c5n.large": {Region: "ca-central-1", Type: "c5n.large", Memory: kresource.MustParse("5376Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.118}, "c5n.xlarge": {Region: "ca-central-1", Type: "c5n.xlarge", Memory: kresource.MustParse("10752Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.236}, "c5n.2xlarge": {Region: "ca-central-1", Type: "c5n.2xlarge", Memory: kresource.MustParse("21504Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.472}, "c5n.4xlarge": {Region: "ca-central-1", Type: "c5n.4xlarge", Memory: kresource.MustParse("43008Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.944}, "c5n.9xlarge": {Region: "ca-central-1", Type: "c5n.9xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 2.124}, "c5n.18xlarge": {Region: "ca-central-1", Type: "c5n.18xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 4.248}, "c5n.metal": {Region: "ca-central-1", Type: "c5n.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 4.248}, "c6g.medium": {Region: "ca-central-1", Type: "c6g.medium", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0372}, "c6g.large": {Region: "ca-central-1", Type: "c6g.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0744}, "c6g.xlarge": {Region: "ca-central-1", Type: "c6g.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1488}, "c6g.2xlarge": {Region: "ca-central-1", Type: "c6g.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.2976}, "c6g.4xlarge": {Region: "ca-central-1", Type: "c6g.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.5952}, "c6g.8xlarge": {Region: "ca-central-1", Type: "c6g.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.1904}, "c6g.12xlarge": {Region: "ca-central-1", Type: "c6g.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 1.7856}, "c6g.16xlarge": {Region: "ca-central-1", Type: "c6g.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.3808}, "c6g.metal": {Region: "ca-central-1", Type: "c6g.metal", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.3808}, "c6gd.medium": {Region: "ca-central-1", Type: "c6gd.medium", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0424}, "c6gd.large": {Region: "ca-central-1", Type: "c6gd.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0848}, "c6gd.xlarge": {Region: "ca-central-1", Type: "c6gd.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1696}, "c6gd.2xlarge": {Region: "ca-central-1", Type: "c6gd.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3392}, "c6gd.4xlarge": {Region: "ca-central-1", Type: "c6gd.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.6784}, "c6gd.8xlarge": {Region: "ca-central-1", Type: "c6gd.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.3568}, "c6gd.12xlarge": {Region: "ca-central-1", Type: "c6gd.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.0352}, "c6gd.16xlarge": {Region: "ca-central-1", Type: "c6gd.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.7136}, "c6gd.metal": {Region: "ca-central-1", Type: "c6gd.metal", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.7136}, "c6gn.medium": {Region: "ca-central-1", Type: "c6gn.medium", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0473}, "c6gn.large": {Region: "ca-central-1", Type: "c6gn.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0946}, "c6gn.xlarge": {Region: "ca-central-1", Type: "c6gn.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1892}, "c6gn.2xlarge": {Region: "ca-central-1", Type: "c6gn.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3784}, "c6gn.4xlarge": {Region: "ca-central-1", Type: "c6gn.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.7568}, "c6gn.8xlarge": {Region: "ca-central-1", Type: "c6gn.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.5136}, "c6gn.12xlarge": {Region: "ca-central-1", Type: "c6gn.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.2704}, "c6gn.16xlarge": {Region: "ca-central-1", Type: "c6gn.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.0272}, "c6i.large": {Region: "ca-central-1", Type: "c6i.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.093}, "c6i.xlarge": {Region: "ca-central-1", Type: "c6i.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.186}, "c6i.2xlarge": {Region: "ca-central-1", Type: "c6i.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.372}, "c6i.4xlarge": {Region: "ca-central-1", Type: "c6i.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.744}, "c6i.8xlarge": {Region: "ca-central-1", Type: "c6i.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.488}, "c6i.12xlarge": {Region: "ca-central-1", Type: "c6i.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.232}, "c6i.16xlarge": {Region: "ca-central-1", Type: "c6i.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.976}, "c6i.24xlarge": {Region: "ca-central-1", Type: "c6i.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.464}, "c6i.32xlarge": {Region: "ca-central-1", Type: "c6i.32xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 5.952}, "c6i.metal": {Region: "ca-central-1", Type: "c6i.metal", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 5.952}, "d2.xlarge": {Region: "ca-central-1", Type: "d2.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.759}, "d2.2xlarge": {Region: "ca-central-1", Type: "d2.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.518}, "d2.4xlarge": {Region: "ca-central-1", Type: "d2.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 3.036}, "d2.8xlarge": {Region: "ca-central-1", Type: "d2.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 6.072}, "g3.4xlarge": {Region: "ca-central-1", Type: "g3.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 1, Inf: 0, Price: 1.416}, "g3.8xlarge": {Region: "ca-central-1", Type: "g3.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 2, Inf: 0, Price: 2.832}, "g3.16xlarge": {Region: "ca-central-1", Type: "g3.16xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("64"), GPU: 4, Inf: 0, Price: 5.664}, "g4ad.xlarge": {Region: "ca-central-1", Type: "g4ad.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 1, Inf: 0, Price: 0.42263}, "g4ad.2xlarge": {Region: "ca-central-1", Type: "g4ad.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 1, Inf: 0, Price: 0.60421}, "g4ad.4xlarge": {Region: "ca-central-1", Type: "g4ad.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 1, Inf: 0, Price: 0.968}, "g4ad.8xlarge": {Region: "ca-central-1", Type: "g4ad.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 2, Inf: 0, Price: 1.936}, "g4ad.16xlarge": {Region: "ca-central-1", Type: "g4ad.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 4, Inf: 0, Price: 3.872}, "g4dn.xlarge": {Region: "ca-central-1", Type: "g4dn.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 1, Inf: 0, Price: 0.584}, "g4dn.2xlarge": {Region: "ca-central-1", Type: "g4dn.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 1, Inf: 0, Price: 0.835}, "g4dn.4xlarge": {Region: "ca-central-1", Type: "g4dn.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 1, Inf: 0, Price: 1.337}, "g4dn.8xlarge": {Region: "ca-central-1", Type: "g4dn.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 1, Inf: 0, Price: 2.416}, "g4dn.12xlarge": {Region: "ca-central-1", Type: "g4dn.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 4, Inf: 0, Price: 4.343}, "g4dn.16xlarge": {Region: "ca-central-1", Type: "g4dn.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 1, Inf: 0, Price: 4.832}, "g4dn.metal": {Region: "ca-central-1", Type: "g4dn.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 8, Inf: 0, Price: 8.687}, "i3.large": {Region: "ca-central-1", Type: "i3.large", Memory: kresource.MustParse("15616Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.172}, "i3.xlarge": {Region: "ca-central-1", Type: "i3.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.344}, "i3.2xlarge": {Region: "ca-central-1", Type: "i3.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.688}, "i3.4xlarge": {Region: "ca-central-1", Type: "i3.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.376}, "i3.8xlarge": {Region: "ca-central-1", Type: "i3.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.752}, "i3.16xlarge": {Region: "ca-central-1", Type: "i3.16xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.504}, "i3.metal": {Region: "ca-central-1", Type: "i3.metal", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.504}, "i3en.large": {Region: "ca-central-1", Type: "i3en.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.25}, "i3en.xlarge": {Region: "ca-central-1", Type: "i3en.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.5}, "i3en.2xlarge": {Region: "ca-central-1", Type: "i3en.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.0}, "i3en.3xlarge": {Region: "ca-central-1", Type: "i3en.3xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("12"), GPU: 0, Inf: 0, Price: 1.5}, "i3en.6xlarge": {Region: "ca-central-1", Type: "i3en.6xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 0, Price: 3.0}, "i3en.12xlarge": {Region: "ca-central-1", Type: "i3en.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 6.0}, "i3en.24xlarge": {Region: "ca-central-1", Type: "i3en.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 12.0}, "i3en.metal": {Region: "ca-central-1", Type: "i3en.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 12.0}, "inf1.xlarge": {Region: "ca-central-1", Type: "inf1.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 1, Price: 0.254}, "inf1.2xlarge": {Region: "ca-central-1", Type: "inf1.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 1, Price: 0.403}, "inf1.6xlarge": {Region: "ca-central-1", Type: "inf1.6xlarge", Memory: kresource.MustParse("49152Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 4, Price: 1.315}, "inf1.24xlarge": {Region: "ca-central-1", Type: "inf1.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 16, Price: 5.26}, "m4.large": {Region: "ca-central-1", Type: "m4.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.111}, "m4.xlarge": {Region: "ca-central-1", Type: "m4.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.222}, "m4.2xlarge": {Region: "ca-central-1", Type: "m4.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.444}, "m4.4xlarge": {Region: "ca-central-1", Type: "m4.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.888}, "m4.10xlarge": {Region: "ca-central-1", Type: "m4.10xlarge", Memory: kresource.MustParse("163840Mi"), CPU: kresource.MustParse("40"), GPU: 0, Inf: 0, Price: 2.22}, "m4.16xlarge": {Region: "ca-central-1", Type: "m4.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.552}, "m5.large": {Region: "ca-central-1", Type: "m5.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.107}, "m5.xlarge": {Region: "ca-central-1", Type: "m5.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.214}, "m5.2xlarge": {Region: "ca-central-1", Type: "m5.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.428}, "m5.4xlarge": {Region: "ca-central-1", Type: "m5.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.856}, "m5.8xlarge": {Region: "ca-central-1", Type: "m5.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.712}, "m5.12xlarge": {Region: "ca-central-1", Type: "m5.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.568}, "m5.16xlarge": {Region: "ca-central-1", Type: "m5.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.424}, "m5.24xlarge": {Region: "ca-central-1", Type: "m5.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.136}, "m5.metal": {Region: "ca-central-1", Type: "m5.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.136}, "m5a.large": {Region: "ca-central-1", Type: "m5a.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.096}, "m5a.xlarge": {Region: "ca-central-1", Type: "m5a.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.192}, "m5a.2xlarge": {Region: "ca-central-1", Type: "m5a.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.384}, "m5a.4xlarge": {Region: "ca-central-1", Type: "m5a.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.768}, "m5a.8xlarge": {Region: "ca-central-1", Type: "m5a.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.536}, "m5a.12xlarge": {Region: "ca-central-1", Type: "m5a.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.304}, "m5a.16xlarge": {Region: "ca-central-1", Type: "m5a.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.072}, "m5a.24xlarge": {Region: "ca-central-1", Type: "m5a.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.608}, "m5ad.large": {Region: "ca-central-1", Type: "m5ad.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.115}, "m5ad.xlarge": {Region: "ca-central-1", Type: "m5ad.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.23}, "m5ad.2xlarge": {Region: "ca-central-1", Type: "m5ad.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.46}, "m5ad.4xlarge": {Region: "ca-central-1", Type: "m5ad.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.92}, "m5ad.8xlarge": {Region: "ca-central-1", Type: "m5ad.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.84}, "m5ad.12xlarge": {Region: "ca-central-1", Type: "m5ad.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.76}, "m5ad.16xlarge": {Region: "ca-central-1", Type: "m5ad.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.68}, "m5ad.24xlarge": {Region: "ca-central-1", Type: "m5ad.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.52}, "m5d.large": {Region: "ca-central-1", Type: "m5d.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.126}, "m5d.xlarge": {Region: "ca-central-1", Type: "m5d.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.252}, "m5d.2xlarge": {Region: "ca-central-1", Type: "m5d.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.504}, "m5d.4xlarge": {Region: "ca-central-1", Type: "m5d.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.008}, "m5d.8xlarge": {Region: "ca-central-1", Type: "m5d.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.016}, "m5d.12xlarge": {Region: "ca-central-1", Type: "m5d.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.024}, "m5d.16xlarge": {Region: "ca-central-1", Type: "m5d.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.032}, "m5d.24xlarge": {Region: "ca-central-1", Type: "m5d.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.048}, "m5d.metal": {Region: "ca-central-1", Type: "m5d.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.048}, "m6g.medium": {Region: "ca-central-1", Type: "m6g.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0428}, "m6g.large": {Region: "ca-central-1", Type: "m6g.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0856}, "m6g.xlarge": {Region: "ca-central-1", Type: "m6g.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1712}, "m6g.2xlarge": {Region: "ca-central-1", Type: "m6g.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3424}, "m6g.4xlarge": {Region: "ca-central-1", Type: "m6g.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.6848}, "m6g.8xlarge": {Region: "ca-central-1", Type: "m6g.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.3696}, "m6g.12xlarge": {Region: "ca-central-1", Type: "m6g.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.0544}, "m6g.16xlarge": {Region: "ca-central-1", Type: "m6g.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.7392}, "m6g.metal": {Region: "ca-central-1", Type: "m6g.metal", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.7392}, "m6i.large": {Region: "ca-central-1", Type: "m6i.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.107}, "m6i.xlarge": {Region: "ca-central-1", Type: "m6i.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.214}, "m6i.2xlarge": {Region: "ca-central-1", Type: "m6i.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.428}, "m6i.4xlarge": {Region: "ca-central-1", Type: "m6i.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.856}, "m6i.8xlarge": {Region: "ca-central-1", Type: "m6i.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.712}, "m6i.12xlarge": {Region: "ca-central-1", Type: "m6i.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.568}, "m6i.16xlarge": {Region: "ca-central-1", Type: "m6i.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.424}, "m6i.24xlarge": {Region: "ca-central-1", Type: "m6i.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.136}, "m6i.32xlarge": {Region: "ca-central-1", Type: "m6i.32xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 6.848}, "m6i.metal": {Region: "ca-central-1", Type: "m6i.metal", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 6.848}, "p3.2xlarge": {Region: "ca-central-1", Type: "p3.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 1, Inf: 0, Price: 3.366}, "p3.8xlarge": {Region: "ca-central-1", Type: "p3.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 4, Inf: 0, Price: 13.464}, "p3.16xlarge": {Region: "ca-central-1", Type: "p3.16xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("64"), GPU: 8, Inf: 0, Price: 26.928}, "r4.large": {Region: "ca-central-1", Type: "r4.large", Memory: kresource.MustParse("15616Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.146}, "r4.xlarge": {Region: "ca-central-1", Type: "r4.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.292}, "r4.2xlarge": {Region: "ca-central-1", Type: "r4.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.584}, "r4.4xlarge": {Region: "ca-central-1", Type: "r4.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.168}, "r4.8xlarge": {Region: "ca-central-1", Type: "r4.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.336}, "r4.16xlarge": {Region: "ca-central-1", Type: "r4.16xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.672}, "r5.large": {Region: "ca-central-1", Type: "r5.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.138}, "r5.xlarge": {Region: "ca-central-1", Type: "r5.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.276}, "r5.2xlarge": {Region: "ca-central-1", Type: "r5.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.552}, "r5.4xlarge": {Region: "ca-central-1", Type: "r5.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.104}, "r5.8xlarge": {Region: "ca-central-1", Type: "r5.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.208}, "r5.12xlarge": {Region: "ca-central-1", Type: "r5.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.312}, "r5.16xlarge": {Region: "ca-central-1", Type: "r5.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.416}, "r5.24xlarge": {Region: "ca-central-1", Type: "r5.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.624}, "r5.metal": {Region: "ca-central-1", Type: "r5.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.624}, "r5a.large": {Region: "ca-central-1", Type: "r5a.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.124}, "r5a.xlarge": {Region: "ca-central-1", Type: "r5a.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.248}, "r5a.2xlarge": {Region: "ca-central-1", Type: "r5a.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.496}, "r5a.4xlarge": {Region: "ca-central-1", Type: "r5a.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.992}, "r5a.8xlarge": {Region: "ca-central-1", Type: "r5a.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.984}, "r5a.12xlarge": {Region: "ca-central-1", Type: "r5a.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.976}, "r5a.16xlarge": {Region: "ca-central-1", Type: "r5a.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.968}, "r5a.24xlarge": {Region: "ca-central-1", Type: "r5a.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.952}, "r5ad.large": {Region: "ca-central-1", Type: "r5ad.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.144}, "r5ad.xlarge": {Region: "ca-central-1", Type: "r5ad.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.288}, "r5ad.2xlarge": {Region: "ca-central-1", Type: "r5ad.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.576}, "r5ad.4xlarge": {Region: "ca-central-1", Type: "r5ad.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.152}, "r5ad.8xlarge": {Region: "ca-central-1", Type: "r5ad.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.304}, "r5ad.12xlarge": {Region: "ca-central-1", Type: "r5ad.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.456}, "r5ad.16xlarge": {Region: "ca-central-1", Type: "r5ad.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.608}, "r5ad.24xlarge": {Region: "ca-central-1", Type: "r5ad.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.912}, "r5d.large": {Region: "ca-central-1", Type: "r5d.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.158}, "r5d.xlarge": {Region: "ca-central-1", Type: "r5d.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.316}, "r5d.2xlarge": {Region: "ca-central-1", Type: "r5d.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.632}, "r5d.4xlarge": {Region: "ca-central-1", Type: "r5d.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.264}, "r5d.8xlarge": {Region: "ca-central-1", Type: "r5d.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.528}, "r5d.12xlarge": {Region: "ca-central-1", Type: "r5d.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.792}, "r5d.16xlarge": {Region: "ca-central-1", Type: "r5d.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.056}, "r5d.24xlarge": {Region: "ca-central-1", Type: "r5d.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.584}, "r5d.metal": {Region: "ca-central-1", Type: "r5d.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.584}, "r5n.large": {Region: "ca-central-1", Type: "r5n.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.163}, "r5n.xlarge": {Region: "ca-central-1", Type: "r5n.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.326}, "r5n.2xlarge": {Region: "ca-central-1", Type: "r5n.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.652}, "r5n.4xlarge": {Region: "ca-central-1", Type: "r5n.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.304}, "r5n.8xlarge": {Region: "ca-central-1", Type: "r5n.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.608}, "r5n.12xlarge": {Region: "ca-central-1", Type: "r5n.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.912}, "r5n.16xlarge": {Region: "ca-central-1", Type: "r5n.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.216}, "r5n.24xlarge": {Region: "ca-central-1", Type: "r5n.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.824}, "r5n.metal": {Region: "ca-central-1", Type: "r5n.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.824}, "r6g.medium": {Region: "ca-central-1", Type: "r6g.medium", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0552}, "r6g.large": {Region: "ca-central-1", Type: "r6g.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.1104}, "r6g.xlarge": {Region: "ca-central-1", Type: "r6g.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.2208}, "r6g.2xlarge": {Region: "ca-central-1", Type: "r6g.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.4416}, "r6g.4xlarge": {Region: "ca-central-1", Type: "r6g.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.8832}, "r6g.8xlarge": {Region: "ca-central-1", Type: "r6g.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.7664}, "r6g.12xlarge": {Region: "ca-central-1", Type: "r6g.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.6496}, "r6g.16xlarge": {Region: "ca-central-1", Type: "r6g.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.5328}, "r6g.metal": {Region: "ca-central-1", Type: "r6g.metal", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.5328}, "r6gd.medium": {Region: "ca-central-1", Type: "r6gd.medium", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0632}, "r6gd.large": {Region: "ca-central-1", Type: "r6gd.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.1264}, "r6gd.xlarge": {Region: "ca-central-1", Type: "r6gd.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.2528}, "r6gd.2xlarge": {Region: "ca-central-1", Type: "r6gd.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.5056}, "r6gd.4xlarge": {Region: "ca-central-1", Type: "r6gd.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.0112}, "r6gd.8xlarge": {Region: "ca-central-1", Type: "r6gd.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.0224}, "r6gd.12xlarge": {Region: "ca-central-1", Type: "r6gd.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.0336}, "r6gd.16xlarge": {Region: "ca-central-1", Type: "r6gd.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.0448}, "r6gd.metal": {Region: "ca-central-1", Type: "r6gd.metal", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.0448}, "r6i.large": {Region: "ca-central-1", Type: "r6i.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.138}, "r6i.xlarge": {Region: "ca-central-1", Type: "r6i.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.276}, "r6i.2xlarge": {Region: "ca-central-1", Type: "r6i.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.552}, "r6i.4xlarge": {Region: "ca-central-1", Type: "r6i.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.104}, "r6i.8xlarge": {Region: "ca-central-1", Type: "r6i.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.208}, "r6i.12xlarge": {Region: "ca-central-1", Type: "r6i.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.312}, "r6i.16xlarge": {Region: "ca-central-1", Type: "r6i.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.416}, "r6i.24xlarge": {Region: "ca-central-1", Type: "r6i.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.624}, "r6i.32xlarge": {Region: "ca-central-1", Type: "r6i.32xlarge", Memory: kresource.MustParse("1048576Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 8.832}, "r6i.metal": {Region: "ca-central-1", Type: "r6i.metal", Memory: kresource.MustParse("1048576Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 8.832}, "t2.nano": {Region: "ca-central-1", Type: "t2.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0064}, "t2.micro": {Region: "ca-central-1", Type: "t2.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0128}, "t2.small": {Region: "ca-central-1", Type: "t2.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0256}, "t2.medium": {Region: "ca-central-1", Type: "t2.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0512}, "t2.large": {Region: "ca-central-1", Type: "t2.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.1024}, "t2.xlarge": {Region: "ca-central-1", Type: "t2.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.2048}, "t2.2xlarge": {Region: "ca-central-1", Type: "t2.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.4096}, "t3.nano": {Region: "ca-central-1", Type: "t3.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0058}, "t3.micro": {Region: "ca-central-1", Type: "t3.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0116}, "t3.small": {Region: "ca-central-1", Type: "t3.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0232}, "t3.medium": {Region: "ca-central-1", Type: "t3.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0464}, "t3.large": {Region: "ca-central-1", Type: "t3.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0928}, "t3.xlarge": {Region: "ca-central-1", Type: "t3.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1856}, "t3.2xlarge": {Region: "ca-central-1", Type: "t3.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3712}, "t3a.nano": {Region: "ca-central-1", Type: "t3a.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0052}, "t3a.micro": {Region: "ca-central-1", Type: "t3a.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0104}, "t3a.small": {Region: "ca-central-1", Type: "t3a.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0209}, "t3a.medium": {Region: "ca-central-1", Type: "t3a.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0418}, "t3a.large": {Region: "ca-central-1", Type: "t3a.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0835}, "t3a.xlarge": {Region: "ca-central-1", Type: "t3a.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.167}, "t3a.2xlarge": {Region: "ca-central-1", Type: "t3a.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3341}, "t4g.nano": {Region: "ca-central-1", Type: "t4g.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0046}, "t4g.micro": {Region: "ca-central-1", Type: "t4g.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0092}, "t4g.small": {Region: "ca-central-1", Type: "t4g.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0184}, "t4g.medium": {Region: "ca-central-1", Type: "t4g.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0368}, "t4g.large": {Region: "ca-central-1", Type: "t4g.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0736}, "t4g.xlarge": {Region: "ca-central-1", Type: "t4g.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1472}, "t4g.2xlarge": {Region: "ca-central-1", Type: "t4g.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.2944}, "x1.16xlarge": {Region: "ca-central-1", Type: "x1.16xlarge", Memory: kresource.MustParse("999424Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 7.336}, "x1.32xlarge": {Region: "ca-central-1", Type: "x1.32xlarge", Memory: kresource.MustParse("1998848Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 14.672}, "x1e.xlarge": {Region: "ca-central-1", Type: "x1e.xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.917}, "x1e.2xlarge": {Region: "ca-central-1", Type: "x1e.2xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.834}, "x1e.4xlarge": {Region: "ca-central-1", Type: "x1e.4xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 3.667}, "x1e.8xlarge": {Region: "ca-central-1", Type: "x1e.8xlarge", Memory: kresource.MustParse("999424Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 7.334}, "x1e.16xlarge": {Region: "ca-central-1", Type: "x1e.16xlarge", Memory: kresource.MustParse("1998848Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 14.669}, "x1e.32xlarge": {Region: "ca-central-1", Type: "x1e.32xlarge", Memory: kresource.MustParse("3997696Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 29.338}, }, "eu-central-1": { "a1.medium": {Region: "eu-central-1", Type: "a1.medium", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0291}, "a1.large": {Region: "eu-central-1", Type: "a1.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0582}, "a1.xlarge": {Region: "eu-central-1", Type: "a1.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1164}, "a1.2xlarge": {Region: "eu-central-1", Type: "a1.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.2328}, "a1.4xlarge": {Region: "eu-central-1", Type: "a1.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.4656}, "a1.metal": {Region: "eu-central-1", Type: "a1.metal", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.466}, "c3.large": {Region: "eu-central-1", Type: "c3.large", Memory: kresource.MustParse("3840Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.129}, "c3.xlarge": {Region: "eu-central-1", Type: "c3.xlarge", Memory: kresource.MustParse("7680Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.258}, "c3.2xlarge": {Region: "eu-central-1", Type: "c3.2xlarge", Memory: kresource.MustParse("15360Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.516}, "c3.4xlarge": {Region: "eu-central-1", Type: "c3.4xlarge", Memory: kresource.MustParse("30720Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.032}, "c3.8xlarge": {Region: "eu-central-1", Type: "c3.8xlarge", Memory: kresource.MustParse("61440Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.064}, "c4.large": {Region: "eu-central-1", Type: "c4.large", Memory: kresource.MustParse("3840Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.114}, "c4.xlarge": {Region: "eu-central-1", Type: "c4.xlarge", Memory: kresource.MustParse("7680Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.227}, "c4.2xlarge": {Region: "eu-central-1", Type: "c4.2xlarge", Memory: kresource.MustParse("15360Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.454}, "c4.4xlarge": {Region: "eu-central-1", Type: "c4.4xlarge", Memory: kresource.MustParse("30720Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.909}, "c4.8xlarge": {Region: "eu-central-1", Type: "c4.8xlarge", Memory: kresource.MustParse("61440Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 1.817}, "c5.large": {Region: "eu-central-1", Type: "c5.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.097}, "c5.xlarge": {Region: "eu-central-1", Type: "c5.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.194}, "c5.2xlarge": {Region: "eu-central-1", Type: "c5.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.388}, "c5.4xlarge": {Region: "eu-central-1", Type: "c5.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.776}, "c5.9xlarge": {Region: "eu-central-1", Type: "c5.9xlarge", Memory: kresource.MustParse("73728Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 1.746}, "c5.12xlarge": {Region: "eu-central-1", Type: "c5.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.328}, "c5.18xlarge": {Region: "eu-central-1", Type: "c5.18xlarge", Memory: kresource.MustParse("147456Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 3.492}, "c5.24xlarge": {Region: "eu-central-1", Type: "c5.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.656}, "c5.metal": {Region: "eu-central-1", Type: "c5.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.656}, "c5a.large": {Region: "eu-central-1", Type: "c5a.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.087}, "c5a.xlarge": {Region: "eu-central-1", Type: "c5a.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.174}, "c5a.2xlarge": {Region: "eu-central-1", Type: "c5a.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.348}, "c5a.4xlarge": {Region: "eu-central-1", Type: "c5a.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.696}, "c5a.8xlarge": {Region: "eu-central-1", Type: "c5a.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.392}, "c5a.12xlarge": {Region: "eu-central-1", Type: "c5a.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.088}, "c5a.16xlarge": {Region: "eu-central-1", Type: "c5a.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.784}, "c5a.24xlarge": {Region: "eu-central-1", Type: "c5a.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.176}, "c5ad.large": {Region: "eu-central-1", Type: "c5ad.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.1}, "c5ad.xlarge": {Region: "eu-central-1", Type: "c5ad.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.2}, "c5ad.2xlarge": {Region: "eu-central-1", Type: "c5ad.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.4}, "c5ad.4xlarge": {Region: "eu-central-1", Type: "c5ad.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.8}, "c5ad.8xlarge": {Region: "eu-central-1", Type: "c5ad.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.6}, "c5ad.12xlarge": {Region: "eu-central-1", Type: "c5ad.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.4}, "c5ad.16xlarge": {Region: "eu-central-1", Type: "c5ad.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.2}, "c5ad.24xlarge": {Region: "eu-central-1", Type: "c5ad.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.8}, "c5d.large": {Region: "eu-central-1", Type: "c5d.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.111}, "c5d.xlarge": {Region: "eu-central-1", Type: "c5d.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.222}, "c5d.2xlarge": {Region: "eu-central-1", Type: "c5d.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.444}, "c5d.4xlarge": {Region: "eu-central-1", Type: "c5d.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.888}, "c5d.9xlarge": {Region: "eu-central-1", Type: "c5d.9xlarge", Memory: kresource.MustParse("73728Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 1.998}, "c5d.12xlarge": {Region: "eu-central-1", Type: "c5d.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.664}, "c5d.18xlarge": {Region: "eu-central-1", Type: "c5d.18xlarge", Memory: kresource.MustParse("147456Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 3.996}, "c5d.24xlarge": {Region: "eu-central-1", Type: "c5d.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.328}, "c5d.metal": {Region: "eu-central-1", Type: "c5d.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.328}, "c5n.large": {Region: "eu-central-1", Type: "c5n.large", Memory: kresource.MustParse("5376Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.123}, "c5n.xlarge": {Region: "eu-central-1", Type: "c5n.xlarge", Memory: kresource.MustParse("10752Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.246}, "c5n.2xlarge": {Region: "eu-central-1", Type: "c5n.2xlarge", Memory: kresource.MustParse("21504Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.492}, "c5n.4xlarge": {Region: "eu-central-1", Type: "c5n.4xlarge", Memory: kresource.MustParse("43008Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.984}, "c5n.9xlarge": {Region: "eu-central-1", Type: "c5n.9xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 2.214}, "c5n.18xlarge": {Region: "eu-central-1", Type: "c5n.18xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 4.428}, "c5n.metal": {Region: "eu-central-1", Type: "c5n.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 4.428}, "c6g.medium": {Region: "eu-central-1", Type: "c6g.medium", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0388}, "c6g.large": {Region: "eu-central-1", Type: "c6g.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0776}, "c6g.xlarge": {Region: "eu-central-1", Type: "c6g.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1552}, "c6g.2xlarge": {Region: "eu-central-1", Type: "c6g.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3104}, "c6g.4xlarge": {Region: "eu-central-1", Type: "c6g.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.6208}, "c6g.8xlarge": {Region: "eu-central-1", Type: "c6g.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.2416}, "c6g.12xlarge": {Region: "eu-central-1", Type: "c6g.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 1.8624}, "c6g.16xlarge": {Region: "eu-central-1", Type: "c6g.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.4832}, "c6g.metal": {Region: "eu-central-1", Type: "c6g.metal", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.4832}, "c6gd.medium": {Region: "eu-central-1", Type: "c6gd.medium", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0445}, "c6gd.large": {Region: "eu-central-1", Type: "c6gd.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.089}, "c6gd.xlarge": {Region: "eu-central-1", Type: "c6gd.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.178}, "c6gd.2xlarge": {Region: "eu-central-1", Type: "c6gd.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.356}, "c6gd.4xlarge": {Region: "eu-central-1", Type: "c6gd.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.712}, "c6gd.8xlarge": {Region: "eu-central-1", Type: "c6gd.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.424}, "c6gd.12xlarge": {Region: "eu-central-1", Type: "c6gd.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.136}, "c6gd.16xlarge": {Region: "eu-central-1", Type: "c6gd.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.848}, "c6gd.metal": {Region: "eu-central-1", Type: "c6gd.metal", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.848}, "c6gn.medium": {Region: "eu-central-1", Type: "c6gn.medium", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0493}, "c6gn.large": {Region: "eu-central-1", Type: "c6gn.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0986}, "c6gn.xlarge": {Region: "eu-central-1", Type: "c6gn.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1972}, "c6gn.2xlarge": {Region: "eu-central-1", Type: "c6gn.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3944}, "c6gn.4xlarge": {Region: "eu-central-1", Type: "c6gn.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.7888}, "c6gn.8xlarge": {Region: "eu-central-1", Type: "c6gn.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.5776}, "c6gn.12xlarge": {Region: "eu-central-1", Type: "c6gn.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.3664}, "c6gn.16xlarge": {Region: "eu-central-1", Type: "c6gn.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.1552}, "d2.xlarge": {Region: "eu-central-1", Type: "d2.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.794}, "d2.2xlarge": {Region: "eu-central-1", Type: "d2.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.588}, "d2.4xlarge": {Region: "eu-central-1", Type: "d2.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 3.176}, "d2.8xlarge": {Region: "eu-central-1", Type: "d2.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 6.352}, "d3.xlarge": {Region: "eu-central-1", Type: "d3.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.658}, "d3.2xlarge": {Region: "eu-central-1", Type: "d3.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.316}, "d3.4xlarge": {Region: "eu-central-1", Type: "d3.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 2.632}, "d3.8xlarge": {Region: "eu-central-1", Type: "d3.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 5.26448}, "f1.2xlarge": {Region: "eu-central-1", Type: "f1.2xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.734}, "f1.4xlarge": {Region: "eu-central-1", Type: "f1.4xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 3.468}, "f1.16xlarge": {Region: "eu-central-1", Type: "f1.16xlarge", Memory: kresource.MustParse("999424Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 13.872}, "g2.2xlarge": {Region: "eu-central-1", Type: "g2.2xlarge", Memory: kresource.MustParse("15360Mi"), CPU: kresource.MustParse("8"), GPU: 1, Inf: 0, Price: 0.772}, "g2.8xlarge": {Region: "eu-central-1", Type: "g2.8xlarge", Memory: kresource.MustParse("61440Mi"), CPU: kresource.MustParse("32"), GPU: 4, Inf: 0, Price: 3.088}, "g3.4xlarge": {Region: "eu-central-1", Type: "g3.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 1, Inf: 0, Price: 1.425}, "g3.8xlarge": {Region: "eu-central-1", Type: "g3.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 2, Inf: 0, Price: 2.85}, "g3.16xlarge": {Region: "eu-central-1", Type: "g3.16xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("64"), GPU: 4, Inf: 0, Price: 5.7}, "g3s.xlarge": {Region: "eu-central-1", Type: "g3s.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 1, Inf: 0, Price: 0.938}, "g4ad.xlarge": {Region: "eu-central-1", Type: "g4ad.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 1, Inf: 0, Price: 0.47327}, "g4ad.2xlarge": {Region: "eu-central-1", Type: "g4ad.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 1, Inf: 0, Price: 0.67662}, "g4ad.4xlarge": {Region: "eu-central-1", Type: "g4ad.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 1, Inf: 0, Price: 1.084}, "g4ad.8xlarge": {Region: "eu-central-1", Type: "g4ad.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 2, Inf: 0, Price: 2.168}, "g4ad.16xlarge": {Region: "eu-central-1", Type: "g4ad.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 4, Inf: 0, Price: 4.336}, "g4dn.xlarge": {Region: "eu-central-1", Type: "g4dn.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 1, Inf: 0, Price: 0.658}, "g4dn.2xlarge": {Region: "eu-central-1", Type: "g4dn.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 1, Inf: 0, Price: 0.94}, "g4dn.4xlarge": {Region: "eu-central-1", Type: "g4dn.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 1, Inf: 0, Price: 1.505}, "g4dn.8xlarge": {Region: "eu-central-1", Type: "g4dn.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 1, Inf: 0, Price: 2.72}, "g4dn.12xlarge": {Region: "eu-central-1", Type: "g4dn.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 4, Inf: 0, Price: 4.89}, "g4dn.16xlarge": {Region: "eu-central-1", Type: "g4dn.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 1, Inf: 0, Price: 5.44}, "g4dn.metal": {Region: "eu-central-1", Type: "g4dn.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 8, Inf: 0, Price: 9.78}, "i2.xlarge": {Region: "eu-central-1", Type: "i2.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 1.013}, "i2.2xlarge": {Region: "eu-central-1", Type: "i2.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 2.026}, "i2.4xlarge": {Region: "eu-central-1", Type: "i2.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 4.051}, "i2.8xlarge": {Region: "eu-central-1", Type: "i2.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 8.102}, "i3.large": {Region: "eu-central-1", Type: "i3.large", Memory: kresource.MustParse("15616Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.186}, "i3.xlarge": {Region: "eu-central-1", Type: "i3.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.372}, "i3.2xlarge": {Region: "eu-central-1", Type: "i3.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.744}, "i3.4xlarge": {Region: "eu-central-1", Type: "i3.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.488}, "i3.8xlarge": {Region: "eu-central-1", Type: "i3.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.976}, "i3.16xlarge": {Region: "eu-central-1", Type: "i3.16xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.952}, "i3.metal": {Region: "eu-central-1", Type: "i3.metal", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.952}, "i3en.large": {Region: "eu-central-1", Type: "i3en.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.27}, "i3en.xlarge": {Region: "eu-central-1", Type: "i3en.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.54}, "i3en.2xlarge": {Region: "eu-central-1", Type: "i3en.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.08}, "i3en.3xlarge": {Region: "eu-central-1", Type: "i3en.3xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("12"), GPU: 0, Inf: 0, Price: 1.62}, "i3en.6xlarge": {Region: "eu-central-1", Type: "i3en.6xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 0, Price: 3.24}, "i3en.12xlarge": {Region: "eu-central-1", Type: "i3en.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 6.48}, "i3en.24xlarge": {Region: "eu-central-1", Type: "i3en.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 12.96}, "i3en.metal": {Region: "eu-central-1", Type: "i3en.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 12.96}, "inf1.xlarge": {Region: "eu-central-1", Type: "inf1.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 1, Price: 0.285}, "inf1.2xlarge": {Region: "eu-central-1", Type: "inf1.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 1, Price: 0.453}, "inf1.6xlarge": {Region: "eu-central-1", Type: "inf1.6xlarge", Memory: kresource.MustParse("49152Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 4, Price: 1.475}, "inf1.24xlarge": {Region: "eu-central-1", Type: "inf1.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 16, Price: 5.902}, "m3.medium": {Region: "eu-central-1", Type: "m3.medium", Memory: kresource.MustParse("3840Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.079}, "m3.large": {Region: "eu-central-1", Type: "m3.large", Memory: kresource.MustParse("7680Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.158}, "m3.xlarge": {Region: "eu-central-1", Type: "m3.xlarge", Memory: kresource.MustParse("15360Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.315}, "m3.2xlarge": {Region: "eu-central-1", Type: "m3.2xlarge", Memory: kresource.MustParse("30720Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.632}, "m4.large": {Region: "eu-central-1", Type: "m4.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.12}, "m4.xlarge": {Region: "eu-central-1", Type: "m4.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.24}, "m4.2xlarge": {Region: "eu-central-1", Type: "m4.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.48}, "m4.4xlarge": {Region: "eu-central-1", Type: "m4.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.96}, "m4.10xlarge": {Region: "eu-central-1", Type: "m4.10xlarge", Memory: kresource.MustParse("163840Mi"), CPU: kresource.MustParse("40"), GPU: 0, Inf: 0, Price: 2.4}, "m4.16xlarge": {Region: "eu-central-1", Type: "m4.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.84}, "m5.large": {Region: "eu-central-1", Type: "m5.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.115}, "m5.xlarge": {Region: "eu-central-1", Type: "m5.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.23}, "m5.2xlarge": {Region: "eu-central-1", Type: "m5.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.46}, "m5.4xlarge": {Region: "eu-central-1", Type: "m5.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.92}, "m5.8xlarge": {Region: "eu-central-1", Type: "m5.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.84}, "m5.12xlarge": {Region: "eu-central-1", Type: "m5.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.76}, "m5.16xlarge": {Region: "eu-central-1", Type: "m5.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.68}, "m5.24xlarge": {Region: "eu-central-1", Type: "m5.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.52}, "m5.metal": {Region: "eu-central-1", Type: "m5.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.52}, "m5a.large": {Region: "eu-central-1", Type: "m5a.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.104}, "m5a.xlarge": {Region: "eu-central-1", Type: "m5a.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.208}, "m5a.2xlarge": {Region: "eu-central-1", Type: "m5a.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.416}, "m5a.4xlarge": {Region: "eu-central-1", Type: "m5a.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.832}, "m5a.8xlarge": {Region: "eu-central-1", Type: "m5a.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.664}, "m5a.12xlarge": {Region: "eu-central-1", Type: "m5a.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.496}, "m5a.16xlarge": {Region: "eu-central-1", Type: "m5a.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.328}, "m5a.24xlarge": {Region: "eu-central-1", Type: "m5a.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.992}, "m5ad.large": {Region: "eu-central-1", Type: "m5ad.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.125}, "m5ad.xlarge": {Region: "eu-central-1", Type: "m5ad.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.25}, "m5ad.2xlarge": {Region: "eu-central-1", Type: "m5ad.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.5}, "m5ad.4xlarge": {Region: "eu-central-1", Type: "m5ad.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.0}, "m5ad.8xlarge": {Region: "eu-central-1", Type: "m5ad.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.0}, "m5ad.12xlarge": {Region: "eu-central-1", Type: "m5ad.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.0}, "m5ad.16xlarge": {Region: "eu-central-1", Type: "m5ad.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.0}, "m5ad.24xlarge": {Region: "eu-central-1", Type: "m5ad.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.0}, "m5d.large": {Region: "eu-central-1", Type: "m5d.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.136}, "m5d.xlarge": {Region: "eu-central-1", Type: "m5d.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.272}, "m5d.2xlarge": {Region: "eu-central-1", Type: "m5d.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.544}, "m5d.4xlarge": {Region: "eu-central-1", Type: "m5d.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.088}, "m5d.8xlarge": {Region: "eu-central-1", Type: "m5d.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.176}, "m5d.12xlarge": {Region: "eu-central-1", Type: "m5d.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.264}, "m5d.16xlarge": {Region: "eu-central-1", Type: "m5d.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.352}, "m5d.24xlarge": {Region: "eu-central-1", Type: "m5d.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.528}, "m5d.metal": {Region: "eu-central-1", Type: "m5d.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.528}, "m5dn.large": {Region: "eu-central-1", Type: "m5dn.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.162}, "m5dn.xlarge": {Region: "eu-central-1", Type: "m5dn.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.324}, "m5dn.2xlarge": {Region: "eu-central-1", Type: "m5dn.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.648}, "m5dn.4xlarge": {Region: "eu-central-1", Type: "m5dn.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.296}, "m5dn.8xlarge": {Region: "eu-central-1", Type: "m5dn.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.592}, "m5dn.12xlarge": {Region: "eu-central-1", Type: "m5dn.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.888}, "m5dn.16xlarge": {Region: "eu-central-1", Type: "m5dn.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.184}, "m5dn.24xlarge": {Region: "eu-central-1", Type: "m5dn.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.776}, "m5dn.metal": {Region: "eu-central-1", Type: "m5dn.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.776}, "m5n.large": {Region: "eu-central-1", Type: "m5n.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.141}, "m5n.xlarge": {Region: "eu-central-1", Type: "m5n.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.282}, "m5n.2xlarge": {Region: "eu-central-1", Type: "m5n.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.564}, "m5n.4xlarge": {Region: "eu-central-1", Type: "m5n.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.128}, "m5n.8xlarge": {Region: "eu-central-1", Type: "m5n.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.256}, "m5n.12xlarge": {Region: "eu-central-1", Type: "m5n.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.384}, "m5n.16xlarge": {Region: "eu-central-1", Type: "m5n.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.512}, "m5n.24xlarge": {Region: "eu-central-1", Type: "m5n.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.768}, "m5n.metal": {Region: "eu-central-1", Type: "m5n.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.768}, "m5zn.large": {Region: "eu-central-1", Type: "m5zn.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.1979}, "m5zn.xlarge": {Region: "eu-central-1", Type: "m5zn.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.3957}, "m5zn.2xlarge": {Region: "eu-central-1", Type: "m5zn.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.7914}, "m5zn.3xlarge": {Region: "eu-central-1", Type: "m5zn.3xlarge", Memory: kresource.MustParse("49152Mi"), CPU: kresource.MustParse("12"), GPU: 0, Inf: 0, Price: 1.1872}, "m5zn.6xlarge": {Region: "eu-central-1", Type: "m5zn.6xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 0, Price: 2.3743}, "m5zn.12xlarge": {Region: "eu-central-1", Type: "m5zn.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.7486}, "m5zn.metal": {Region: "eu-central-1", Type: "m5zn.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.7486}, "m6g.medium": {Region: "eu-central-1", Type: "m6g.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.046}, "m6g.large": {Region: "eu-central-1", Type: "m6g.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.092}, "m6g.xlarge": {Region: "eu-central-1", Type: "m6g.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.184}, "m6g.2xlarge": {Region: "eu-central-1", Type: "m6g.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.368}, "m6g.4xlarge": {Region: "eu-central-1", Type: "m6g.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.736}, "m6g.8xlarge": {Region: "eu-central-1", Type: "m6g.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.472}, "m6g.12xlarge": {Region: "eu-central-1", Type: "m6g.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.208}, "m6g.16xlarge": {Region: "eu-central-1", Type: "m6g.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.944}, "m6g.metal": {Region: "eu-central-1", Type: "m6g.metal", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.944}, "m6gd.medium": {Region: "eu-central-1", Type: "m6gd.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0545}, "m6gd.large": {Region: "eu-central-1", Type: "m6gd.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.109}, "m6gd.xlarge": {Region: "eu-central-1", Type: "m6gd.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.218}, "m6gd.2xlarge": {Region: "eu-central-1", Type: "m6gd.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.436}, "m6gd.4xlarge": {Region: "eu-central-1", Type: "m6gd.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.872}, "m6gd.8xlarge": {Region: "eu-central-1", Type: "m6gd.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.744}, "m6gd.12xlarge": {Region: "eu-central-1", Type: "m6gd.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.616}, "m6gd.16xlarge": {Region: "eu-central-1", Type: "m6gd.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.488}, "m6gd.metal": {Region: "eu-central-1", Type: "m6gd.metal", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.488}, "m6i.large": {Region: "eu-central-1", Type: "m6i.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.115}, "m6i.xlarge": {Region: "eu-central-1", Type: "m6i.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.23}, "m6i.2xlarge": {Region: "eu-central-1", Type: "m6i.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.46}, "m6i.4xlarge": {Region: "eu-central-1", Type: "m6i.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.92}, "m6i.8xlarge": {Region: "eu-central-1", Type: "m6i.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.84}, "m6i.12xlarge": {Region: "eu-central-1", Type: "m6i.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.76}, "m6i.16xlarge": {Region: "eu-central-1", Type: "m6i.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.68}, "m6i.24xlarge": {Region: "eu-central-1", Type: "m6i.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.52}, "m6i.32xlarge": {Region: "eu-central-1", Type: "m6i.32xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 7.36}, "m6i.metal": {Region: "eu-central-1", Type: "m6i.metal", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 7.36}, "p2.xlarge": {Region: "eu-central-1", Type: "p2.xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("4"), GPU: 1, Inf: 0, Price: 1.326}, "p2.8xlarge": {Region: "eu-central-1", Type: "p2.8xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("32"), GPU: 8, Inf: 0, Price: 10.608}, "p2.16xlarge": {Region: "eu-central-1", Type: "p2.16xlarge", Memory: kresource.MustParse("749568Mi"), CPU: kresource.MustParse("64"), GPU: 16, Inf: 0, Price: 21.216}, "p3.2xlarge": {Region: "eu-central-1", Type: "p3.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 1, Inf: 0, Price: 3.823}, "p3.8xlarge": {Region: "eu-central-1", Type: "p3.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 4, Inf: 0, Price: 15.292}, "p3.16xlarge": {Region: "eu-central-1", Type: "p3.16xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("64"), GPU: 8, Inf: 0, Price: 30.584}, "p4d.24xlarge": {Region: "eu-central-1", Type: "p4d.24xlarge", Memory: kresource.MustParse("1179648Mi"), CPU: kresource.MustParse("96"), GPU: 8, Inf: 0, Price: 40.94475}, "r3.large": {Region: "eu-central-1", Type: "r3.large", Memory: kresource.MustParse("15616Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.2}, "r3.xlarge": {Region: "eu-central-1", Type: "r3.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.4}, "r3.2xlarge": {Region: "eu-central-1", Type: "r3.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.8}, "r3.4xlarge": {Region: "eu-central-1", Type: "r3.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.6}, "r3.8xlarge": {Region: "eu-central-1", Type: "r3.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 3.201}, "r4.large": {Region: "eu-central-1", Type: "r4.large", Memory: kresource.MustParse("15616Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.16005}, "r4.xlarge": {Region: "eu-central-1", Type: "r4.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.3201}, "r4.2xlarge": {Region: "eu-central-1", Type: "r4.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.6402}, "r4.4xlarge": {Region: "eu-central-1", Type: "r4.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.2804}, "r4.8xlarge": {Region: "eu-central-1", Type: "r4.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.5608}, "r4.16xlarge": {Region: "eu-central-1", Type: "r4.16xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.1216}, "r5.large": {Region: "eu-central-1", Type: "r5.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.152}, "r5.xlarge": {Region: "eu-central-1", Type: "r5.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.304}, "r5.2xlarge": {Region: "eu-central-1", Type: "r5.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.608}, "r5.4xlarge": {Region: "eu-central-1", Type: "r5.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.216}, "r5.8xlarge": {Region: "eu-central-1", Type: "r5.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.432}, "r5.12xlarge": {Region: "eu-central-1", Type: "r5.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.648}, "r5.16xlarge": {Region: "eu-central-1", Type: "r5.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.864}, "r5.24xlarge": {Region: "eu-central-1", Type: "r5.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.296}, "r5.metal": {Region: "eu-central-1", Type: "r5.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.296}, "r5a.large": {Region: "eu-central-1", Type: "r5a.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.137}, "r5a.xlarge": {Region: "eu-central-1", Type: "r5a.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.274}, "r5a.2xlarge": {Region: "eu-central-1", Type: "r5a.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.548}, "r5a.4xlarge": {Region: "eu-central-1", Type: "r5a.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.096}, "r5a.8xlarge": {Region: "eu-central-1", Type: "r5a.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.192}, "r5a.12xlarge": {Region: "eu-central-1", Type: "r5a.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.288}, "r5a.16xlarge": {Region: "eu-central-1", Type: "r5a.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.384}, "r5a.24xlarge": {Region: "eu-central-1", Type: "r5a.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.576}, "r5ad.large": {Region: "eu-central-1", Type: "r5ad.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.158}, "r5ad.xlarge": {Region: "eu-central-1", Type: "r5ad.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.316}, "r5ad.2xlarge": {Region: "eu-central-1", Type: "r5ad.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.632}, "r5ad.4xlarge": {Region: "eu-central-1", Type: "r5ad.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.264}, "r5ad.8xlarge": {Region: "eu-central-1", Type: "r5ad.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.528}, "r5ad.12xlarge": {Region: "eu-central-1", Type: "r5ad.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.792}, "r5ad.16xlarge": {Region: "eu-central-1", Type: "r5ad.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.056}, "r5ad.24xlarge": {Region: "eu-central-1", Type: "r5ad.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.584}, "r5b.large": {Region: "eu-central-1", Type: "r5b.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.178}, "r5b.xlarge": {Region: "eu-central-1", Type: "r5b.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.356}, "r5b.2xlarge": {Region: "eu-central-1", Type: "r5b.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.712}, "r5b.4xlarge": {Region: "eu-central-1", Type: "r5b.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.424}, "r5b.8xlarge": {Region: "eu-central-1", Type: "r5b.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.848}, "r5b.12xlarge": {Region: "eu-central-1", Type: "r5b.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.272}, "r5b.16xlarge": {Region: "eu-central-1", Type: "r5b.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.696}, "r5b.24xlarge": {Region: "eu-central-1", Type: "r5b.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.544}, "r5b.metal": {Region: "eu-central-1", Type: "r5b.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.544}, "r5d.large": {Region: "eu-central-1", Type: "r5d.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.173}, "r5d.xlarge": {Region: "eu-central-1", Type: "r5d.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.346}, "r5d.2xlarge": {Region: "eu-central-1", Type: "r5d.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.692}, "r5d.4xlarge": {Region: "eu-central-1", Type: "r5d.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.384}, "r5d.8xlarge": {Region: "eu-central-1", Type: "r5d.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.768}, "r5d.12xlarge": {Region: "eu-central-1", Type: "r5d.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.152}, "r5d.16xlarge": {Region: "eu-central-1", Type: "r5d.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.536}, "r5d.24xlarge": {Region: "eu-central-1", Type: "r5d.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.304}, "r5d.metal": {Region: "eu-central-1", Type: "r5d.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.304}, "r5dn.large": {Region: "eu-central-1", Type: "r5dn.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.199}, "r5dn.xlarge": {Region: "eu-central-1", Type: "r5dn.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.398}, "r5dn.2xlarge": {Region: "eu-central-1", Type: "r5dn.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.796}, "r5dn.4xlarge": {Region: "eu-central-1", Type: "r5dn.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.592}, "r5dn.8xlarge": {Region: "eu-central-1", Type: "r5dn.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 3.184}, "r5dn.12xlarge": {Region: "eu-central-1", Type: "r5dn.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.776}, "r5dn.16xlarge": {Region: "eu-central-1", Type: "r5dn.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 6.368}, "r5dn.24xlarge": {Region: "eu-central-1", Type: "r5dn.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 9.552}, "r5dn.metal": {Region: "eu-central-1", Type: "r5dn.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 9.552}, "r5n.large": {Region: "eu-central-1", Type: "r5n.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.178}, "r5n.xlarge": {Region: "eu-central-1", Type: "r5n.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.356}, "r5n.2xlarge": {Region: "eu-central-1", Type: "r5n.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.712}, "r5n.4xlarge": {Region: "eu-central-1", Type: "r5n.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.424}, "r5n.8xlarge": {Region: "eu-central-1", Type: "r5n.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.848}, "r5n.12xlarge": {Region: "eu-central-1", Type: "r5n.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.272}, "r5n.16xlarge": {Region: "eu-central-1", Type: "r5n.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.696}, "r5n.24xlarge": {Region: "eu-central-1", Type: "r5n.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.544}, "r5n.metal": {Region: "eu-central-1", Type: "r5n.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.544}, "r6g.medium": {Region: "eu-central-1", Type: "r6g.medium", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0608}, "r6g.large": {Region: "eu-central-1", Type: "r6g.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.1216}, "r6g.xlarge": {Region: "eu-central-1", Type: "r6g.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.2432}, "r6g.2xlarge": {Region: "eu-central-1", Type: "r6g.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.4864}, "r6g.4xlarge": {Region: "eu-central-1", Type: "r6g.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.9728}, "r6g.8xlarge": {Region: "eu-central-1", Type: "r6g.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.9456}, "r6g.12xlarge": {Region: "eu-central-1", Type: "r6g.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.9184}, "r6g.16xlarge": {Region: "eu-central-1", Type: "r6g.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.8912}, "r6g.metal": {Region: "eu-central-1", Type: "r6g.metal", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.8912}, "r6gd.medium": {Region: "eu-central-1", Type: "r6gd.medium", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.069}, "r6gd.large": {Region: "eu-central-1", Type: "r6gd.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.138}, "r6gd.xlarge": {Region: "eu-central-1", Type: "r6gd.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.276}, "r6gd.2xlarge": {Region: "eu-central-1", Type: "r6gd.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.552}, "r6gd.4xlarge": {Region: "eu-central-1", Type: "r6gd.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.104}, "r6gd.8xlarge": {Region: "eu-central-1", Type: "r6gd.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.208}, "r6gd.12xlarge": {Region: "eu-central-1", Type: "r6gd.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.312}, "r6gd.16xlarge": {Region: "eu-central-1", Type: "r6gd.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.416}, "r6gd.metal": {Region: "eu-central-1", Type: "r6gd.metal", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.416}, "t2.nano": {Region: "eu-central-1", Type: "t2.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0067}, "t2.micro": {Region: "eu-central-1", Type: "t2.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0134}, "t2.small": {Region: "eu-central-1", Type: "t2.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0268}, "t2.medium": {Region: "eu-central-1", Type: "t2.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0536}, "t2.large": {Region: "eu-central-1", Type: "t2.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.1072}, "t2.xlarge": {Region: "eu-central-1", Type: "t2.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.2144}, "t2.2xlarge": {Region: "eu-central-1", Type: "t2.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.4288}, "t3.nano": {Region: "eu-central-1", Type: "t3.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.006}, "t3.micro": {Region: "eu-central-1", Type: "t3.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.012}, "t3.small": {Region: "eu-central-1", Type: "t3.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.024}, "t3.medium": {Region: "eu-central-1", Type: "t3.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.048}, "t3.large": {Region: "eu-central-1", Type: "t3.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.096}, "t3.xlarge": {Region: "eu-central-1", Type: "t3.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.192}, "t3.2xlarge": {Region: "eu-central-1", Type: "t3.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.384}, "t3a.nano": {Region: "eu-central-1", Type: "t3a.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0054}, "t3a.micro": {Region: "eu-central-1", Type: "t3a.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0108}, "t3a.small": {Region: "eu-central-1", Type: "t3a.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0216}, "t3a.medium": {Region: "eu-central-1", Type: "t3a.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0432}, "t3a.large": {Region: "eu-central-1", Type: "t3a.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0864}, "t3a.xlarge": {Region: "eu-central-1", Type: "t3a.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1728}, "t3a.2xlarge": {Region: "eu-central-1", Type: "t3a.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3456}, "t4g.nano": {Region: "eu-central-1", Type: "t4g.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0048}, "t4g.micro": {Region: "eu-central-1", Type: "t4g.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0096}, "t4g.small": {Region: "eu-central-1", Type: "t4g.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0192}, "t4g.medium": {Region: "eu-central-1", Type: "t4g.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0384}, "t4g.large": {Region: "eu-central-1", Type: "t4g.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0768}, "t4g.xlarge": {Region: "eu-central-1", Type: "t4g.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1536}, "t4g.2xlarge": {Region: "eu-central-1", Type: "t4g.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3072}, "u-12tb1.112xlarge": {Region: "eu-central-1", Type: "u-12tb1.112xlarge", Memory: kresource.MustParse("12582912Mi"), CPU: kresource.MustParse("448"), GPU: 0, Inf: 0, Price: 131.733}, "u-3tb1.56xlarge": {Region: "eu-central-1", Type: "u-3tb1.56xlarge", Memory: kresource.MustParse("3145728Mi"), CPU: kresource.MustParse("224"), GPU: 0, Inf: 0, Price: 32.9335}, "u-6tb1.56xlarge": {Region: "eu-central-1", Type: "u-6tb1.56xlarge", Memory: kresource.MustParse("6291456Mi"), CPU: kresource.MustParse("224"), GPU: 0, Inf: 0, Price: 55.9796}, "u-6tb1.112xlarge": {Region: "eu-central-1", Type: "u-6tb1.112xlarge", Memory: kresource.MustParse("6291456Mi"), CPU: kresource.MustParse("448"), GPU: 0, Inf: 0, Price: 65.867}, "u-9tb1.112xlarge": {Region: "eu-central-1", Type: "u-9tb1.112xlarge", Memory: kresource.MustParse("9437184Mi"), CPU: kresource.MustParse("448"), GPU: 0, Inf: 0, Price: 98.8}, "x1.16xlarge": {Region: "eu-central-1", Type: "x1.16xlarge", Memory: kresource.MustParse("999424Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 9.337}, "x1.32xlarge": {Region: "eu-central-1", Type: "x1.32xlarge", Memory: kresource.MustParse("1998848Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 18.674}, "x1e.xlarge": {Region: "eu-central-1", Type: "x1e.xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 1.167}, "x1e.2xlarge": {Region: "eu-central-1", Type: "x1e.2xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 2.334}, "x1e.4xlarge": {Region: "eu-central-1", Type: "x1e.4xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 4.668}, "x1e.8xlarge": {Region: "eu-central-1", Type: "x1e.8xlarge", Memory: kresource.MustParse("999424Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 9.336}, "x1e.16xlarge": {Region: "eu-central-1", Type: "x1e.16xlarge", Memory: kresource.MustParse("1998848Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 18.672}, "x1e.32xlarge": {Region: "eu-central-1", Type: "x1e.32xlarge", Memory: kresource.MustParse("3997696Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 37.344}, "x2idn.16xlarge": {Region: "eu-central-1", Type: "x2idn.16xlarge", Memory: kresource.MustParse("1048576Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 9.337}, "x2idn.24xlarge": {Region: "eu-central-1", Type: "x2idn.24xlarge", Memory: kresource.MustParse("1572864Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 14.0055}, "x2idn.32xlarge": {Region: "eu-central-1", Type: "x2idn.32xlarge", Memory: kresource.MustParse("2097152Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 18.674}, "x2iedn.xlarge": {Region: "eu-central-1", Type: "x2iedn.xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 1.16713}, "x2iedn.2xlarge": {Region: "eu-central-1", Type: "x2iedn.2xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 2.33425}, "x2iedn.4xlarge": {Region: "eu-central-1", Type: "x2iedn.4xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 4.6685}, "x2iedn.8xlarge": {Region: "eu-central-1", Type: "x2iedn.8xlarge", Memory: kresource.MustParse("1048576Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 9.337}, "x2iedn.16xlarge": {Region: "eu-central-1", Type: "x2iedn.16xlarge", Memory: kresource.MustParse("2097152Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 18.674}, "x2iedn.24xlarge": {Region: "eu-central-1", Type: "x2iedn.24xlarge", Memory: kresource.MustParse("3145728Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 28.011}, "x2iedn.32xlarge": {Region: "eu-central-1", Type: "x2iedn.32xlarge", Memory: kresource.MustParse("4194304Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 37.348}, "z1d.large": {Region: "eu-central-1", Type: "z1d.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.225}, "z1d.xlarge": {Region: "eu-central-1", Type: "z1d.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.45}, "z1d.2xlarge": {Region: "eu-central-1", Type: "z1d.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.9}, "z1d.3xlarge": {Region: "eu-central-1", Type: "z1d.3xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("12"), GPU: 0, Inf: 0, Price: 1.35}, "z1d.6xlarge": {Region: "eu-central-1", Type: "z1d.6xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 0, Price: 2.7}, "z1d.12xlarge": {Region: "eu-central-1", Type: "z1d.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 5.4}, "z1d.metal": {Region: "eu-central-1", Type: "z1d.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 5.4}, }, "eu-north-1": { "c5.large": {Region: "eu-north-1", Type: "c5.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.091}, "c5.xlarge": {Region: "eu-north-1", Type: "c5.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.182}, "c5.2xlarge": {Region: "eu-north-1", Type: "c5.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.364}, "c5.4xlarge": {Region: "eu-north-1", Type: "c5.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.728}, "c5.9xlarge": {Region: "eu-north-1", Type: "c5.9xlarge", Memory: kresource.MustParse("73728Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 1.638}, "c5.12xlarge": {Region: "eu-north-1", Type: "c5.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.184}, "c5.18xlarge": {Region: "eu-north-1", Type: "c5.18xlarge", Memory: kresource.MustParse("147456Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 3.276}, "c5.24xlarge": {Region: "eu-north-1", Type: "c5.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.368}, "c5.metal": {Region: "eu-north-1", Type: "c5.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.368}, "c5a.large": {Region: "eu-north-1", Type: "c5a.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.082}, "c5a.xlarge": {Region: "eu-north-1", Type: "c5a.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.164}, "c5a.2xlarge": {Region: "eu-north-1", Type: "c5a.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.328}, "c5a.4xlarge": {Region: "eu-north-1", Type: "c5a.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.656}, "c5a.8xlarge": {Region: "eu-north-1", Type: "c5a.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.312}, "c5a.12xlarge": {Region: "eu-north-1", Type: "c5a.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 1.968}, "c5a.16xlarge": {Region: "eu-north-1", Type: "c5a.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.624}, "c5a.24xlarge": {Region: "eu-north-1", Type: "c5a.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 3.936}, "c5d.large": {Region: "eu-north-1", Type: "c5d.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.104}, "c5d.xlarge": {Region: "eu-north-1", Type: "c5d.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.208}, "c5d.2xlarge": {Region: "eu-north-1", Type: "c5d.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.416}, "c5d.4xlarge": {Region: "eu-north-1", Type: "c5d.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.832}, "c5d.9xlarge": {Region: "eu-north-1", Type: "c5d.9xlarge", Memory: kresource.MustParse("73728Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 1.872}, "c5d.12xlarge": {Region: "eu-north-1", Type: "c5d.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.496}, "c5d.18xlarge": {Region: "eu-north-1", Type: "c5d.18xlarge", Memory: kresource.MustParse("147456Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 3.744}, "c5d.24xlarge": {Region: "eu-north-1", Type: "c5d.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.992}, "c5d.metal": {Region: "eu-north-1", Type: "c5d.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.992}, "c5n.large": {Region: "eu-north-1", Type: "c5n.large", Memory: kresource.MustParse("5376Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.116}, "c5n.xlarge": {Region: "eu-north-1", Type: "c5n.xlarge", Memory: kresource.MustParse("10752Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.232}, "c5n.2xlarge": {Region: "eu-north-1", Type: "c5n.2xlarge", Memory: kresource.MustParse("21504Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.464}, "c5n.4xlarge": {Region: "eu-north-1", Type: "c5n.4xlarge", Memory: kresource.MustParse("43008Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.928}, "c5n.9xlarge": {Region: "eu-north-1", Type: "c5n.9xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 2.088}, "c5n.18xlarge": {Region: "eu-north-1", Type: "c5n.18xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 4.176}, "c5n.metal": {Region: "eu-north-1", Type: "c5n.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 4.176}, "c6g.medium": {Region: "eu-north-1", Type: "c6g.medium", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0365}, "c6g.large": {Region: "eu-north-1", Type: "c6g.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.073}, "c6g.xlarge": {Region: "eu-north-1", Type: "c6g.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.146}, "c6g.2xlarge": {Region: "eu-north-1", Type: "c6g.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.292}, "c6g.4xlarge": {Region: "eu-north-1", Type: "c6g.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.584}, "c6g.8xlarge": {Region: "eu-north-1", Type: "c6g.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.168}, "c6g.12xlarge": {Region: "eu-north-1", Type: "c6g.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 1.752}, "c6g.16xlarge": {Region: "eu-north-1", Type: "c6g.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.336}, "c6g.metal": {Region: "eu-north-1", Type: "c6g.metal", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.336}, "c6gn.medium": {Region: "eu-north-1", Type: "c6gn.medium", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0465}, "c6gn.large": {Region: "eu-north-1", Type: "c6gn.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.093}, "c6gn.xlarge": {Region: "eu-north-1", Type: "c6gn.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.186}, "c6gn.2xlarge": {Region: "eu-north-1", Type: "c6gn.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.372}, "c6gn.4xlarge": {Region: "eu-north-1", Type: "c6gn.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.744}, "c6gn.8xlarge": {Region: "eu-north-1", Type: "c6gn.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.488}, "c6gn.12xlarge": {Region: "eu-north-1", Type: "c6gn.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.232}, "c6gn.16xlarge": {Region: "eu-north-1", Type: "c6gn.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.976}, "d2.xlarge": {Region: "eu-north-1", Type: "d2.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.698}, "d2.2xlarge": {Region: "eu-north-1", Type: "d2.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.396}, "d2.4xlarge": {Region: "eu-north-1", Type: "d2.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 2.792}, "d2.8xlarge": {Region: "eu-north-1", Type: "d2.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 5.584}, "g4dn.xlarge": {Region: "eu-north-1", Type: "g4dn.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 1, Inf: 0, Price: 0.558}, "g4dn.2xlarge": {Region: "eu-north-1", Type: "g4dn.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 1, Inf: 0, Price: 0.798}, "g4dn.4xlarge": {Region: "eu-north-1", Type: "g4dn.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 1, Inf: 0, Price: 1.277}, "g4dn.8xlarge": {Region: "eu-north-1", Type: "g4dn.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 1, Inf: 0, Price: 2.308}, "g4dn.12xlarge": {Region: "eu-north-1", Type: "g4dn.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 4, Inf: 0, Price: 4.15}, "g4dn.16xlarge": {Region: "eu-north-1", Type: "g4dn.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 1, Inf: 0, Price: 4.617}, "g4dn.metal": {Region: "eu-north-1", Type: "g4dn.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 8, Inf: 0, Price: 8.3}, "i3.large": {Region: "eu-north-1", Type: "i3.large", Memory: kresource.MustParse("15616Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.163}, "i3.xlarge": {Region: "eu-north-1", Type: "i3.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.326}, "i3.2xlarge": {Region: "eu-north-1", Type: "i3.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.652}, "i3.4xlarge": {Region: "eu-north-1", Type: "i3.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.304}, "i3.8xlarge": {Region: "eu-north-1", Type: "i3.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.608}, "i3.16xlarge": {Region: "eu-north-1", Type: "i3.16xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.216}, "i3.metal": {Region: "eu-north-1", Type: "i3.metal", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.216}, "i3en.large": {Region: "eu-north-1", Type: "i3en.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.237}, "i3en.xlarge": {Region: "eu-north-1", Type: "i3en.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.474}, "i3en.2xlarge": {Region: "eu-north-1", Type: "i3en.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.948}, "i3en.3xlarge": {Region: "eu-north-1", Type: "i3en.3xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("12"), GPU: 0, Inf: 0, Price: 1.422}, "i3en.6xlarge": {Region: "eu-north-1", Type: "i3en.6xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 0, Price: 2.844}, "i3en.12xlarge": {Region: "eu-north-1", Type: "i3en.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 5.688}, "i3en.24xlarge": {Region: "eu-north-1", Type: "i3en.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 11.376}, "i3en.metal": {Region: "eu-north-1", Type: "i3en.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 11.376}, "inf1.xlarge": {Region: "eu-north-1", Type: "inf1.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 1, Price: 0.242}, "inf1.2xlarge": {Region: "eu-north-1", Type: "inf1.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 1, Price: 0.385}, "inf1.6xlarge": {Region: "eu-north-1", Type: "inf1.6xlarge", Memory: kresource.MustParse("49152Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 4, Price: 1.254}, "inf1.24xlarge": {Region: "eu-north-1", Type: "inf1.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 16, Price: 5.016}, "m5.large": {Region: "eu-north-1", Type: "m5.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.102}, "m5.xlarge": {Region: "eu-north-1", Type: "m5.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.204}, "m5.2xlarge": {Region: "eu-north-1", Type: "m5.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.408}, "m5.4xlarge": {Region: "eu-north-1", Type: "m5.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.816}, "m5.8xlarge": {Region: "eu-north-1", Type: "m5.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.632}, "m5.12xlarge": {Region: "eu-north-1", Type: "m5.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.448}, "m5.16xlarge": {Region: "eu-north-1", Type: "m5.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.264}, "m5.24xlarge": {Region: "eu-north-1", Type: "m5.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.896}, "m5.metal": {Region: "eu-north-1", Type: "m5.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.896}, "m5d.large": {Region: "eu-north-1", Type: "m5d.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.12}, "m5d.xlarge": {Region: "eu-north-1", Type: "m5d.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.24}, "m5d.2xlarge": {Region: "eu-north-1", Type: "m5d.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.48}, "m5d.4xlarge": {Region: "eu-north-1", Type: "m5d.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.96}, "m5d.8xlarge": {Region: "eu-north-1", Type: "m5d.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.92}, "m5d.12xlarge": {Region: "eu-north-1", Type: "m5d.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.88}, "m5d.16xlarge": {Region: "eu-north-1", Type: "m5d.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.84}, "m5d.24xlarge": {Region: "eu-north-1", Type: "m5d.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.76}, "m5d.metal": {Region: "eu-north-1", Type: "m5d.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.76}, "m6g.medium": {Region: "eu-north-1", Type: "m6g.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.041}, "m6g.large": {Region: "eu-north-1", Type: "m6g.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.082}, "m6g.xlarge": {Region: "eu-north-1", Type: "m6g.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.164}, "m6g.2xlarge": {Region: "eu-north-1", Type: "m6g.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.328}, "m6g.4xlarge": {Region: "eu-north-1", Type: "m6g.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.656}, "m6g.8xlarge": {Region: "eu-north-1", Type: "m6g.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.312}, "m6g.12xlarge": {Region: "eu-north-1", Type: "m6g.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 1.968}, "m6g.16xlarge": {Region: "eu-north-1", Type: "m6g.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.624}, "m6g.metal": {Region: "eu-north-1", Type: "m6g.metal", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.624}, "r5.large": {Region: "eu-north-1", Type: "r5.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.134}, "r5.xlarge": {Region: "eu-north-1", Type: "r5.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.268}, "r5.2xlarge": {Region: "eu-north-1", Type: "r5.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.536}, "r5.4xlarge": {Region: "eu-north-1", Type: "r5.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.072}, "r5.8xlarge": {Region: "eu-north-1", Type: "r5.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.144}, "r5.12xlarge": {Region: "eu-north-1", Type: "r5.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.216}, "r5.16xlarge": {Region: "eu-north-1", Type: "r5.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.288}, "r5.24xlarge": {Region: "eu-north-1", Type: "r5.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.432}, "r5.metal": {Region: "eu-north-1", Type: "r5.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.432}, "r5d.large": {Region: "eu-north-1", Type: "r5d.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.152}, "r5d.xlarge": {Region: "eu-north-1", Type: "r5d.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.304}, "r5d.2xlarge": {Region: "eu-north-1", Type: "r5d.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.608}, "r5d.4xlarge": {Region: "eu-north-1", Type: "r5d.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.216}, "r5d.8xlarge": {Region: "eu-north-1", Type: "r5d.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.432}, "r5d.12xlarge": {Region: "eu-north-1", Type: "r5d.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.648}, "r5d.16xlarge": {Region: "eu-north-1", Type: "r5d.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.864}, "r5d.24xlarge": {Region: "eu-north-1", Type: "r5d.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.296}, "r5d.metal": {Region: "eu-north-1", Type: "r5d.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.296}, "r5dn.large": {Region: "eu-north-1", Type: "r5dn.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.177}, "r5dn.xlarge": {Region: "eu-north-1", Type: "r5dn.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.354}, "r5dn.2xlarge": {Region: "eu-north-1", Type: "r5dn.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.708}, "r5dn.4xlarge": {Region: "eu-north-1", Type: "r5dn.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.416}, "r5dn.8xlarge": {Region: "eu-north-1", Type: "r5dn.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.832}, "r5dn.12xlarge": {Region: "eu-north-1", Type: "r5dn.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.248}, "r5dn.16xlarge": {Region: "eu-north-1", Type: "r5dn.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.664}, "r5dn.24xlarge": {Region: "eu-north-1", Type: "r5dn.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.496}, "r5dn.metal": {Region: "eu-north-1", Type: "r5dn.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.496}, "r5n.large": {Region: "eu-north-1", Type: "r5n.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.159}, "r5n.xlarge": {Region: "eu-north-1", Type: "r5n.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.318}, "r5n.2xlarge": {Region: "eu-north-1", Type: "r5n.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.636}, "r5n.4xlarge": {Region: "eu-north-1", Type: "r5n.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.272}, "r5n.8xlarge": {Region: "eu-north-1", Type: "r5n.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.544}, "r5n.12xlarge": {Region: "eu-north-1", Type: "r5n.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.816}, "r5n.16xlarge": {Region: "eu-north-1", Type: "r5n.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.088}, "r5n.24xlarge": {Region: "eu-north-1", Type: "r5n.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.632}, "r5n.metal": {Region: "eu-north-1", Type: "r5n.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.632}, "r6g.medium": {Region: "eu-north-1", Type: "r6g.medium", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0535}, "r6g.large": {Region: "eu-north-1", Type: "r6g.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.107}, "r6g.xlarge": {Region: "eu-north-1", Type: "r6g.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.214}, "r6g.2xlarge": {Region: "eu-north-1", Type: "r6g.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.428}, "r6g.4xlarge": {Region: "eu-north-1", Type: "r6g.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.856}, "r6g.8xlarge": {Region: "eu-north-1", Type: "r6g.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.712}, "r6g.12xlarge": {Region: "eu-north-1", Type: "r6g.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.568}, "r6g.16xlarge": {Region: "eu-north-1", Type: "r6g.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.424}, "r6g.metal": {Region: "eu-north-1", Type: "r6g.metal", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.424}, "t3.nano": {Region: "eu-north-1", Type: "t3.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0054}, "t3.micro": {Region: "eu-north-1", Type: "t3.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0108}, "t3.small": {Region: "eu-north-1", Type: "t3.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0216}, "t3.medium": {Region: "eu-north-1", Type: "t3.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0432}, "t3.large": {Region: "eu-north-1", Type: "t3.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0864}, "t3.xlarge": {Region: "eu-north-1", Type: "t3.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1728}, "t3.2xlarge": {Region: "eu-north-1", Type: "t3.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3456}, "t4g.nano": {Region: "eu-north-1", Type: "t4g.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0043}, "t4g.micro": {Region: "eu-north-1", Type: "t4g.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0086}, "t4g.small": {Region: "eu-north-1", Type: "t4g.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0172}, "t4g.medium": {Region: "eu-north-1", Type: "t4g.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0344}, "t4g.large": {Region: "eu-north-1", Type: "t4g.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0688}, "t4g.xlarge": {Region: "eu-north-1", Type: "t4g.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1376}, "t4g.2xlarge": {Region: "eu-north-1", Type: "t4g.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.2752}, "u-6tb1.56xlarge": {Region: "eu-north-1", Type: "u-6tb1.56xlarge", Memory: kresource.MustParse("6291456Mi"), CPU: kresource.MustParse("224"), GPU: 0, Inf: 0, Price: 49.33177}, "u-6tb1.112xlarge": {Region: "eu-north-1", Type: "u-6tb1.112xlarge", Memory: kresource.MustParse("6291456Mi"), CPU: kresource.MustParse("448"), GPU: 0, Inf: 0, Price: 58.045}, }, "eu-south-1": { "c5.large": {Region: "eu-south-1", Type: "c5.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.101}, "c5.xlarge": {Region: "eu-south-1", Type: "c5.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.202}, "c5.2xlarge": {Region: "eu-south-1", Type: "c5.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.404}, "c5.4xlarge": {Region: "eu-south-1", Type: "c5.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.808}, "c5.9xlarge": {Region: "eu-south-1", Type: "c5.9xlarge", Memory: kresource.MustParse("73728Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 1.818}, "c5.12xlarge": {Region: "eu-south-1", Type: "c5.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.424}, "c5.18xlarge": {Region: "eu-south-1", Type: "c5.18xlarge", Memory: kresource.MustParse("147456Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 3.636}, "c5.24xlarge": {Region: "eu-south-1", Type: "c5.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.848}, "c5.metal": {Region: "eu-south-1", Type: "c5.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.848}, "c5a.large": {Region: "eu-south-1", Type: "c5a.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.091}, "c5a.xlarge": {Region: "eu-south-1", Type: "c5a.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.182}, "c5a.2xlarge": {Region: "eu-south-1", Type: "c5a.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.364}, "c5a.4xlarge": {Region: "eu-south-1", Type: "c5a.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.728}, "c5a.8xlarge": {Region: "eu-south-1", Type: "c5a.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.456}, "c5a.12xlarge": {Region: "eu-south-1", Type: "c5a.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.184}, "c5a.16xlarge": {Region: "eu-south-1", Type: "c5a.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.912}, "c5a.24xlarge": {Region: "eu-south-1", Type: "c5a.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.368}, "c5ad.large": {Region: "eu-south-1", Type: "c5ad.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.103}, "c5ad.xlarge": {Region: "eu-south-1", Type: "c5ad.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.206}, "c5ad.2xlarge": {Region: "eu-south-1", Type: "c5ad.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.412}, "c5ad.4xlarge": {Region: "eu-south-1", Type: "c5ad.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.824}, "c5ad.8xlarge": {Region: "eu-south-1", Type: "c5ad.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.648}, "c5ad.12xlarge": {Region: "eu-south-1", Type: "c5ad.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.472}, "c5ad.16xlarge": {Region: "eu-south-1", Type: "c5ad.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.296}, "c5ad.24xlarge": {Region: "eu-south-1", Type: "c5ad.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.944}, "c5d.large": {Region: "eu-south-1", Type: "c5d.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.114}, "c5d.xlarge": {Region: "eu-south-1", Type: "c5d.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.228}, "c5d.2xlarge": {Region: "eu-south-1", Type: "c5d.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.456}, "c5d.4xlarge": {Region: "eu-south-1", Type: "c5d.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.912}, "c5d.9xlarge": {Region: "eu-south-1", Type: "c5d.9xlarge", Memory: kresource.MustParse("73728Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 2.052}, "c5d.12xlarge": {Region: "eu-south-1", Type: "c5d.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.736}, "c5d.18xlarge": {Region: "eu-south-1", Type: "c5d.18xlarge", Memory: kresource.MustParse("147456Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 4.104}, "c5d.24xlarge": {Region: "eu-south-1", Type: "c5d.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.472}, "c5d.metal": {Region: "eu-south-1", Type: "c5d.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.472}, "c5n.large": {Region: "eu-south-1", Type: "c5n.large", Memory: kresource.MustParse("5376Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.129}, "c5n.xlarge": {Region: "eu-south-1", Type: "c5n.xlarge", Memory: kresource.MustParse("10752Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.258}, "c5n.2xlarge": {Region: "eu-south-1", Type: "c5n.2xlarge", Memory: kresource.MustParse("21504Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.516}, "c5n.4xlarge": {Region: "eu-south-1", Type: "c5n.4xlarge", Memory: kresource.MustParse("43008Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.032}, "c5n.9xlarge": {Region: "eu-south-1", Type: "c5n.9xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 2.322}, "c5n.18xlarge": {Region: "eu-south-1", Type: "c5n.18xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 4.644}, "c5n.metal": {Region: "eu-south-1", Type: "c5n.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 4.644}, "c6g.medium": {Region: "eu-south-1", Type: "c6g.medium", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0404}, "c6g.large": {Region: "eu-south-1", Type: "c6g.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0808}, "c6g.xlarge": {Region: "eu-south-1", Type: "c6g.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1616}, "c6g.2xlarge": {Region: "eu-south-1", Type: "c6g.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3232}, "c6g.4xlarge": {Region: "eu-south-1", Type: "c6g.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.6464}, "c6g.8xlarge": {Region: "eu-south-1", Type: "c6g.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.2928}, "c6g.12xlarge": {Region: "eu-south-1", Type: "c6g.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 1.9392}, "c6g.16xlarge": {Region: "eu-south-1", Type: "c6g.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.5856}, "c6g.metal": {Region: "eu-south-1", Type: "c6g.metal", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.5856}, "d2.xlarge": {Region: "eu-south-1", Type: "d2.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.772}, "d2.2xlarge": {Region: "eu-south-1", Type: "d2.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.544}, "d2.4xlarge": {Region: "eu-south-1", Type: "d2.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 3.088}, "d2.8xlarge": {Region: "eu-south-1", Type: "d2.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 6.176}, "g4dn.xlarge": {Region: "eu-south-1", Type: "g4dn.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 1, Inf: 0, Price: 0.616}, "g4dn.2xlarge": {Region: "eu-south-1", Type: "g4dn.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 1, Inf: 0, Price: 0.88}, "g4dn.4xlarge": {Region: "eu-south-1", Type: "g4dn.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 1, Inf: 0, Price: 1.41}, "g4dn.8xlarge": {Region: "eu-south-1", Type: "g4dn.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 1, Inf: 0, Price: 2.547}, "g4dn.12xlarge": {Region: "eu-south-1", Type: "g4dn.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 4, Inf: 0, Price: 4.58}, "g4dn.16xlarge": {Region: "eu-south-1", Type: "g4dn.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 1, Inf: 0, Price: 5.095}, "g4dn.metal": {Region: "eu-south-1", Type: "g4dn.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 8, Inf: 0, Price: 9.16}, "i3.large": {Region: "eu-south-1", Type: "i3.large", Memory: kresource.MustParse("15616Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.181}, "i3.xlarge": {Region: "eu-south-1", Type: "i3.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.362}, "i3.2xlarge": {Region: "eu-south-1", Type: "i3.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.724}, "i3.4xlarge": {Region: "eu-south-1", Type: "i3.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.448}, "i3.8xlarge": {Region: "eu-south-1", Type: "i3.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.896}, "i3.16xlarge": {Region: "eu-south-1", Type: "i3.16xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.792}, "i3.metal": {Region: "eu-south-1", Type: "i3.metal", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.792}, "i3en.large": {Region: "eu-south-1", Type: "i3en.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.262}, "i3en.xlarge": {Region: "eu-south-1", Type: "i3en.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.524}, "i3en.2xlarge": {Region: "eu-south-1", Type: "i3en.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.049}, "i3en.3xlarge": {Region: "eu-south-1", Type: "i3en.3xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("12"), GPU: 0, Inf: 0, Price: 1.573}, "i3en.6xlarge": {Region: "eu-south-1", Type: "i3en.6xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 0, Price: 3.146}, "i3en.12xlarge": {Region: "eu-south-1", Type: "i3en.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 6.293}, "i3en.24xlarge": {Region: "eu-south-1", Type: "i3en.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 12.586}, "i3en.metal": {Region: "eu-south-1", Type: "i3en.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 12.586}, "inf1.xlarge": {Region: "eu-south-1", Type: "inf1.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 1, Price: 0.267}, "inf1.2xlarge": {Region: "eu-south-1", Type: "inf1.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 1, Price: 0.424}, "inf1.6xlarge": {Region: "eu-south-1", Type: "inf1.6xlarge", Memory: kresource.MustParse("49152Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 4, Price: 1.382}, "inf1.24xlarge": {Region: "eu-south-1", Type: "inf1.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 16, Price: 5.53}, "m5.large": {Region: "eu-south-1", Type: "m5.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.112}, "m5.xlarge": {Region: "eu-south-1", Type: "m5.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.224}, "m5.2xlarge": {Region: "eu-south-1", Type: "m5.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.448}, "m5.4xlarge": {Region: "eu-south-1", Type: "m5.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.896}, "m5.8xlarge": {Region: "eu-south-1", Type: "m5.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.792}, "m5.12xlarge": {Region: "eu-south-1", Type: "m5.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.688}, "m5.16xlarge": {Region: "eu-south-1", Type: "m5.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.584}, "m5.24xlarge": {Region: "eu-south-1", Type: "m5.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.376}, "m5.metal": {Region: "eu-south-1", Type: "m5.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.376}, "m5a.large": {Region: "eu-south-1", Type: "m5a.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.101}, "m5a.xlarge": {Region: "eu-south-1", Type: "m5a.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.202}, "m5a.2xlarge": {Region: "eu-south-1", Type: "m5a.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.404}, "m5a.4xlarge": {Region: "eu-south-1", Type: "m5a.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.808}, "m5a.8xlarge": {Region: "eu-south-1", Type: "m5a.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.616}, "m5a.12xlarge": {Region: "eu-south-1", Type: "m5a.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.424}, "m5a.16xlarge": {Region: "eu-south-1", Type: "m5a.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.232}, "m5a.24xlarge": {Region: "eu-south-1", Type: "m5a.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.848}, "m5d.large": {Region: "eu-south-1", Type: "m5d.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.132}, "m5d.xlarge": {Region: "eu-south-1", Type: "m5d.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.264}, "m5d.2xlarge": {Region: "eu-south-1", Type: "m5d.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.528}, "m5d.4xlarge": {Region: "eu-south-1", Type: "m5d.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.056}, "m5d.8xlarge": {Region: "eu-south-1", Type: "m5d.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.112}, "m5d.12xlarge": {Region: "eu-south-1", Type: "m5d.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.168}, "m5d.16xlarge": {Region: "eu-south-1", Type: "m5d.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.224}, "m5d.24xlarge": {Region: "eu-south-1", Type: "m5d.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.336}, "m5d.metal": {Region: "eu-south-1", Type: "m5d.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.336}, "m6g.medium": {Region: "eu-south-1", Type: "m6g.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0448}, "m6g.large": {Region: "eu-south-1", Type: "m6g.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0896}, "m6g.xlarge": {Region: "eu-south-1", Type: "m6g.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1792}, "m6g.2xlarge": {Region: "eu-south-1", Type: "m6g.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3584}, "m6g.4xlarge": {Region: "eu-south-1", Type: "m6g.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.7168}, "m6g.8xlarge": {Region: "eu-south-1", Type: "m6g.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.4336}, "m6g.12xlarge": {Region: "eu-south-1", Type: "m6g.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.1504}, "m6g.16xlarge": {Region: "eu-south-1", Type: "m6g.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.8672}, "m6g.metal": {Region: "eu-south-1", Type: "m6g.metal", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.8672}, "r5.large": {Region: "eu-south-1", Type: "r5.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.148}, "r5.xlarge": {Region: "eu-south-1", Type: "r5.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.296}, "r5.2xlarge": {Region: "eu-south-1", Type: "r5.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.592}, "r5.4xlarge": {Region: "eu-south-1", Type: "r5.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.184}, "r5.8xlarge": {Region: "eu-south-1", Type: "r5.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.368}, "r5.12xlarge": {Region: "eu-south-1", Type: "r5.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.552}, "r5.16xlarge": {Region: "eu-south-1", Type: "r5.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.736}, "r5.24xlarge": {Region: "eu-south-1", Type: "r5.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.104}, "r5.metal": {Region: "eu-south-1", Type: "r5.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.104}, "r5a.large": {Region: "eu-south-1", Type: "r5a.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.133}, "r5a.xlarge": {Region: "eu-south-1", Type: "r5a.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.266}, "r5a.2xlarge": {Region: "eu-south-1", Type: "r5a.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.532}, "r5a.4xlarge": {Region: "eu-south-1", Type: "r5a.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.064}, "r5a.8xlarge": {Region: "eu-south-1", Type: "r5a.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.128}, "r5a.12xlarge": {Region: "eu-south-1", Type: "r5a.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.192}, "r5a.16xlarge": {Region: "eu-south-1", Type: "r5a.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.256}, "r5a.24xlarge": {Region: "eu-south-1", Type: "r5a.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.384}, "r5d.large": {Region: "eu-south-1", Type: "r5d.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.168}, "r5d.xlarge": {Region: "eu-south-1", Type: "r5d.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.336}, "r5d.2xlarge": {Region: "eu-south-1", Type: "r5d.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.672}, "r5d.4xlarge": {Region: "eu-south-1", Type: "r5d.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.344}, "r5d.8xlarge": {Region: "eu-south-1", Type: "r5d.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.688}, "r5d.12xlarge": {Region: "eu-south-1", Type: "r5d.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.032}, "r5d.16xlarge": {Region: "eu-south-1", Type: "r5d.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.376}, "r5d.24xlarge": {Region: "eu-south-1", Type: "r5d.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.064}, "r5d.metal": {Region: "eu-south-1", Type: "r5d.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.064}, "r5dn.large": {Region: "eu-south-1", Type: "r5dn.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.196}, "r5dn.xlarge": {Region: "eu-south-1", Type: "r5dn.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.392}, "r5dn.2xlarge": {Region: "eu-south-1", Type: "r5dn.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.784}, "r5dn.4xlarge": {Region: "eu-south-1", Type: "r5dn.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.568}, "r5dn.8xlarge": {Region: "eu-south-1", Type: "r5dn.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 3.136}, "r5dn.12xlarge": {Region: "eu-south-1", Type: "r5dn.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.704}, "r5dn.16xlarge": {Region: "eu-south-1", Type: "r5dn.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 6.272}, "r5dn.24xlarge": {Region: "eu-south-1", Type: "r5dn.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 9.408}, "r5dn.metal": {Region: "eu-south-1", Type: "r5dn.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 9.408}, "r6g.medium": {Region: "eu-south-1", Type: "r6g.medium", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0592}, "r6g.large": {Region: "eu-south-1", Type: "r6g.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.1184}, "r6g.xlarge": {Region: "eu-south-1", Type: "r6g.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.2368}, "r6g.2xlarge": {Region: "eu-south-1", Type: "r6g.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.4736}, "r6g.4xlarge": {Region: "eu-south-1", Type: "r6g.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.9472}, "r6g.8xlarge": {Region: "eu-south-1", Type: "r6g.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.8944}, "r6g.12xlarge": {Region: "eu-south-1", Type: "r6g.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.8416}, "r6g.16xlarge": {Region: "eu-south-1", Type: "r6g.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.7888}, "r6g.metal": {Region: "eu-south-1", Type: "r6g.metal", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.7888}, "t3.nano": {Region: "eu-south-1", Type: "t3.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.006}, "t3.micro": {Region: "eu-south-1", Type: "t3.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.012}, "t3.small": {Region: "eu-south-1", Type: "t3.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.024}, "t3.medium": {Region: "eu-south-1", Type: "t3.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0479}, "t3.large": {Region: "eu-south-1", Type: "t3.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0958}, "t3.xlarge": {Region: "eu-south-1", Type: "t3.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1917}, "t3.2xlarge": {Region: "eu-south-1", Type: "t3.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3834}, "t3a.nano": {Region: "eu-south-1", Type: "t3a.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0054}, "t3a.micro": {Region: "eu-south-1", Type: "t3a.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0108}, "t3a.small": {Region: "eu-south-1", Type: "t3a.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0216}, "t3a.medium": {Region: "eu-south-1", Type: "t3a.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0431}, "t3a.large": {Region: "eu-south-1", Type: "t3a.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0862}, "t3a.xlarge": {Region: "eu-south-1", Type: "t3a.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1725}, "t3a.2xlarge": {Region: "eu-south-1", Type: "t3a.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.345}, "t4g.nano": {Region: "eu-south-1", Type: "t4g.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0048}, "t4g.micro": {Region: "eu-south-1", Type: "t4g.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0096}, "t4g.small": {Region: "eu-south-1", Type: "t4g.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0192}, "t4g.medium": {Region: "eu-south-1", Type: "t4g.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0384}, "t4g.large": {Region: "eu-south-1", Type: "t4g.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0768}, "t4g.xlarge": {Region: "eu-south-1", Type: "t4g.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1536}, "t4g.2xlarge": {Region: "eu-south-1", Type: "t4g.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3072}, "u-3tb1.56xlarge": {Region: "eu-south-1", Type: "u-3tb1.56xlarge", Memory: kresource.MustParse("3145728Mi"), CPU: kresource.MustParse("224"), GPU: 0, Inf: 0, Price: 32.0775}, "u-6tb1.56xlarge": {Region: "eu-south-1", Type: "u-6tb1.56xlarge", Memory: kresource.MustParse("6291456Mi"), CPU: kresource.MustParse("224"), GPU: 0, Inf: 0, Price: 54.52459}, "u-6tb1.112xlarge": {Region: "eu-south-1", Type: "u-6tb1.112xlarge", Memory: kresource.MustParse("6291456Mi"), CPU: kresource.MustParse("448"), GPU: 0, Inf: 0, Price: 64.155}, }, "eu-west-1": { "a1.medium": {Region: "eu-west-1", Type: "a1.medium", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0288}, "a1.large": {Region: "eu-west-1", Type: "a1.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0576}, "a1.xlarge": {Region: "eu-west-1", Type: "a1.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1152}, "a1.2xlarge": {Region: "eu-west-1", Type: "a1.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.2304}, "a1.4xlarge": {Region: "eu-west-1", Type: "a1.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.4608}, "a1.metal": {Region: "eu-west-1", Type: "a1.metal", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.461}, "c1.medium": {Region: "eu-west-1", Type: "c1.medium", Memory: kresource.MustParse("1740Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.148}, "c1.xlarge": {Region: "eu-west-1", Type: "c1.xlarge", Memory: kresource.MustParse("7168Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.592}, "c3.large": {Region: "eu-west-1", Type: "c3.large", Memory: kresource.MustParse("3840Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.12}, "c3.xlarge": {Region: "eu-west-1", Type: "c3.xlarge", Memory: kresource.MustParse("7680Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.239}, "c3.2xlarge": {Region: "eu-west-1", Type: "c3.2xlarge", Memory: kresource.MustParse("15360Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.478}, "c3.4xlarge": {Region: "eu-west-1", Type: "c3.4xlarge", Memory: kresource.MustParse("30720Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.956}, "c3.8xlarge": {Region: "eu-west-1", Type: "c3.8xlarge", Memory: kresource.MustParse("61440Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.912}, "c4.large": {Region: "eu-west-1", Type: "c4.large", Memory: kresource.MustParse("3840Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.113}, "c4.xlarge": {Region: "eu-west-1", Type: "c4.xlarge", Memory: kresource.MustParse("7680Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.226}, "c4.2xlarge": {Region: "eu-west-1", Type: "c4.2xlarge", Memory: kresource.MustParse("15360Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.453}, "c4.4xlarge": {Region: "eu-west-1", Type: "c4.4xlarge", Memory: kresource.MustParse("30720Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.905}, "c4.8xlarge": {Region: "eu-west-1", Type: "c4.8xlarge", Memory: kresource.MustParse("61440Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 1.811}, "c5.large": {Region: "eu-west-1", Type: "c5.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.096}, "c5.xlarge": {Region: "eu-west-1", Type: "c5.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.192}, "c5.2xlarge": {Region: "eu-west-1", Type: "c5.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.384}, "c5.4xlarge": {Region: "eu-west-1", Type: "c5.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.768}, "c5.9xlarge": {Region: "eu-west-1", Type: "c5.9xlarge", Memory: kresource.MustParse("73728Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 1.728}, "c5.12xlarge": {Region: "eu-west-1", Type: "c5.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.304}, "c5.18xlarge": {Region: "eu-west-1", Type: "c5.18xlarge", Memory: kresource.MustParse("147456Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 3.456}, "c5.24xlarge": {Region: "eu-west-1", Type: "c5.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.608}, "c5.metal": {Region: "eu-west-1", Type: "c5.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.608}, "c5a.large": {Region: "eu-west-1", Type: "c5a.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.086}, "c5a.xlarge": {Region: "eu-west-1", Type: "c5a.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.172}, "c5a.2xlarge": {Region: "eu-west-1", Type: "c5a.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.344}, "c5a.4xlarge": {Region: "eu-west-1", Type: "c5a.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.688}, "c5a.8xlarge": {Region: "eu-west-1", Type: "c5a.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.376}, "c5a.12xlarge": {Region: "eu-west-1", Type: "c5a.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.064}, "c5a.16xlarge": {Region: "eu-west-1", Type: "c5a.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.752}, "c5a.24xlarge": {Region: "eu-west-1", Type: "c5a.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.128}, "c5ad.large": {Region: "eu-west-1", Type: "c5ad.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.098}, "c5ad.xlarge": {Region: "eu-west-1", Type: "c5ad.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.196}, "c5ad.2xlarge": {Region: "eu-west-1", Type: "c5ad.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.392}, "c5ad.4xlarge": {Region: "eu-west-1", Type: "c5ad.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.784}, "c5ad.8xlarge": {Region: "eu-west-1", Type: "c5ad.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.568}, "c5ad.12xlarge": {Region: "eu-west-1", Type: "c5ad.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.352}, "c5ad.16xlarge": {Region: "eu-west-1", Type: "c5ad.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.136}, "c5ad.24xlarge": {Region: "eu-west-1", Type: "c5ad.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.704}, "c5d.large": {Region: "eu-west-1", Type: "c5d.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.109}, "c5d.xlarge": {Region: "eu-west-1", Type: "c5d.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.218}, "c5d.2xlarge": {Region: "eu-west-1", Type: "c5d.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.436}, "c5d.4xlarge": {Region: "eu-west-1", Type: "c5d.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.872}, "c5d.9xlarge": {Region: "eu-west-1", Type: "c5d.9xlarge", Memory: kresource.MustParse("73728Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 1.962}, "c5d.12xlarge": {Region: "eu-west-1", Type: "c5d.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.616}, "c5d.18xlarge": {Region: "eu-west-1", Type: "c5d.18xlarge", Memory: kresource.MustParse("147456Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 3.924}, "c5d.24xlarge": {Region: "eu-west-1", Type: "c5d.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.232}, "c5d.metal": {Region: "eu-west-1", Type: "c5d.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.232}, "c5n.large": {Region: "eu-west-1", Type: "c5n.large", Memory: kresource.MustParse("5376Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.122}, "c5n.xlarge": {Region: "eu-west-1", Type: "c5n.xlarge", Memory: kresource.MustParse("10752Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.244}, "c5n.2xlarge": {Region: "eu-west-1", Type: "c5n.2xlarge", Memory: kresource.MustParse("21504Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.488}, "c5n.4xlarge": {Region: "eu-west-1", Type: "c5n.4xlarge", Memory: kresource.MustParse("43008Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.976}, "c5n.9xlarge": {Region: "eu-west-1", Type: "c5n.9xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 2.196}, "c5n.18xlarge": {Region: "eu-west-1", Type: "c5n.18xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 4.392}, "c5n.metal": {Region: "eu-west-1", Type: "c5n.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 4.392}, "c6a.large": {Region: "eu-west-1", Type: "c6a.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.08208}, "c6a.xlarge": {Region: "eu-west-1", Type: "c6a.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.16416}, "c6a.2xlarge": {Region: "eu-west-1", Type: "c6a.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.32832}, "c6a.4xlarge": {Region: "eu-west-1", Type: "c6a.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.65664}, "c6a.8xlarge": {Region: "eu-west-1", Type: "c6a.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.31328}, "c6a.12xlarge": {Region: "eu-west-1", Type: "c6a.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 1.96992}, "c6a.16xlarge": {Region: "eu-west-1", Type: "c6a.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.62656}, "c6a.24xlarge": {Region: "eu-west-1", Type: "c6a.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 3.93984}, "c6a.32xlarge": {Region: "eu-west-1", Type: "c6a.32xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 5.25312}, "c6a.48xlarge": {Region: "eu-west-1", Type: "c6a.48xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("192"), GPU: 0, Inf: 0, Price: 7.87968}, "c6a.metal": {Region: "eu-west-1", Type: "c6a.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("192"), GPU: 0, Inf: 0, Price: 7.87968}, "c6g.medium": {Region: "eu-west-1", Type: "c6g.medium", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0365}, "c6g.large": {Region: "eu-west-1", Type: "c6g.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.073}, "c6g.xlarge": {Region: "eu-west-1", Type: "c6g.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1459}, "c6g.2xlarge": {Region: "eu-west-1", Type: "c6g.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.2918}, "c6g.4xlarge": {Region: "eu-west-1", Type: "c6g.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.5837}, "c6g.8xlarge": {Region: "eu-west-1", Type: "c6g.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.1674}, "c6g.12xlarge": {Region: "eu-west-1", Type: "c6g.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 1.751}, "c6g.16xlarge": {Region: "eu-west-1", Type: "c6g.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.3347}, "c6g.metal": {Region: "eu-west-1", Type: "c6g.metal", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.3347}, "c6gd.medium": {Region: "eu-west-1", Type: "c6gd.medium", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0436}, "c6gd.large": {Region: "eu-west-1", Type: "c6gd.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0872}, "c6gd.xlarge": {Region: "eu-west-1", Type: "c6gd.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1744}, "c6gd.2xlarge": {Region: "eu-west-1", Type: "c6gd.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3488}, "c6gd.4xlarge": {Region: "eu-west-1", Type: "c6gd.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.6976}, "c6gd.8xlarge": {Region: "eu-west-1", Type: "c6gd.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.3952}, "c6gd.12xlarge": {Region: "eu-west-1", Type: "c6gd.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.0928}, "c6gd.16xlarge": {Region: "eu-west-1", Type: "c6gd.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.7904}, "c6gd.metal": {Region: "eu-west-1", Type: "c6gd.metal", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.7904}, "c6gn.medium": {Region: "eu-west-1", Type: "c6gn.medium", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0488}, "c6gn.large": {Region: "eu-west-1", Type: "c6gn.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0976}, "c6gn.xlarge": {Region: "eu-west-1", Type: "c6gn.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1952}, "c6gn.2xlarge": {Region: "eu-west-1", Type: "c6gn.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3904}, "c6gn.4xlarge": {Region: "eu-west-1", Type: "c6gn.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.7808}, "c6gn.8xlarge": {Region: "eu-west-1", Type: "c6gn.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.5616}, "c6gn.12xlarge": {Region: "eu-west-1", Type: "c6gn.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.3424}, "c6gn.16xlarge": {Region: "eu-west-1", Type: "c6gn.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.1232}, "c6i.large": {Region: "eu-west-1", Type: "c6i.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0912}, "c6i.xlarge": {Region: "eu-west-1", Type: "c6i.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1824}, "c6i.2xlarge": {Region: "eu-west-1", Type: "c6i.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3648}, "c6i.4xlarge": {Region: "eu-west-1", Type: "c6i.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.7296}, "c6i.8xlarge": {Region: "eu-west-1", Type: "c6i.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.4592}, "c6i.12xlarge": {Region: "eu-west-1", Type: "c6i.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.1888}, "c6i.16xlarge": {Region: "eu-west-1", Type: "c6i.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.9184}, "c6i.24xlarge": {Region: "eu-west-1", Type: "c6i.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.3776}, "c6i.32xlarge": {Region: "eu-west-1", Type: "c6i.32xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 5.8368}, "c6i.metal": {Region: "eu-west-1", Type: "c6i.metal", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 5.8368}, "cc2.8xlarge": {Region: "eu-west-1", Type: "cc2.8xlarge", Memory: kresource.MustParse("61952Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.25}, "cr1.8xlarge": {Region: "eu-west-1", Type: "cr1.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 3.75}, "d2.xlarge": {Region: "eu-west-1", Type: "d2.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.735}, "d2.2xlarge": {Region: "eu-west-1", Type: "d2.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.47}, "d2.4xlarge": {Region: "eu-west-1", Type: "d2.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 2.94}, "d2.8xlarge": {Region: "eu-west-1", Type: "d2.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 5.88}, "d3.xlarge": {Region: "eu-west-1", Type: "d3.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.609}, "d3.2xlarge": {Region: "eu-west-1", Type: "d3.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.219}, "d3.4xlarge": {Region: "eu-west-1", Type: "d3.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 2.437}, "d3.8xlarge": {Region: "eu-west-1", Type: "d3.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 4.87448}, "d3en.xlarge": {Region: "eu-west-1", Type: "d3en.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.641}, "d3en.2xlarge": {Region: "eu-west-1", Type: "d3en.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.283}, "d3en.4xlarge": {Region: "eu-west-1", Type: "d3en.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 2.566}, "d3en.6xlarge": {Region: "eu-west-1", Type: "d3en.6xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 0, Price: 3.848}, "d3en.8xlarge": {Region: "eu-west-1", Type: "d3en.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 5.13104}, "d3en.12xlarge": {Region: "eu-west-1", Type: "d3en.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 7.69656}, "f1.2xlarge": {Region: "eu-west-1", Type: "f1.2xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.815}, "f1.4xlarge": {Region: "eu-west-1", Type: "f1.4xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 3.63}, "f1.16xlarge": {Region: "eu-west-1", Type: "f1.16xlarge", Memory: kresource.MustParse("999424Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 14.52}, "g2.2xlarge": {Region: "eu-west-1", Type: "g2.2xlarge", Memory: kresource.MustParse("15360Mi"), CPU: kresource.MustParse("8"), GPU: 1, Inf: 0, Price: 0.702}, "g2.8xlarge": {Region: "eu-west-1", Type: "g2.8xlarge", Memory: kresource.MustParse("61440Mi"), CPU: kresource.MustParse("32"), GPU: 4, Inf: 0, Price: 2.808}, "g3.4xlarge": {Region: "eu-west-1", Type: "g3.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 1, Inf: 0, Price: 1.21}, "g3.8xlarge": {Region: "eu-west-1", Type: "g3.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 2, Inf: 0, Price: 2.42}, "g3.16xlarge": {Region: "eu-west-1", Type: "g3.16xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("64"), GPU: 4, Inf: 0, Price: 4.84}, "g3s.xlarge": {Region: "eu-west-1", Type: "g3s.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 1, Inf: 0, Price: 0.796}, "g4ad.xlarge": {Region: "eu-west-1", Type: "g4ad.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 1, Inf: 0, Price: 0.42263}, "g4ad.2xlarge": {Region: "eu-west-1", Type: "g4ad.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 1, Inf: 0, Price: 0.60421}, "g4ad.4xlarge": {Region: "eu-west-1", Type: "g4ad.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 1, Inf: 0, Price: 0.968}, "g4ad.8xlarge": {Region: "eu-west-1", Type: "g4ad.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 2, Inf: 0, Price: 1.936}, "g4ad.16xlarge": {Region: "eu-west-1", Type: "g4ad.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 4, Inf: 0, Price: 3.872}, "g4dn.xlarge": {Region: "eu-west-1", Type: "g4dn.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 1, Inf: 0, Price: 0.587}, "g4dn.2xlarge": {Region: "eu-west-1", Type: "g4dn.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 1, Inf: 0, Price: 0.838}, "g4dn.4xlarge": {Region: "eu-west-1", Type: "g4dn.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 1, Inf: 0, Price: 1.342}, "g4dn.8xlarge": {Region: "eu-west-1", Type: "g4dn.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 1, Inf: 0, Price: 2.426}, "g4dn.12xlarge": {Region: "eu-west-1", Type: "g4dn.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 4, Inf: 0, Price: 4.362}, "g4dn.16xlarge": {Region: "eu-west-1", Type: "g4dn.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 1, Inf: 0, Price: 4.853}, "g4dn.metal": {Region: "eu-west-1", Type: "g4dn.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 8, Inf: 0, Price: 8.724}, "g5.xlarge": {Region: "eu-west-1", Type: "g5.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 1, Inf: 0, Price: 1.123}, "g5.2xlarge": {Region: "eu-west-1", Type: "g5.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 1, Inf: 0, Price: 1.35296}, "g5.4xlarge": {Region: "eu-west-1", Type: "g5.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 1, Inf: 0, Price: 1.81287}, "g5.8xlarge": {Region: "eu-west-1", Type: "g5.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 1, Inf: 0, Price: 2.73271}, "g5.12xlarge": {Region: "eu-west-1", Type: "g5.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 4, Inf: 0, Price: 6.33167}, "g5.16xlarge": {Region: "eu-west-1", Type: "g5.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 1, Inf: 0, Price: 4.57237}, "g5.24xlarge": {Region: "eu-west-1", Type: "g5.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 4, Inf: 0, Price: 9.09117}, "g5.48xlarge": {Region: "eu-west-1", Type: "g5.48xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("192"), GPU: 8, Inf: 0, Price: 18.18233}, "h1.2xlarge": {Region: "eu-west-1", Type: "h1.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.519}, "h1.4xlarge": {Region: "eu-west-1", Type: "h1.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.038}, "h1.8xlarge": {Region: "eu-west-1", Type: "h1.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.076}, "h1.16xlarge": {Region: "eu-west-1", Type: "h1.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.152}, "hs1.8xlarge": {Region: "eu-west-1", Type: "hs1.8xlarge", Memory: kresource.MustParse("119808Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 4.9}, "i2.xlarge": {Region: "eu-west-1", Type: "i2.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.938}, "i2.2xlarge": {Region: "eu-west-1", Type: "i2.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.876}, "i2.4xlarge": {Region: "eu-west-1", Type: "i2.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 3.751}, "i2.8xlarge": {Region: "eu-west-1", Type: "i2.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 7.502}, "i3.large": {Region: "eu-west-1", Type: "i3.large", Memory: kresource.MustParse("15616Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.172}, "i3.xlarge": {Region: "eu-west-1", Type: "i3.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.344}, "i3.2xlarge": {Region: "eu-west-1", Type: "i3.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.688}, "i3.4xlarge": {Region: "eu-west-1", Type: "i3.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.376}, "i3.8xlarge": {Region: "eu-west-1", Type: "i3.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.752}, "i3.16xlarge": {Region: "eu-west-1", Type: "i3.16xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.504}, "i3.metal": {Region: "eu-west-1", Type: "i3.metal", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.504}, "i3en.large": {Region: "eu-west-1", Type: "i3en.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.25}, "i3en.xlarge": {Region: "eu-west-1", Type: "i3en.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.5}, "i3en.2xlarge": {Region: "eu-west-1", Type: "i3en.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.0}, "i3en.3xlarge": {Region: "eu-west-1", Type: "i3en.3xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("12"), GPU: 0, Inf: 0, Price: 1.5}, "i3en.6xlarge": {Region: "eu-west-1", Type: "i3en.6xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 0, Price: 3.0}, "i3en.12xlarge": {Region: "eu-west-1", Type: "i3en.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 6.0}, "i3en.24xlarge": {Region: "eu-west-1", Type: "i3en.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 12.0}, "i3en.metal": {Region: "eu-west-1", Type: "i3en.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 12.0}, "im4gn.large": {Region: "eu-west-1", Type: "im4gn.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.20055}, "im4gn.xlarge": {Region: "eu-west-1", Type: "im4gn.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.4011}, "im4gn.2xlarge": {Region: "eu-west-1", Type: "im4gn.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.80221}, "im4gn.4xlarge": {Region: "eu-west-1", Type: "im4gn.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.60442}, "im4gn.8xlarge": {Region: "eu-west-1", Type: "im4gn.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 3.20883}, "im4gn.16xlarge": {Region: "eu-west-1", Type: "im4gn.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 6.41766}, "inf1.xlarge": {Region: "eu-west-1", Type: "inf1.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 1, Price: 0.254}, "inf1.2xlarge": {Region: "eu-west-1", Type: "inf1.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 1, Price: 0.403}, "inf1.6xlarge": {Region: "eu-west-1", Type: "inf1.6xlarge", Memory: kresource.MustParse("49152Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 4, Price: 1.315}, "inf1.24xlarge": {Region: "eu-west-1", Type: "inf1.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 16, Price: 5.26}, "is4gen.medium": {Region: "eu-west-1", Type: "is4gen.medium", Memory: kresource.MustParse("6144Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.15938}, "is4gen.large": {Region: "eu-west-1", Type: "is4gen.large", Memory: kresource.MustParse("12288Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.31875}, "is4gen.xlarge": {Region: "eu-west-1", Type: "is4gen.xlarge", Memory: kresource.MustParse("24576Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.6375}, "is4gen.2xlarge": {Region: "eu-west-1", Type: "is4gen.2xlarge", Memory: kresource.MustParse("49152Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.275}, "is4gen.4xlarge": {Region: "eu-west-1", Type: "is4gen.4xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 2.55}, "is4gen.8xlarge": {Region: "eu-west-1", Type: "is4gen.8xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 5.1}, "m1.small": {Region: "eu-west-1", Type: "m1.small", Memory: kresource.MustParse("1740Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.047}, "m1.medium": {Region: "eu-west-1", Type: "m1.medium", Memory: kresource.MustParse("3840Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.095}, "m1.large": {Region: "eu-west-1", Type: "m1.large", Memory: kresource.MustParse("7680Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.19}, "m1.xlarge": {Region: "eu-west-1", Type: "m1.xlarge", Memory: kresource.MustParse("15360Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.379}, "m2.xlarge": {Region: "eu-west-1", Type: "m2.xlarge", Memory: kresource.MustParse("17510Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.275}, "m2.2xlarge": {Region: "eu-west-1", Type: "m2.2xlarge", Memory: kresource.MustParse("35020Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.55}, "m2.4xlarge": {Region: "eu-west-1", Type: "m2.4xlarge", Memory: kresource.MustParse("70041Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.1}, "m3.medium": {Region: "eu-west-1", Type: "m3.medium", Memory: kresource.MustParse("3840Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.073}, "m3.large": {Region: "eu-west-1", Type: "m3.large", Memory: kresource.MustParse("7680Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.146}, "m3.xlarge": {Region: "eu-west-1", Type: "m3.xlarge", Memory: kresource.MustParse("15360Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.293}, "m3.2xlarge": {Region: "eu-west-1", Type: "m3.2xlarge", Memory: kresource.MustParse("30720Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.585}, "m4.large": {Region: "eu-west-1", Type: "m4.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.111}, "m4.xlarge": {Region: "eu-west-1", Type: "m4.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.222}, "m4.2xlarge": {Region: "eu-west-1", Type: "m4.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.444}, "m4.4xlarge": {Region: "eu-west-1", Type: "m4.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.888}, "m4.10xlarge": {Region: "eu-west-1", Type: "m4.10xlarge", Memory: kresource.MustParse("163840Mi"), CPU: kresource.MustParse("40"), GPU: 0, Inf: 0, Price: 2.22}, "m4.16xlarge": {Region: "eu-west-1", Type: "m4.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.552}, "m5.large": {Region: "eu-west-1", Type: "m5.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.107}, "m5.xlarge": {Region: "eu-west-1", Type: "m5.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.214}, "m5.2xlarge": {Region: "eu-west-1", Type: "m5.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.428}, "m5.4xlarge": {Region: "eu-west-1", Type: "m5.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.856}, "m5.8xlarge": {Region: "eu-west-1", Type: "m5.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.712}, "m5.12xlarge": {Region: "eu-west-1", Type: "m5.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.568}, "m5.16xlarge": {Region: "eu-west-1", Type: "m5.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.424}, "m5.24xlarge": {Region: "eu-west-1", Type: "m5.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.136}, "m5.metal": {Region: "eu-west-1", Type: "m5.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.136}, "m5a.large": {Region: "eu-west-1", Type: "m5a.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.096}, "m5a.xlarge": {Region: "eu-west-1", Type: "m5a.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.192}, "m5a.2xlarge": {Region: "eu-west-1", Type: "m5a.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.384}, "m5a.4xlarge": {Region: "eu-west-1", Type: "m5a.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.768}, "m5a.8xlarge": {Region: "eu-west-1", Type: "m5a.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.536}, "m5a.12xlarge": {Region: "eu-west-1", Type: "m5a.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.304}, "m5a.16xlarge": {Region: "eu-west-1", Type: "m5a.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.072}, "m5a.24xlarge": {Region: "eu-west-1", Type: "m5a.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.608}, "m5ad.large": {Region: "eu-west-1", Type: "m5ad.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.115}, "m5ad.xlarge": {Region: "eu-west-1", Type: "m5ad.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.23}, "m5ad.2xlarge": {Region: "eu-west-1", Type: "m5ad.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.46}, "m5ad.4xlarge": {Region: "eu-west-1", Type: "m5ad.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.92}, "m5ad.8xlarge": {Region: "eu-west-1", Type: "m5ad.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.84}, "m5ad.12xlarge": {Region: "eu-west-1", Type: "m5ad.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.76}, "m5ad.16xlarge": {Region: "eu-west-1", Type: "m5ad.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.68}, "m5ad.24xlarge": {Region: "eu-west-1", Type: "m5ad.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.52}, "m5d.large": {Region: "eu-west-1", Type: "m5d.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.126}, "m5d.xlarge": {Region: "eu-west-1", Type: "m5d.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.252}, "m5d.2xlarge": {Region: "eu-west-1", Type: "m5d.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.504}, "m5d.4xlarge": {Region: "eu-west-1", Type: "m5d.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.008}, "m5d.8xlarge": {Region: "eu-west-1", Type: "m5d.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.016}, "m5d.12xlarge": {Region: "eu-west-1", Type: "m5d.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.024}, "m5d.16xlarge": {Region: "eu-west-1", Type: "m5d.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.032}, "m5d.24xlarge": {Region: "eu-west-1", Type: "m5d.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.048}, "m5d.metal": {Region: "eu-west-1", Type: "m5d.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.048}, "m5dn.large": {Region: "eu-west-1", Type: "m5dn.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.152}, "m5dn.xlarge": {Region: "eu-west-1", Type: "m5dn.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.304}, "m5dn.2xlarge": {Region: "eu-west-1", Type: "m5dn.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.608}, "m5dn.4xlarge": {Region: "eu-west-1", Type: "m5dn.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.216}, "m5dn.8xlarge": {Region: "eu-west-1", Type: "m5dn.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.432}, "m5dn.12xlarge": {Region: "eu-west-1", Type: "m5dn.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.648}, "m5dn.16xlarge": {Region: "eu-west-1", Type: "m5dn.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.864}, "m5dn.24xlarge": {Region: "eu-west-1", Type: "m5dn.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.296}, "m5dn.metal": {Region: "eu-west-1", Type: "m5dn.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.296}, "m5n.large": {Region: "eu-west-1", Type: "m5n.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.133}, "m5n.xlarge": {Region: "eu-west-1", Type: "m5n.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.266}, "m5n.2xlarge": {Region: "eu-west-1", Type: "m5n.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.532}, "m5n.4xlarge": {Region: "eu-west-1", Type: "m5n.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.064}, "m5n.8xlarge": {Region: "eu-west-1", Type: "m5n.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.128}, "m5n.12xlarge": {Region: "eu-west-1", Type: "m5n.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.192}, "m5n.16xlarge": {Region: "eu-west-1", Type: "m5n.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.256}, "m5n.24xlarge": {Region: "eu-west-1", Type: "m5n.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.384}, "m5n.metal": {Region: "eu-west-1", Type: "m5n.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.384}, "m5zn.large": {Region: "eu-west-1", Type: "m5zn.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.1841}, "m5zn.xlarge": {Region: "eu-west-1", Type: "m5zn.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.3682}, "m5zn.2xlarge": {Region: "eu-west-1", Type: "m5zn.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.7364}, "m5zn.3xlarge": {Region: "eu-west-1", Type: "m5zn.3xlarge", Memory: kresource.MustParse("49152Mi"), CPU: kresource.MustParse("12"), GPU: 0, Inf: 0, Price: 1.1046}, "m5zn.6xlarge": {Region: "eu-west-1", Type: "m5zn.6xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 0, Price: 2.2092}, "m5zn.12xlarge": {Region: "eu-west-1", Type: "m5zn.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.4184}, "m5zn.metal": {Region: "eu-west-1", Type: "m5zn.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.4184}, "m6a.large": {Region: "eu-west-1", Type: "m6a.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0963}, "m6a.xlarge": {Region: "eu-west-1", Type: "m6a.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1926}, "m6a.2xlarge": {Region: "eu-west-1", Type: "m6a.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3852}, "m6a.4xlarge": {Region: "eu-west-1", Type: "m6a.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.7704}, "m6a.8xlarge": {Region: "eu-west-1", Type: "m6a.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.5408}, "m6a.12xlarge": {Region: "eu-west-1", Type: "m6a.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.3112}, "m6a.16xlarge": {Region: "eu-west-1", Type: "m6a.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.0816}, "m6a.24xlarge": {Region: "eu-west-1", Type: "m6a.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.6224}, "m6a.32xlarge": {Region: "eu-west-1", Type: "m6a.32xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 6.1632}, "m6a.48xlarge": {Region: "eu-west-1", Type: "m6a.48xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("192"), GPU: 0, Inf: 0, Price: 9.2448}, "m6a.metal": {Region: "eu-west-1", Type: "m6a.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("192"), GPU: 0, Inf: 0, Price: 9.2448}, "m6g.medium": {Region: "eu-west-1", Type: "m6g.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.043}, "m6g.large": {Region: "eu-west-1", Type: "m6g.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.086}, "m6g.xlarge": {Region: "eu-west-1", Type: "m6g.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.172}, "m6g.2xlarge": {Region: "eu-west-1", Type: "m6g.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.344}, "m6g.4xlarge": {Region: "eu-west-1", Type: "m6g.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.688}, "m6g.8xlarge": {Region: "eu-west-1", Type: "m6g.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.376}, "m6g.12xlarge": {Region: "eu-west-1", Type: "m6g.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.064}, "m6g.16xlarge": {Region: "eu-west-1", Type: "m6g.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.752}, "m6g.metal": {Region: "eu-west-1", Type: "m6g.metal", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.752}, "m6gd.medium": {Region: "eu-west-1", Type: "m6gd.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0504}, "m6gd.large": {Region: "eu-west-1", Type: "m6gd.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.1008}, "m6gd.xlarge": {Region: "eu-west-1", Type: "m6gd.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.2016}, "m6gd.2xlarge": {Region: "eu-west-1", Type: "m6gd.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.4032}, "m6gd.4xlarge": {Region: "eu-west-1", Type: "m6gd.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.8064}, "m6gd.8xlarge": {Region: "eu-west-1", Type: "m6gd.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.6128}, "m6gd.12xlarge": {Region: "eu-west-1", Type: "m6gd.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.4192}, "m6gd.16xlarge": {Region: "eu-west-1", Type: "m6gd.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.2256}, "m6gd.metal": {Region: "eu-west-1", Type: "m6gd.metal", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.2256}, "m6i.large": {Region: "eu-west-1", Type: "m6i.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.107}, "m6i.xlarge": {Region: "eu-west-1", Type: "m6i.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.214}, "m6i.2xlarge": {Region: "eu-west-1", Type: "m6i.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.428}, "m6i.4xlarge": {Region: "eu-west-1", Type: "m6i.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.856}, "m6i.8xlarge": {Region: "eu-west-1", Type: "m6i.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.712}, "m6i.12xlarge": {Region: "eu-west-1", Type: "m6i.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.568}, "m6i.16xlarge": {Region: "eu-west-1", Type: "m6i.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.424}, "m6i.24xlarge": {Region: "eu-west-1", Type: "m6i.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.136}, "m6i.32xlarge": {Region: "eu-west-1", Type: "m6i.32xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 6.848}, "m6i.metal": {Region: "eu-west-1", Type: "m6i.metal", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 6.848}, "p2.xlarge": {Region: "eu-west-1", Type: "p2.xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("4"), GPU: 1, Inf: 0, Price: 0.972}, "p2.8xlarge": {Region: "eu-west-1", Type: "p2.8xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("32"), GPU: 8, Inf: 0, Price: 7.776}, "p2.16xlarge": {Region: "eu-west-1", Type: "p2.16xlarge", Memory: kresource.MustParse("749568Mi"), CPU: kresource.MustParse("64"), GPU: 16, Inf: 0, Price: 15.552}, "p3.2xlarge": {Region: "eu-west-1", Type: "p3.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 1, Inf: 0, Price: 3.305}, "p3.8xlarge": {Region: "eu-west-1", Type: "p3.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 4, Inf: 0, Price: 13.22}, "p3.16xlarge": {Region: "eu-west-1", Type: "p3.16xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("64"), GPU: 8, Inf: 0, Price: 26.44}, "p3dn.24xlarge": {Region: "eu-west-1", Type: "p3dn.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 8, Inf: 0, Price: 33.711}, "p4d.24xlarge": {Region: "eu-west-1", Type: "p4d.24xlarge", Memory: kresource.MustParse("1179648Mi"), CPU: kresource.MustParse("96"), GPU: 8, Inf: 0, Price: 35.39655}, "r3.large": {Region: "eu-west-1", Type: "r3.large", Memory: kresource.MustParse("15616Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.185}, "r3.xlarge": {Region: "eu-west-1", Type: "r3.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.371}, "r3.2xlarge": {Region: "eu-west-1", Type: "r3.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.741}, "r3.4xlarge": {Region: "eu-west-1", Type: "r3.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.482}, "r3.8xlarge": {Region: "eu-west-1", Type: "r3.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.964}, "r4.large": {Region: "eu-west-1", Type: "r4.large", Memory: kresource.MustParse("15616Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.1482}, "r4.xlarge": {Region: "eu-west-1", Type: "r4.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.2964}, "r4.2xlarge": {Region: "eu-west-1", Type: "r4.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.5928}, "r4.4xlarge": {Region: "eu-west-1", Type: "r4.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.1856}, "r4.8xlarge": {Region: "eu-west-1", Type: "r4.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.3712}, "r4.16xlarge": {Region: "eu-west-1", Type: "r4.16xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.7424}, "r5.large": {Region: "eu-west-1", Type: "r5.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.141}, "r5.xlarge": {Region: "eu-west-1", Type: "r5.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.282}, "r5.2xlarge": {Region: "eu-west-1", Type: "r5.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.564}, "r5.4xlarge": {Region: "eu-west-1", Type: "r5.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.128}, "r5.8xlarge": {Region: "eu-west-1", Type: "r5.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.256}, "r5.12xlarge": {Region: "eu-west-1", Type: "r5.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.384}, "r5.16xlarge": {Region: "eu-west-1", Type: "r5.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.512}, "r5.24xlarge": {Region: "eu-west-1", Type: "r5.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.768}, "r5.metal": {Region: "eu-west-1", Type: "r5.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.768}, "r5a.large": {Region: "eu-west-1", Type: "r5a.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.127}, "r5a.xlarge": {Region: "eu-west-1", Type: "r5a.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.254}, "r5a.2xlarge": {Region: "eu-west-1", Type: "r5a.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.508}, "r5a.4xlarge": {Region: "eu-west-1", Type: "r5a.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.016}, "r5a.8xlarge": {Region: "eu-west-1", Type: "r5a.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.032}, "r5a.12xlarge": {Region: "eu-west-1", Type: "r5a.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.048}, "r5a.16xlarge": {Region: "eu-west-1", Type: "r5a.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.064}, "r5a.24xlarge": {Region: "eu-west-1", Type: "r5a.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.096}, "r5ad.large": {Region: "eu-west-1", Type: "r5ad.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.146}, "r5ad.xlarge": {Region: "eu-west-1", Type: "r5ad.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.292}, "r5ad.2xlarge": {Region: "eu-west-1", Type: "r5ad.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.584}, "r5ad.4xlarge": {Region: "eu-west-1", Type: "r5ad.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.168}, "r5ad.8xlarge": {Region: "eu-west-1", Type: "r5ad.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.336}, "r5ad.12xlarge": {Region: "eu-west-1", Type: "r5ad.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.504}, "r5ad.16xlarge": {Region: "eu-west-1", Type: "r5ad.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.672}, "r5ad.24xlarge": {Region: "eu-west-1", Type: "r5ad.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.008}, "r5b.large": {Region: "eu-west-1", Type: "r5b.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.167}, "r5b.xlarge": {Region: "eu-west-1", Type: "r5b.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.334}, "r5b.2xlarge": {Region: "eu-west-1", Type: "r5b.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.668}, "r5b.4xlarge": {Region: "eu-west-1", Type: "r5b.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.336}, "r5b.8xlarge": {Region: "eu-west-1", Type: "r5b.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.672}, "r5b.12xlarge": {Region: "eu-west-1", Type: "r5b.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.008}, "r5b.16xlarge": {Region: "eu-west-1", Type: "r5b.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.344}, "r5b.24xlarge": {Region: "eu-west-1", Type: "r5b.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.016}, "r5b.metal": {Region: "eu-west-1", Type: "r5b.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.016}, "r5d.large": {Region: "eu-west-1", Type: "r5d.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.16}, "r5d.xlarge": {Region: "eu-west-1", Type: "r5d.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.32}, "r5d.2xlarge": {Region: "eu-west-1", Type: "r5d.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.64}, "r5d.4xlarge": {Region: "eu-west-1", Type: "r5d.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.28}, "r5d.8xlarge": {Region: "eu-west-1", Type: "r5d.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.56}, "r5d.12xlarge": {Region: "eu-west-1", Type: "r5d.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.84}, "r5d.16xlarge": {Region: "eu-west-1", Type: "r5d.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.12}, "r5d.24xlarge": {Region: "eu-west-1", Type: "r5d.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.68}, "r5d.metal": {Region: "eu-west-1", Type: "r5d.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.68}, "r5dn.large": {Region: "eu-west-1", Type: "r5dn.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.186}, "r5dn.xlarge": {Region: "eu-west-1", Type: "r5dn.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.372}, "r5dn.2xlarge": {Region: "eu-west-1", Type: "r5dn.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.744}, "r5dn.4xlarge": {Region: "eu-west-1", Type: "r5dn.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.488}, "r5dn.8xlarge": {Region: "eu-west-1", Type: "r5dn.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.976}, "r5dn.12xlarge": {Region: "eu-west-1", Type: "r5dn.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.464}, "r5dn.16xlarge": {Region: "eu-west-1", Type: "r5dn.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.952}, "r5dn.24xlarge": {Region: "eu-west-1", Type: "r5dn.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.928}, "r5dn.metal": {Region: "eu-west-1", Type: "r5dn.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.928}, "r5n.large": {Region: "eu-west-1", Type: "r5n.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.167}, "r5n.xlarge": {Region: "eu-west-1", Type: "r5n.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.334}, "r5n.2xlarge": {Region: "eu-west-1", Type: "r5n.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.668}, "r5n.4xlarge": {Region: "eu-west-1", Type: "r5n.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.336}, "r5n.8xlarge": {Region: "eu-west-1", Type: "r5n.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.672}, "r5n.12xlarge": {Region: "eu-west-1", Type: "r5n.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.008}, "r5n.16xlarge": {Region: "eu-west-1", Type: "r5n.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.344}, "r5n.24xlarge": {Region: "eu-west-1", Type: "r5n.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.016}, "r5n.metal": {Region: "eu-west-1", Type: "r5n.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.016}, "r6g.medium": {Region: "eu-west-1", Type: "r6g.medium", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0564}, "r6g.large": {Region: "eu-west-1", Type: "r6g.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.1128}, "r6g.xlarge": {Region: "eu-west-1", Type: "r6g.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.2256}, "r6g.2xlarge": {Region: "eu-west-1", Type: "r6g.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.4512}, "r6g.4xlarge": {Region: "eu-west-1", Type: "r6g.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.9024}, "r6g.8xlarge": {Region: "eu-west-1", Type: "r6g.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.8048}, "r6g.12xlarge": {Region: "eu-west-1", Type: "r6g.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.7072}, "r6g.16xlarge": {Region: "eu-west-1", Type: "r6g.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.6096}, "r6g.metal": {Region: "eu-west-1", Type: "r6g.metal", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.6096}, "r6gd.medium": {Region: "eu-west-1", Type: "r6gd.medium", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.064}, "r6gd.large": {Region: "eu-west-1", Type: "r6gd.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.128}, "r6gd.xlarge": {Region: "eu-west-1", Type: "r6gd.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.256}, "r6gd.2xlarge": {Region: "eu-west-1", Type: "r6gd.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.512}, "r6gd.4xlarge": {Region: "eu-west-1", Type: "r6gd.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.024}, "r6gd.8xlarge": {Region: "eu-west-1", Type: "r6gd.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.048}, "r6gd.12xlarge": {Region: "eu-west-1", Type: "r6gd.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.072}, "r6gd.16xlarge": {Region: "eu-west-1", Type: "r6gd.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.096}, "r6gd.metal": {Region: "eu-west-1", Type: "r6gd.metal", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.096}, "r6i.large": {Region: "eu-west-1", Type: "r6i.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.141}, "r6i.xlarge": {Region: "eu-west-1", Type: "r6i.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.282}, "r6i.2xlarge": {Region: "eu-west-1", Type: "r6i.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.564}, "r6i.4xlarge": {Region: "eu-west-1", Type: "r6i.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.128}, "r6i.8xlarge": {Region: "eu-west-1", Type: "r6i.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.256}, "r6i.12xlarge": {Region: "eu-west-1", Type: "r6i.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.384}, "r6i.16xlarge": {Region: "eu-west-1", Type: "r6i.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.512}, "r6i.24xlarge": {Region: "eu-west-1", Type: "r6i.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.768}, "r6i.32xlarge": {Region: "eu-west-1", Type: "r6i.32xlarge", Memory: kresource.MustParse("1048576Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 9.024}, "r6i.metal": {Region: "eu-west-1", Type: "r6i.metal", Memory: kresource.MustParse("1048576Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 9.024}, "t1.micro": {Region: "eu-west-1", Type: "t1.micro", Memory: kresource.MustParse("627Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.02}, "t2.nano": {Region: "eu-west-1", Type: "t2.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0063}, "t2.micro": {Region: "eu-west-1", Type: "t2.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0126}, "t2.small": {Region: "eu-west-1", Type: "t2.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.025}, "t2.medium": {Region: "eu-west-1", Type: "t2.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.05}, "t2.large": {Region: "eu-west-1", Type: "t2.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.1008}, "t2.xlarge": {Region: "eu-west-1", Type: "t2.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.2016}, "t2.2xlarge": {Region: "eu-west-1", Type: "t2.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.4032}, "t3.nano": {Region: "eu-west-1", Type: "t3.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0057}, "t3.micro": {Region: "eu-west-1", Type: "t3.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0114}, "t3.small": {Region: "eu-west-1", Type: "t3.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0228}, "t3.medium": {Region: "eu-west-1", Type: "t3.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0456}, "t3.large": {Region: "eu-west-1", Type: "t3.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0912}, "t3.xlarge": {Region: "eu-west-1", Type: "t3.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1824}, "t3.2xlarge": {Region: "eu-west-1", Type: "t3.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3648}, "t3a.nano": {Region: "eu-west-1", Type: "t3a.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0051}, "t3a.micro": {Region: "eu-west-1", Type: "t3a.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0102}, "t3a.small": {Region: "eu-west-1", Type: "t3a.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0204}, "t3a.medium": {Region: "eu-west-1", Type: "t3a.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0408}, "t3a.large": {Region: "eu-west-1", Type: "t3a.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0816}, "t3a.xlarge": {Region: "eu-west-1", Type: "t3a.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1632}, "t3a.2xlarge": {Region: "eu-west-1", Type: "t3a.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3264}, "t4g.nano": {Region: "eu-west-1", Type: "t4g.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0046}, "t4g.micro": {Region: "eu-west-1", Type: "t4g.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0092}, "t4g.small": {Region: "eu-west-1", Type: "t4g.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0184}, "t4g.medium": {Region: "eu-west-1", Type: "t4g.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0368}, "t4g.large": {Region: "eu-west-1", Type: "t4g.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0736}, "t4g.xlarge": {Region: "eu-west-1", Type: "t4g.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1472}, "t4g.2xlarge": {Region: "eu-west-1", Type: "t4g.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.2944}, "u-12tb1.112xlarge": {Region: "eu-west-1", Type: "u-12tb1.112xlarge", Memory: kresource.MustParse("12582912Mi"), CPU: kresource.MustParse("448"), GPU: 0, Inf: 0, Price: 122.2}, "u-3tb1.56xlarge": {Region: "eu-west-1", Type: "u-3tb1.56xlarge", Memory: kresource.MustParse("3145728Mi"), CPU: kresource.MustParse("224"), GPU: 0, Inf: 0, Price: 30.55}, "u-6tb1.56xlarge": {Region: "eu-west-1", Type: "u-6tb1.56xlarge", Memory: kresource.MustParse("6291456Mi"), CPU: kresource.MustParse("224"), GPU: 0, Inf: 0, Price: 51.92818}, "u-6tb1.112xlarge": {Region: "eu-west-1", Type: "u-6tb1.112xlarge", Memory: kresource.MustParse("6291456Mi"), CPU: kresource.MustParse("448"), GPU: 0, Inf: 0, Price: 61.1}, "u-9tb1.112xlarge": {Region: "eu-west-1", Type: "u-9tb1.112xlarge", Memory: kresource.MustParse("9437184Mi"), CPU: kresource.MustParse("448"), GPU: 0, Inf: 0, Price: 91.65}, "vt1.3xlarge": {Region: "eu-west-1", Type: "vt1.3xlarge", Memory: kresource.MustParse("24576Mi"), CPU: kresource.MustParse("12"), GPU: 0, Inf: 0, Price: 0.73412}, "vt1.6xlarge": {Region: "eu-west-1", Type: "vt1.6xlarge", Memory: kresource.MustParse("49152Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 0, Price: 1.46824}, "vt1.24xlarge": {Region: "eu-west-1", Type: "vt1.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.87294}, "x1.16xlarge": {Region: "eu-west-1", Type: "x1.16xlarge", Memory: kresource.MustParse("999424Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 8.003}, "x1.32xlarge": {Region: "eu-west-1", Type: "x1.32xlarge", Memory: kresource.MustParse("1998848Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 16.006}, "x1e.xlarge": {Region: "eu-west-1", Type: "x1e.xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 1.0}, "x1e.2xlarge": {Region: "eu-west-1", Type: "x1e.2xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 2.0}, "x1e.4xlarge": {Region: "eu-west-1", Type: "x1e.4xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 4.0}, "x1e.8xlarge": {Region: "eu-west-1", Type: "x1e.8xlarge", Memory: kresource.MustParse("999424Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 8.0}, "x1e.16xlarge": {Region: "eu-west-1", Type: "x1e.16xlarge", Memory: kresource.MustParse("1998848Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 16.0}, "x1e.32xlarge": {Region: "eu-west-1", Type: "x1e.32xlarge", Memory: kresource.MustParse("3997696Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 32.0}, "x2gd.medium": {Region: "eu-west-1", Type: "x2gd.medium", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.1}, "x2gd.large": {Region: "eu-west-1", Type: "x2gd.large", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.2}, "x2gd.xlarge": {Region: "eu-west-1", Type: "x2gd.xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.4}, "x2gd.2xlarge": {Region: "eu-west-1", Type: "x2gd.2xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.8}, "x2gd.4xlarge": {Region: "eu-west-1", Type: "x2gd.4xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.6}, "x2gd.8xlarge": {Region: "eu-west-1", Type: "x2gd.8xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 3.2}, "x2gd.12xlarge": {Region: "eu-west-1", Type: "x2gd.12xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.8}, "x2gd.16xlarge": {Region: "eu-west-1", Type: "x2gd.16xlarge", Memory: kresource.MustParse("1048576Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 6.4}, "x2gd.metal": {Region: "eu-west-1", Type: "x2gd.metal", Memory: kresource.MustParse("1048576Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 6.4}, "x2idn.16xlarge": {Region: "eu-west-1", Type: "x2idn.16xlarge", Memory: kresource.MustParse("1048576Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 8.003}, "x2idn.24xlarge": {Region: "eu-west-1", Type: "x2idn.24xlarge", Memory: kresource.MustParse("1572864Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 12.0045}, "x2idn.32xlarge": {Region: "eu-west-1", Type: "x2idn.32xlarge", Memory: kresource.MustParse("2097152Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 16.006}, "x2iedn.xlarge": {Region: "eu-west-1", Type: "x2iedn.xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 1.00038}, "x2iedn.2xlarge": {Region: "eu-west-1", Type: "x2iedn.2xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 2.00075}, "x2iedn.4xlarge": {Region: "eu-west-1", Type: "x2iedn.4xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 4.0015}, "x2iedn.8xlarge": {Region: "eu-west-1", Type: "x2iedn.8xlarge", Memory: kresource.MustParse("1048576Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 8.003}, "x2iedn.16xlarge": {Region: "eu-west-1", Type: "x2iedn.16xlarge", Memory: kresource.MustParse("2097152Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 16.006}, "x2iedn.24xlarge": {Region: "eu-west-1", Type: "x2iedn.24xlarge", Memory: kresource.MustParse("3145728Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 24.009}, "x2iedn.32xlarge": {Region: "eu-west-1", Type: "x2iedn.32xlarge", Memory: kresource.MustParse("4194304Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 32.012}, "x2iezn.2xlarge": {Region: "eu-west-1", Type: "x2iezn.2xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 2.0}, "x2iezn.4xlarge": {Region: "eu-west-1", Type: "x2iezn.4xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 4.0}, "x2iezn.6xlarge": {Region: "eu-west-1", Type: "x2iezn.6xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 0, Price: 6.0}, "x2iezn.8xlarge": {Region: "eu-west-1", Type: "x2iezn.8xlarge", Memory: kresource.MustParse("1048576Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 8.0}, "x2iezn.12xlarge": {Region: "eu-west-1", Type: "x2iezn.12xlarge", Memory: kresource.MustParse("1572864Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 12.0}, "x2iezn.metal": {Region: "eu-west-1", Type: "x2iezn.metal", Memory: kresource.MustParse("1572864Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 12.0}, "z1d.large": {Region: "eu-west-1", Type: "z1d.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.208}, "z1d.xlarge": {Region: "eu-west-1", Type: "z1d.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.416}, "z1d.2xlarge": {Region: "eu-west-1", Type: "z1d.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.832}, "z1d.3xlarge": {Region: "eu-west-1", Type: "z1d.3xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("12"), GPU: 0, Inf: 0, Price: 1.248}, "z1d.6xlarge": {Region: "eu-west-1", Type: "z1d.6xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 0, Price: 2.496}, "z1d.12xlarge": {Region: "eu-west-1", Type: "z1d.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.992}, "z1d.metal": {Region: "eu-west-1", Type: "z1d.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.992}, }, "eu-west-2": { "c4.large": {Region: "eu-west-2", Type: "c4.large", Memory: kresource.MustParse("3840Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.119}, "c4.xlarge": {Region: "eu-west-2", Type: "c4.xlarge", Memory: kresource.MustParse("7680Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.237}, "c4.2xlarge": {Region: "eu-west-2", Type: "c4.2xlarge", Memory: kresource.MustParse("15360Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.476}, "c4.4xlarge": {Region: "eu-west-2", Type: "c4.4xlarge", Memory: kresource.MustParse("30720Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.95}, "c4.8xlarge": {Region: "eu-west-2", Type: "c4.8xlarge", Memory: kresource.MustParse("61440Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 1.902}, "c5.large": {Region: "eu-west-2", Type: "c5.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.101}, "c5.xlarge": {Region: "eu-west-2", Type: "c5.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.202}, "c5.2xlarge": {Region: "eu-west-2", Type: "c5.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.404}, "c5.4xlarge": {Region: "eu-west-2", Type: "c5.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.808}, "c5.9xlarge": {Region: "eu-west-2", Type: "c5.9xlarge", Memory: kresource.MustParse("73728Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 1.818}, "c5.12xlarge": {Region: "eu-west-2", Type: "c5.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.424}, "c5.18xlarge": {Region: "eu-west-2", Type: "c5.18xlarge", Memory: kresource.MustParse("147456Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 3.636}, "c5.24xlarge": {Region: "eu-west-2", Type: "c5.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.848}, "c5.metal": {Region: "eu-west-2", Type: "c5.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.848}, "c5a.large": {Region: "eu-west-2", Type: "c5a.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.091}, "c5a.xlarge": {Region: "eu-west-2", Type: "c5a.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.182}, "c5a.2xlarge": {Region: "eu-west-2", Type: "c5a.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.364}, "c5a.4xlarge": {Region: "eu-west-2", Type: "c5a.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.728}, "c5a.8xlarge": {Region: "eu-west-2", Type: "c5a.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.456}, "c5a.12xlarge": {Region: "eu-west-2", Type: "c5a.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.184}, "c5a.16xlarge": {Region: "eu-west-2", Type: "c5a.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.912}, "c5a.24xlarge": {Region: "eu-west-2", Type: "c5a.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.368}, "c5d.large": {Region: "eu-west-2", Type: "c5d.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.115}, "c5d.xlarge": {Region: "eu-west-2", Type: "c5d.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.23}, "c5d.2xlarge": {Region: "eu-west-2", Type: "c5d.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.46}, "c5d.4xlarge": {Region: "eu-west-2", Type: "c5d.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.92}, "c5d.9xlarge": {Region: "eu-west-2", Type: "c5d.9xlarge", Memory: kresource.MustParse("73728Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 2.07}, "c5d.12xlarge": {Region: "eu-west-2", Type: "c5d.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.76}, "c5d.18xlarge": {Region: "eu-west-2", Type: "c5d.18xlarge", Memory: kresource.MustParse("147456Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 4.14}, "c5d.24xlarge": {Region: "eu-west-2", Type: "c5d.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.52}, "c5d.metal": {Region: "eu-west-2", Type: "c5d.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.52}, "c5n.large": {Region: "eu-west-2", Type: "c5n.large", Memory: kresource.MustParse("5376Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.128}, "c5n.xlarge": {Region: "eu-west-2", Type: "c5n.xlarge", Memory: kresource.MustParse("10752Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.256}, "c5n.2xlarge": {Region: "eu-west-2", Type: "c5n.2xlarge", Memory: kresource.MustParse("21504Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.512}, "c5n.4xlarge": {Region: "eu-west-2", Type: "c5n.4xlarge", Memory: kresource.MustParse("43008Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.024}, "c5n.9xlarge": {Region: "eu-west-2", Type: "c5n.9xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 2.304}, "c5n.18xlarge": {Region: "eu-west-2", Type: "c5n.18xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 4.608}, "c5n.metal": {Region: "eu-west-2", Type: "c5n.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 4.608}, "c6g.medium": {Region: "eu-west-2", Type: "c6g.medium", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0404}, "c6g.large": {Region: "eu-west-2", Type: "c6g.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0808}, "c6g.xlarge": {Region: "eu-west-2", Type: "c6g.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1616}, "c6g.2xlarge": {Region: "eu-west-2", Type: "c6g.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3232}, "c6g.4xlarge": {Region: "eu-west-2", Type: "c6g.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.6464}, "c6g.8xlarge": {Region: "eu-west-2", Type: "c6g.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.2928}, "c6g.12xlarge": {Region: "eu-west-2", Type: "c6g.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 1.9392}, "c6g.16xlarge": {Region: "eu-west-2", Type: "c6g.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.5856}, "c6g.metal": {Region: "eu-west-2", Type: "c6g.metal", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.5856}, "c6gd.medium": {Region: "eu-west-2", Type: "c6gd.medium", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.046}, "c6gd.large": {Region: "eu-west-2", Type: "c6gd.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.092}, "c6gd.xlarge": {Region: "eu-west-2", Type: "c6gd.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.184}, "c6gd.2xlarge": {Region: "eu-west-2", Type: "c6gd.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.368}, "c6gd.4xlarge": {Region: "eu-west-2", Type: "c6gd.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.736}, "c6gd.8xlarge": {Region: "eu-west-2", Type: "c6gd.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.472}, "c6gd.12xlarge": {Region: "eu-west-2", Type: "c6gd.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.208}, "c6gd.16xlarge": {Region: "eu-west-2", Type: "c6gd.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.944}, "c6gd.metal": {Region: "eu-west-2", Type: "c6gd.metal", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.944}, "c6gn.medium": {Region: "eu-west-2", Type: "c6gn.medium", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.05125}, "c6gn.large": {Region: "eu-west-2", Type: "c6gn.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.1025}, "c6gn.xlarge": {Region: "eu-west-2", Type: "c6gn.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.205}, "c6gn.2xlarge": {Region: "eu-west-2", Type: "c6gn.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.41}, "c6gn.4xlarge": {Region: "eu-west-2", Type: "c6gn.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.82}, "c6gn.8xlarge": {Region: "eu-west-2", Type: "c6gn.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.64}, "c6gn.12xlarge": {Region: "eu-west-2", Type: "c6gn.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.46}, "c6gn.16xlarge": {Region: "eu-west-2", Type: "c6gn.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.28}, "c6i.large": {Region: "eu-west-2", Type: "c6i.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.101}, "c6i.xlarge": {Region: "eu-west-2", Type: "c6i.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.202}, "c6i.2xlarge": {Region: "eu-west-2", Type: "c6i.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.404}, "c6i.4xlarge": {Region: "eu-west-2", Type: "c6i.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.808}, "c6i.8xlarge": {Region: "eu-west-2", Type: "c6i.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.616}, "c6i.12xlarge": {Region: "eu-west-2", Type: "c6i.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.424}, "c6i.16xlarge": {Region: "eu-west-2", Type: "c6i.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.232}, "c6i.24xlarge": {Region: "eu-west-2", Type: "c6i.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.848}, "c6i.32xlarge": {Region: "eu-west-2", Type: "c6i.32xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 6.464}, "c6i.metal": {Region: "eu-west-2", Type: "c6i.metal", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 6.464}, "d2.xlarge": {Region: "eu-west-2", Type: "d2.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.772}, "d2.2xlarge": {Region: "eu-west-2", Type: "d2.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.544}, "d2.4xlarge": {Region: "eu-west-2", Type: "d2.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 3.087}, "d2.8xlarge": {Region: "eu-west-2", Type: "d2.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 6.174}, "d3.xlarge": {Region: "eu-west-2", Type: "d3.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.64}, "d3.2xlarge": {Region: "eu-west-2", Type: "d3.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.28}, "d3.4xlarge": {Region: "eu-west-2", Type: "d3.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 2.559}, "d3.8xlarge": {Region: "eu-west-2", Type: "d3.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 5.11824}, "f1.2xlarge": {Region: "eu-west-2", Type: "f1.2xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.906}, "f1.4xlarge": {Region: "eu-west-2", Type: "f1.4xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 3.812}, "g3.4xlarge": {Region: "eu-west-2", Type: "g3.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 1, Inf: 0, Price: 1.429}, "g3.8xlarge": {Region: "eu-west-2", Type: "g3.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 2, Inf: 0, Price: 2.858}, "g3.16xlarge": {Region: "eu-west-2", Type: "g3.16xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("64"), GPU: 4, Inf: 0, Price: 5.716}, "g3s.xlarge": {Region: "eu-west-2", Type: "g3s.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 1, Inf: 0, Price: 0.94}, "g4ad.xlarge": {Region: "eu-west-2", Type: "g4ad.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 1, Inf: 0, Price: 0.44271}, "g4ad.2xlarge": {Region: "eu-west-2", Type: "g4ad.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 1, Inf: 0, Price: 0.63292}, "g4ad.4xlarge": {Region: "eu-west-2", Type: "g4ad.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 1, Inf: 0, Price: 1.014}, "g4ad.8xlarge": {Region: "eu-west-2", Type: "g4ad.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 2, Inf: 0, Price: 2.028}, "g4ad.16xlarge": {Region: "eu-west-2", Type: "g4ad.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 4, Inf: 0, Price: 4.056}, "g4dn.xlarge": {Region: "eu-west-2", Type: "g4dn.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 1, Inf: 0, Price: 0.615}, "g4dn.2xlarge": {Region: "eu-west-2", Type: "g4dn.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 1, Inf: 0, Price: 0.88}, "g4dn.4xlarge": {Region: "eu-west-2", Type: "g4dn.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 1, Inf: 0, Price: 1.409}, "g4dn.8xlarge": {Region: "eu-west-2", Type: "g4dn.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 1, Inf: 0, Price: 2.546}, "g4dn.12xlarge": {Region: "eu-west-2", Type: "g4dn.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 4, Inf: 0, Price: 4.577}, "g4dn.16xlarge": {Region: "eu-west-2", Type: "g4dn.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 1, Inf: 0, Price: 5.092}, "g4dn.metal": {Region: "eu-west-2", Type: "g4dn.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 8, Inf: 0, Price: 9.154}, "i3.large": {Region: "eu-west-2", Type: "i3.large", Memory: kresource.MustParse("15616Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.181}, "i3.xlarge": {Region: "eu-west-2", Type: "i3.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.362}, "i3.2xlarge": {Region: "eu-west-2", Type: "i3.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.724}, "i3.4xlarge": {Region: "eu-west-2", Type: "i3.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.448}, "i3.8xlarge": {Region: "eu-west-2", Type: "i3.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.896}, "i3.16xlarge": {Region: "eu-west-2", Type: "i3.16xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.792}, "i3.metal": {Region: "eu-west-2", Type: "i3.metal", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.792}, "i3en.large": {Region: "eu-west-2", Type: "i3en.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.263}, "i3en.xlarge": {Region: "eu-west-2", Type: "i3en.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.526}, "i3en.2xlarge": {Region: "eu-west-2", Type: "i3en.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.052}, "i3en.3xlarge": {Region: "eu-west-2", Type: "i3en.3xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("12"), GPU: 0, Inf: 0, Price: 1.578}, "i3en.6xlarge": {Region: "eu-west-2", Type: "i3en.6xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 0, Price: 3.156}, "i3en.12xlarge": {Region: "eu-west-2", Type: "i3en.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 6.312}, "i3en.24xlarge": {Region: "eu-west-2", Type: "i3en.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 12.624}, "i3en.metal": {Region: "eu-west-2", Type: "i3en.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 12.624}, "inf1.xlarge": {Region: "eu-west-2", Type: "inf1.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 1, Price: 0.267}, "inf1.2xlarge": {Region: "eu-west-2", Type: "inf1.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 1, Price: 0.424}, "inf1.6xlarge": {Region: "eu-west-2", Type: "inf1.6xlarge", Memory: kresource.MustParse("49152Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 4, Price: 1.382}, "inf1.24xlarge": {Region: "eu-west-2", Type: "inf1.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 16, Price: 5.53}, "m4.large": {Region: "eu-west-2", Type: "m4.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.116}, "m4.xlarge": {Region: "eu-west-2", Type: "m4.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.232}, "m4.2xlarge": {Region: "eu-west-2", Type: "m4.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.464}, "m4.4xlarge": {Region: "eu-west-2", Type: "m4.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.928}, "m4.10xlarge": {Region: "eu-west-2", Type: "m4.10xlarge", Memory: kresource.MustParse("163840Mi"), CPU: kresource.MustParse("40"), GPU: 0, Inf: 0, Price: 2.32}, "m4.16xlarge": {Region: "eu-west-2", Type: "m4.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.712}, "m5.large": {Region: "eu-west-2", Type: "m5.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.111}, "m5.xlarge": {Region: "eu-west-2", Type: "m5.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.222}, "m5.2xlarge": {Region: "eu-west-2", Type: "m5.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.444}, "m5.4xlarge": {Region: "eu-west-2", Type: "m5.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.888}, "m5.8xlarge": {Region: "eu-west-2", Type: "m5.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.776}, "m5.12xlarge": {Region: "eu-west-2", Type: "m5.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.664}, "m5.16xlarge": {Region: "eu-west-2", Type: "m5.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.552}, "m5.24xlarge": {Region: "eu-west-2", Type: "m5.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.328}, "m5.metal": {Region: "eu-west-2", Type: "m5.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.328}, "m5a.large": {Region: "eu-west-2", Type: "m5a.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.1}, "m5a.xlarge": {Region: "eu-west-2", Type: "m5a.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.2}, "m5a.2xlarge": {Region: "eu-west-2", Type: "m5a.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.4}, "m5a.4xlarge": {Region: "eu-west-2", Type: "m5a.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.8}, "m5a.8xlarge": {Region: "eu-west-2", Type: "m5a.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.6}, "m5a.12xlarge": {Region: "eu-west-2", Type: "m5a.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.4}, "m5a.16xlarge": {Region: "eu-west-2", Type: "m5a.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.2}, "m5a.24xlarge": {Region: "eu-west-2", Type: "m5a.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.8}, "m5ad.large": {Region: "eu-west-2", Type: "m5ad.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.12}, "m5ad.xlarge": {Region: "eu-west-2", Type: "m5ad.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.24}, "m5ad.2xlarge": {Region: "eu-west-2", Type: "m5ad.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.48}, "m5ad.4xlarge": {Region: "eu-west-2", Type: "m5ad.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.96}, "m5ad.8xlarge": {Region: "eu-west-2", Type: "m5ad.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.92}, "m5ad.12xlarge": {Region: "eu-west-2", Type: "m5ad.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.88}, "m5ad.16xlarge": {Region: "eu-west-2", Type: "m5ad.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.84}, "m5ad.24xlarge": {Region: "eu-west-2", Type: "m5ad.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.76}, "m5d.large": {Region: "eu-west-2", Type: "m5d.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.131}, "m5d.xlarge": {Region: "eu-west-2", Type: "m5d.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.262}, "m5d.2xlarge": {Region: "eu-west-2", Type: "m5d.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.524}, "m5d.4xlarge": {Region: "eu-west-2", Type: "m5d.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.048}, "m5d.8xlarge": {Region: "eu-west-2", Type: "m5d.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.096}, "m5d.12xlarge": {Region: "eu-west-2", Type: "m5d.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.144}, "m5d.16xlarge": {Region: "eu-west-2", Type: "m5d.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.192}, "m5d.24xlarge": {Region: "eu-west-2", Type: "m5d.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.288}, "m5d.metal": {Region: "eu-west-2", Type: "m5d.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.288}, "m6g.medium": {Region: "eu-west-2", Type: "m6g.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0444}, "m6g.large": {Region: "eu-west-2", Type: "m6g.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0888}, "m6g.xlarge": {Region: "eu-west-2", Type: "m6g.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1776}, "m6g.2xlarge": {Region: "eu-west-2", Type: "m6g.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3552}, "m6g.4xlarge": {Region: "eu-west-2", Type: "m6g.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.7104}, "m6g.8xlarge": {Region: "eu-west-2", Type: "m6g.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.4208}, "m6g.12xlarge": {Region: "eu-west-2", Type: "m6g.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.1312}, "m6g.16xlarge": {Region: "eu-west-2", Type: "m6g.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.8416}, "m6g.metal": {Region: "eu-west-2", Type: "m6g.metal", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.8416}, "m6gd.medium": {Region: "eu-west-2", Type: "m6gd.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0524}, "m6gd.large": {Region: "eu-west-2", Type: "m6gd.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.1048}, "m6gd.xlarge": {Region: "eu-west-2", Type: "m6gd.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.2096}, "m6gd.2xlarge": {Region: "eu-west-2", Type: "m6gd.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.4192}, "m6gd.4xlarge": {Region: "eu-west-2", Type: "m6gd.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.8384}, "m6gd.8xlarge": {Region: "eu-west-2", Type: "m6gd.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.6768}, "m6gd.12xlarge": {Region: "eu-west-2", Type: "m6gd.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.5152}, "m6gd.16xlarge": {Region: "eu-west-2", Type: "m6gd.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.3536}, "m6gd.metal": {Region: "eu-west-2", Type: "m6gd.metal", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.3536}, "m6i.large": {Region: "eu-west-2", Type: "m6i.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.111}, "m6i.xlarge": {Region: "eu-west-2", Type: "m6i.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.222}, "m6i.2xlarge": {Region: "eu-west-2", Type: "m6i.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.444}, "m6i.4xlarge": {Region: "eu-west-2", Type: "m6i.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.888}, "m6i.8xlarge": {Region: "eu-west-2", Type: "m6i.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.776}, "m6i.12xlarge": {Region: "eu-west-2", Type: "m6i.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.664}, "m6i.16xlarge": {Region: "eu-west-2", Type: "m6i.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.552}, "m6i.24xlarge": {Region: "eu-west-2", Type: "m6i.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.328}, "m6i.32xlarge": {Region: "eu-west-2", Type: "m6i.32xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 7.104}, "m6i.metal": {Region: "eu-west-2", Type: "m6i.metal", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 7.104}, "p3.2xlarge": {Region: "eu-west-2", Type: "p3.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 1, Inf: 0, Price: 3.589}, "p3.8xlarge": {Region: "eu-west-2", Type: "p3.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 4, Inf: 0, Price: 14.356}, "p3.16xlarge": {Region: "eu-west-2", Type: "p3.16xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("64"), GPU: 8, Inf: 0, Price: 28.712}, "r4.large": {Region: "eu-west-2", Type: "r4.large", Memory: kresource.MustParse("15616Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.156}, "r4.xlarge": {Region: "eu-west-2", Type: "r4.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.312}, "r4.2xlarge": {Region: "eu-west-2", Type: "r4.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.624}, "r4.4xlarge": {Region: "eu-west-2", Type: "r4.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.248}, "r4.8xlarge": {Region: "eu-west-2", Type: "r4.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.496}, "r4.16xlarge": {Region: "eu-west-2", Type: "r4.16xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.992}, "r5.large": {Region: "eu-west-2", Type: "r5.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.148}, "r5.xlarge": {Region: "eu-west-2", Type: "r5.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.296}, "r5.2xlarge": {Region: "eu-west-2", Type: "r5.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.592}, "r5.4xlarge": {Region: "eu-west-2", Type: "r5.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.184}, "r5.8xlarge": {Region: "eu-west-2", Type: "r5.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.368}, "r5.12xlarge": {Region: "eu-west-2", Type: "r5.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.552}, "r5.16xlarge": {Region: "eu-west-2", Type: "r5.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.736}, "r5.24xlarge": {Region: "eu-west-2", Type: "r5.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.104}, "r5.metal": {Region: "eu-west-2", Type: "r5.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.104}, "r5a.large": {Region: "eu-west-2", Type: "r5a.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.133}, "r5a.xlarge": {Region: "eu-west-2", Type: "r5a.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.266}, "r5a.2xlarge": {Region: "eu-west-2", Type: "r5a.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.532}, "r5a.4xlarge": {Region: "eu-west-2", Type: "r5a.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.064}, "r5a.8xlarge": {Region: "eu-west-2", Type: "r5a.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.128}, "r5a.12xlarge": {Region: "eu-west-2", Type: "r5a.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.192}, "r5a.16xlarge": {Region: "eu-west-2", Type: "r5a.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.256}, "r5a.24xlarge": {Region: "eu-west-2", Type: "r5a.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.384}, "r5ad.large": {Region: "eu-west-2", Type: "r5ad.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.154}, "r5ad.xlarge": {Region: "eu-west-2", Type: "r5ad.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.308}, "r5ad.2xlarge": {Region: "eu-west-2", Type: "r5ad.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.616}, "r5ad.4xlarge": {Region: "eu-west-2", Type: "r5ad.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.232}, "r5ad.8xlarge": {Region: "eu-west-2", Type: "r5ad.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.464}, "r5ad.12xlarge": {Region: "eu-west-2", Type: "r5ad.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.696}, "r5ad.16xlarge": {Region: "eu-west-2", Type: "r5ad.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.928}, "r5ad.24xlarge": {Region: "eu-west-2", Type: "r5ad.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.392}, "r5b.large": {Region: "eu-west-2", Type: "r5b.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.175}, "r5b.xlarge": {Region: "eu-west-2", Type: "r5b.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.35}, "r5b.2xlarge": {Region: "eu-west-2", Type: "r5b.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.7}, "r5b.4xlarge": {Region: "eu-west-2", Type: "r5b.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.4}, "r5b.8xlarge": {Region: "eu-west-2", Type: "r5b.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.8}, "r5b.12xlarge": {Region: "eu-west-2", Type: "r5b.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.2}, "r5b.16xlarge": {Region: "eu-west-2", Type: "r5b.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.6}, "r5b.24xlarge": {Region: "eu-west-2", Type: "r5b.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.4}, "r5b.metal": {Region: "eu-west-2", Type: "r5b.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.4}, "r5d.large": {Region: "eu-west-2", Type: "r5d.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.169}, "r5d.xlarge": {Region: "eu-west-2", Type: "r5d.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.338}, "r5d.2xlarge": {Region: "eu-west-2", Type: "r5d.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.676}, "r5d.4xlarge": {Region: "eu-west-2", Type: "r5d.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.352}, "r5d.8xlarge": {Region: "eu-west-2", Type: "r5d.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.704}, "r5d.12xlarge": {Region: "eu-west-2", Type: "r5d.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.056}, "r5d.16xlarge": {Region: "eu-west-2", Type: "r5d.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.408}, "r5d.24xlarge": {Region: "eu-west-2", Type: "r5d.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.112}, "r5d.metal": {Region: "eu-west-2", Type: "r5d.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.112}, "r5n.large": {Region: "eu-west-2", Type: "r5n.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.175}, "r5n.xlarge": {Region: "eu-west-2", Type: "r5n.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.35}, "r5n.2xlarge": {Region: "eu-west-2", Type: "r5n.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.7}, "r5n.4xlarge": {Region: "eu-west-2", Type: "r5n.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.4}, "r5n.8xlarge": {Region: "eu-west-2", Type: "r5n.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.8}, "r5n.12xlarge": {Region: "eu-west-2", Type: "r5n.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.2}, "r5n.16xlarge": {Region: "eu-west-2", Type: "r5n.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.6}, "r5n.24xlarge": {Region: "eu-west-2", Type: "r5n.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.4}, "r5n.metal": {Region: "eu-west-2", Type: "r5n.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.4}, "r6g.medium": {Region: "eu-west-2", Type: "r6g.medium", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0592}, "r6g.large": {Region: "eu-west-2", Type: "r6g.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.1184}, "r6g.xlarge": {Region: "eu-west-2", Type: "r6g.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.2368}, "r6g.2xlarge": {Region: "eu-west-2", Type: "r6g.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.4736}, "r6g.4xlarge": {Region: "eu-west-2", Type: "r6g.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.9472}, "r6g.8xlarge": {Region: "eu-west-2", Type: "r6g.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.8944}, "r6g.12xlarge": {Region: "eu-west-2", Type: "r6g.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.8416}, "r6g.16xlarge": {Region: "eu-west-2", Type: "r6g.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.7888}, "r6g.metal": {Region: "eu-west-2", Type: "r6g.metal", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.7888}, "r6i.large": {Region: "eu-west-2", Type: "r6i.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.148}, "r6i.xlarge": {Region: "eu-west-2", Type: "r6i.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.296}, "r6i.2xlarge": {Region: "eu-west-2", Type: "r6i.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.592}, "r6i.4xlarge": {Region: "eu-west-2", Type: "r6i.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.184}, "r6i.8xlarge": {Region: "eu-west-2", Type: "r6i.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.368}, "r6i.12xlarge": {Region: "eu-west-2", Type: "r6i.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.552}, "r6i.16xlarge": {Region: "eu-west-2", Type: "r6i.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.736}, "r6i.24xlarge": {Region: "eu-west-2", Type: "r6i.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.104}, "r6i.32xlarge": {Region: "eu-west-2", Type: "r6i.32xlarge", Memory: kresource.MustParse("1048576Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 9.472}, "r6i.metal": {Region: "eu-west-2", Type: "r6i.metal", Memory: kresource.MustParse("1048576Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 9.472}, "t2.nano": {Region: "eu-west-2", Type: "t2.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0066}, "t2.micro": {Region: "eu-west-2", Type: "t2.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0132}, "t2.small": {Region: "eu-west-2", Type: "t2.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.026}, "t2.medium": {Region: "eu-west-2", Type: "t2.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.052}, "t2.large": {Region: "eu-west-2", Type: "t2.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.1056}, "t2.xlarge": {Region: "eu-west-2", Type: "t2.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.2112}, "t2.2xlarge": {Region: "eu-west-2", Type: "t2.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.4224}, "t3.nano": {Region: "eu-west-2", Type: "t3.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0059}, "t3.micro": {Region: "eu-west-2", Type: "t3.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0118}, "t3.small": {Region: "eu-west-2", Type: "t3.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0236}, "t3.medium": {Region: "eu-west-2", Type: "t3.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0472}, "t3.large": {Region: "eu-west-2", Type: "t3.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0944}, "t3.xlarge": {Region: "eu-west-2", Type: "t3.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1888}, "t3.2xlarge": {Region: "eu-west-2", Type: "t3.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3776}, "t3a.nano": {Region: "eu-west-2", Type: "t3a.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0053}, "t3a.micro": {Region: "eu-west-2", Type: "t3a.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0106}, "t3a.small": {Region: "eu-west-2", Type: "t3a.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0212}, "t3a.medium": {Region: "eu-west-2", Type: "t3a.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0425}, "t3a.large": {Region: "eu-west-2", Type: "t3a.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.085}, "t3a.xlarge": {Region: "eu-west-2", Type: "t3a.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1699}, "t3a.2xlarge": {Region: "eu-west-2", Type: "t3a.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3398}, "t4g.nano": {Region: "eu-west-2", Type: "t4g.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0047}, "t4g.micro": {Region: "eu-west-2", Type: "t4g.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0094}, "t4g.small": {Region: "eu-west-2", Type: "t4g.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0188}, "t4g.medium": {Region: "eu-west-2", Type: "t4g.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0376}, "t4g.large": {Region: "eu-west-2", Type: "t4g.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0752}, "t4g.xlarge": {Region: "eu-west-2", Type: "t4g.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1504}, "t4g.2xlarge": {Region: "eu-west-2", Type: "t4g.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3008}, "x1.16xlarge": {Region: "eu-west-2", Type: "x1.16xlarge", Memory: kresource.MustParse("999424Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 8.403}, "x1.32xlarge": {Region: "eu-west-2", Type: "x1.32xlarge", Memory: kresource.MustParse("1998848Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 16.806}, "z1d.large": {Region: "eu-west-2", Type: "z1d.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.22}, "z1d.xlarge": {Region: "eu-west-2", Type: "z1d.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.439}, "z1d.2xlarge": {Region: "eu-west-2", Type: "z1d.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.879}, "z1d.3xlarge": {Region: "eu-west-2", Type: "z1d.3xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("12"), GPU: 0, Inf: 0, Price: 1.318}, "z1d.6xlarge": {Region: "eu-west-2", Type: "z1d.6xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 0, Price: 2.636}, "z1d.12xlarge": {Region: "eu-west-2", Type: "z1d.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 5.273}, "z1d.metal": {Region: "eu-west-2", Type: "z1d.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 5.273}, }, "eu-west-3": { "c5.large": {Region: "eu-west-3", Type: "c5.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.101}, "c5.xlarge": {Region: "eu-west-3", Type: "c5.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.202}, "c5.2xlarge": {Region: "eu-west-3", Type: "c5.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.404}, "c5.4xlarge": {Region: "eu-west-3", Type: "c5.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.808}, "c5.9xlarge": {Region: "eu-west-3", Type: "c5.9xlarge", Memory: kresource.MustParse("73728Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 1.818}, "c5.12xlarge": {Region: "eu-west-3", Type: "c5.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.424}, "c5.18xlarge": {Region: "eu-west-3", Type: "c5.18xlarge", Memory: kresource.MustParse("147456Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 3.636}, "c5.24xlarge": {Region: "eu-west-3", Type: "c5.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.848}, "c5.metal": {Region: "eu-west-3", Type: "c5.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.848}, "c5a.large": {Region: "eu-west-3", Type: "c5a.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.091}, "c5a.xlarge": {Region: "eu-west-3", Type: "c5a.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.182}, "c5a.2xlarge": {Region: "eu-west-3", Type: "c5a.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.364}, "c5a.4xlarge": {Region: "eu-west-3", Type: "c5a.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.728}, "c5a.8xlarge": {Region: "eu-west-3", Type: "c5a.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.456}, "c5a.12xlarge": {Region: "eu-west-3", Type: "c5a.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.184}, "c5a.16xlarge": {Region: "eu-west-3", Type: "c5a.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.912}, "c5a.24xlarge": {Region: "eu-west-3", Type: "c5a.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.368}, "c5d.large": {Region: "eu-west-3", Type: "c5d.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.115}, "c5d.xlarge": {Region: "eu-west-3", Type: "c5d.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.23}, "c5d.2xlarge": {Region: "eu-west-3", Type: "c5d.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.46}, "c5d.4xlarge": {Region: "eu-west-3", Type: "c5d.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.92}, "c5d.9xlarge": {Region: "eu-west-3", Type: "c5d.9xlarge", Memory: kresource.MustParse("73728Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 2.07}, "c5d.18xlarge": {Region: "eu-west-3", Type: "c5d.18xlarge", Memory: kresource.MustParse("147456Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 4.14}, "c5n.large": {Region: "eu-west-3", Type: "c5n.large", Memory: kresource.MustParse("5376Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.128}, "c5n.xlarge": {Region: "eu-west-3", Type: "c5n.xlarge", Memory: kresource.MustParse("10752Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.256}, "c5n.2xlarge": {Region: "eu-west-3", Type: "c5n.2xlarge", Memory: kresource.MustParse("21504Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.512}, "c5n.4xlarge": {Region: "eu-west-3", Type: "c5n.4xlarge", Memory: kresource.MustParse("43008Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.024}, "c5n.9xlarge": {Region: "eu-west-3", Type: "c5n.9xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 2.304}, "c5n.18xlarge": {Region: "eu-west-3", Type: "c5n.18xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 4.608}, "c5n.metal": {Region: "eu-west-3", Type: "c5n.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 4.608}, "c6g.medium": {Region: "eu-west-3", Type: "c6g.medium", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0405}, "c6g.large": {Region: "eu-west-3", Type: "c6g.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.081}, "c6g.xlarge": {Region: "eu-west-3", Type: "c6g.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.162}, "c6g.2xlarge": {Region: "eu-west-3", Type: "c6g.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.324}, "c6g.4xlarge": {Region: "eu-west-3", Type: "c6g.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.648}, "c6g.8xlarge": {Region: "eu-west-3", Type: "c6g.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.296}, "c6g.12xlarge": {Region: "eu-west-3", Type: "c6g.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 1.944}, "c6g.16xlarge": {Region: "eu-west-3", Type: "c6g.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.592}, "c6g.metal": {Region: "eu-west-3", Type: "c6g.metal", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.592}, "c6i.large": {Region: "eu-west-3", Type: "c6i.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.101}, "c6i.xlarge": {Region: "eu-west-3", Type: "c6i.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.202}, "c6i.2xlarge": {Region: "eu-west-3", Type: "c6i.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.404}, "c6i.4xlarge": {Region: "eu-west-3", Type: "c6i.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.808}, "c6i.8xlarge": {Region: "eu-west-3", Type: "c6i.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.616}, "c6i.12xlarge": {Region: "eu-west-3", Type: "c6i.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.424}, "c6i.16xlarge": {Region: "eu-west-3", Type: "c6i.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.232}, "c6i.24xlarge": {Region: "eu-west-3", Type: "c6i.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.848}, "c6i.32xlarge": {Region: "eu-west-3", Type: "c6i.32xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 6.464}, "c6i.metal": {Region: "eu-west-3", Type: "c6i.metal", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 6.464}, "d2.xlarge": {Region: "eu-west-3", Type: "d2.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.772}, "d2.2xlarge": {Region: "eu-west-3", Type: "d2.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.544}, "d2.4xlarge": {Region: "eu-west-3", Type: "d2.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 3.088}, "d2.8xlarge": {Region: "eu-west-3", Type: "d2.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 6.176}, "g4dn.xlarge": {Region: "eu-west-3", Type: "g4dn.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 1, Inf: 0, Price: 0.615}, "g4dn.2xlarge": {Region: "eu-west-3", Type: "g4dn.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 1, Inf: 0, Price: 0.879}, "g4dn.4xlarge": {Region: "eu-west-3", Type: "g4dn.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 1, Inf: 0, Price: 1.408}, "g4dn.8xlarge": {Region: "eu-west-3", Type: "g4dn.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 1, Inf: 0, Price: 2.544}, "g4dn.12xlarge": {Region: "eu-west-3", Type: "g4dn.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 4, Inf: 0, Price: 4.574}, "g4dn.16xlarge": {Region: "eu-west-3", Type: "g4dn.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 1, Inf: 0, Price: 5.088}, "g4dn.metal": {Region: "eu-west-3", Type: "g4dn.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 8, Inf: 0, Price: 9.148}, "i3.large": {Region: "eu-west-3", Type: "i3.large", Memory: kresource.MustParse("15616Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.181}, "i3.xlarge": {Region: "eu-west-3", Type: "i3.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.362}, "i3.2xlarge": {Region: "eu-west-3", Type: "i3.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.724}, "i3.4xlarge": {Region: "eu-west-3", Type: "i3.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.448}, "i3.8xlarge": {Region: "eu-west-3", Type: "i3.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.896}, "i3.16xlarge": {Region: "eu-west-3", Type: "i3.16xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.792}, "i3.metal": {Region: "eu-west-3", Type: "i3.metal", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.792}, "i3en.large": {Region: "eu-west-3", Type: "i3en.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.263}, "i3en.xlarge": {Region: "eu-west-3", Type: "i3en.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.526}, "i3en.2xlarge": {Region: "eu-west-3", Type: "i3en.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.052}, "i3en.3xlarge": {Region: "eu-west-3", Type: "i3en.3xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("12"), GPU: 0, Inf: 0, Price: 1.578}, "i3en.6xlarge": {Region: "eu-west-3", Type: "i3en.6xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 0, Price: 3.156}, "i3en.12xlarge": {Region: "eu-west-3", Type: "i3en.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 6.312}, "i3en.24xlarge": {Region: "eu-west-3", Type: "i3en.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 12.624}, "i3en.metal": {Region: "eu-west-3", Type: "i3en.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 12.624}, "inf1.xlarge": {Region: "eu-west-3", Type: "inf1.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 1, Price: 0.267}, "inf1.2xlarge": {Region: "eu-west-3", Type: "inf1.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 1, Price: 0.423}, "inf1.6xlarge": {Region: "eu-west-3", Type: "inf1.6xlarge", Memory: kresource.MustParse("49152Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 4, Price: 1.379}, "inf1.24xlarge": {Region: "eu-west-3", Type: "inf1.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 16, Price: 5.517}, "m5.large": {Region: "eu-west-3", Type: "m5.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.112}, "m5.xlarge": {Region: "eu-west-3", Type: "m5.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.224}, "m5.2xlarge": {Region: "eu-west-3", Type: "m5.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.448}, "m5.4xlarge": {Region: "eu-west-3", Type: "m5.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.896}, "m5.8xlarge": {Region: "eu-west-3", Type: "m5.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.792}, "m5.12xlarge": {Region: "eu-west-3", Type: "m5.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.688}, "m5.16xlarge": {Region: "eu-west-3", Type: "m5.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.584}, "m5.24xlarge": {Region: "eu-west-3", Type: "m5.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.376}, "m5.metal": {Region: "eu-west-3", Type: "m5.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.376}, "m5a.large": {Region: "eu-west-3", Type: "m5a.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.101}, "m5a.xlarge": {Region: "eu-west-3", Type: "m5a.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.202}, "m5a.2xlarge": {Region: "eu-west-3", Type: "m5a.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.404}, "m5a.4xlarge": {Region: "eu-west-3", Type: "m5a.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.808}, "m5a.8xlarge": {Region: "eu-west-3", Type: "m5a.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.616}, "m5a.12xlarge": {Region: "eu-west-3", Type: "m5a.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.424}, "m5a.16xlarge": {Region: "eu-west-3", Type: "m5a.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.232}, "m5a.24xlarge": {Region: "eu-west-3", Type: "m5a.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.848}, "m5ad.large": {Region: "eu-west-3", Type: "m5ad.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.121}, "m5ad.xlarge": {Region: "eu-west-3", Type: "m5ad.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.242}, "m5ad.2xlarge": {Region: "eu-west-3", Type: "m5ad.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.484}, "m5ad.4xlarge": {Region: "eu-west-3", Type: "m5ad.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.968}, "m5ad.8xlarge": {Region: "eu-west-3", Type: "m5ad.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.936}, "m5ad.12xlarge": {Region: "eu-west-3", Type: "m5ad.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.904}, "m5ad.16xlarge": {Region: "eu-west-3", Type: "m5ad.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.872}, "m5ad.24xlarge": {Region: "eu-west-3", Type: "m5ad.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.808}, "m5d.large": {Region: "eu-west-3", Type: "m5d.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.132}, "m5d.xlarge": {Region: "eu-west-3", Type: "m5d.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.264}, "m5d.2xlarge": {Region: "eu-west-3", Type: "m5d.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.528}, "m5d.4xlarge": {Region: "eu-west-3", Type: "m5d.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.056}, "m5d.8xlarge": {Region: "eu-west-3", Type: "m5d.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.112}, "m5d.12xlarge": {Region: "eu-west-3", Type: "m5d.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.168}, "m5d.16xlarge": {Region: "eu-west-3", Type: "m5d.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.224}, "m5d.24xlarge": {Region: "eu-west-3", Type: "m5d.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.336}, "m5d.metal": {Region: "eu-west-3", Type: "m5d.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.336}, "m6g.medium": {Region: "eu-west-3", Type: "m6g.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.045}, "m6g.large": {Region: "eu-west-3", Type: "m6g.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.09}, "m6g.xlarge": {Region: "eu-west-3", Type: "m6g.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.18}, "m6g.2xlarge": {Region: "eu-west-3", Type: "m6g.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.36}, "m6g.4xlarge": {Region: "eu-west-3", Type: "m6g.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.72}, "m6g.8xlarge": {Region: "eu-west-3", Type: "m6g.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.44}, "m6g.12xlarge": {Region: "eu-west-3", Type: "m6g.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.16}, "m6g.16xlarge": {Region: "eu-west-3", Type: "m6g.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.88}, "m6g.metal": {Region: "eu-west-3", Type: "m6g.metal", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.88}, "m6i.large": {Region: "eu-west-3", Type: "m6i.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.112}, "m6i.xlarge": {Region: "eu-west-3", Type: "m6i.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.224}, "m6i.2xlarge": {Region: "eu-west-3", Type: "m6i.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.448}, "m6i.4xlarge": {Region: "eu-west-3", Type: "m6i.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.896}, "m6i.8xlarge": {Region: "eu-west-3", Type: "m6i.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.792}, "m6i.12xlarge": {Region: "eu-west-3", Type: "m6i.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.688}, "m6i.16xlarge": {Region: "eu-west-3", Type: "m6i.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.584}, "m6i.24xlarge": {Region: "eu-west-3", Type: "m6i.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.376}, "m6i.32xlarge": {Region: "eu-west-3", Type: "m6i.32xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 7.168}, "m6i.metal": {Region: "eu-west-3", Type: "m6i.metal", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 7.168}, "r4.large": {Region: "eu-west-3", Type: "r4.large", Memory: kresource.MustParse("15616Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.156}, "r4.xlarge": {Region: "eu-west-3", Type: "r4.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.312}, "r4.2xlarge": {Region: "eu-west-3", Type: "r4.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.624}, "r4.4xlarge": {Region: "eu-west-3", Type: "r4.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.248}, "r4.8xlarge": {Region: "eu-west-3", Type: "r4.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.496}, "r4.16xlarge": {Region: "eu-west-3", Type: "r4.16xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.992}, "r5.large": {Region: "eu-west-3", Type: "r5.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.148}, "r5.xlarge": {Region: "eu-west-3", Type: "r5.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.296}, "r5.2xlarge": {Region: "eu-west-3", Type: "r5.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.592}, "r5.4xlarge": {Region: "eu-west-3", Type: "r5.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.184}, "r5.8xlarge": {Region: "eu-west-3", Type: "r5.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.368}, "r5.12xlarge": {Region: "eu-west-3", Type: "r5.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.552}, "r5.16xlarge": {Region: "eu-west-3", Type: "r5.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.736}, "r5.24xlarge": {Region: "eu-west-3", Type: "r5.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.104}, "r5.metal": {Region: "eu-west-3", Type: "r5.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.104}, "r5a.large": {Region: "eu-west-3", Type: "r5a.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.133}, "r5a.xlarge": {Region: "eu-west-3", Type: "r5a.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.266}, "r5a.2xlarge": {Region: "eu-west-3", Type: "r5a.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.532}, "r5a.4xlarge": {Region: "eu-west-3", Type: "r5a.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.064}, "r5a.8xlarge": {Region: "eu-west-3", Type: "r5a.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.128}, "r5a.12xlarge": {Region: "eu-west-3", Type: "r5a.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.192}, "r5a.16xlarge": {Region: "eu-west-3", Type: "r5a.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.256}, "r5a.24xlarge": {Region: "eu-west-3", Type: "r5a.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.384}, "r5ad.large": {Region: "eu-west-3", Type: "r5ad.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.153}, "r5ad.xlarge": {Region: "eu-west-3", Type: "r5ad.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.306}, "r5ad.2xlarge": {Region: "eu-west-3", Type: "r5ad.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.612}, "r5ad.4xlarge": {Region: "eu-west-3", Type: "r5ad.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.224}, "r5ad.8xlarge": {Region: "eu-west-3", Type: "r5ad.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.448}, "r5ad.12xlarge": {Region: "eu-west-3", Type: "r5ad.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.672}, "r5ad.16xlarge": {Region: "eu-west-3", Type: "r5ad.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.896}, "r5ad.24xlarge": {Region: "eu-west-3", Type: "r5ad.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.344}, "r5d.large": {Region: "eu-west-3", Type: "r5d.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.169}, "r5d.xlarge": {Region: "eu-west-3", Type: "r5d.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.338}, "r5d.2xlarge": {Region: "eu-west-3", Type: "r5d.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.676}, "r5d.4xlarge": {Region: "eu-west-3", Type: "r5d.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.352}, "r5d.8xlarge": {Region: "eu-west-3", Type: "r5d.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.704}, "r5d.12xlarge": {Region: "eu-west-3", Type: "r5d.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.056}, "r5d.16xlarge": {Region: "eu-west-3", Type: "r5d.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.408}, "r5d.24xlarge": {Region: "eu-west-3", Type: "r5d.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.112}, "r5d.metal": {Region: "eu-west-3", Type: "r5d.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.112}, "r5dn.large": {Region: "eu-west-3", Type: "r5dn.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.196}, "r5dn.xlarge": {Region: "eu-west-3", Type: "r5dn.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.392}, "r5dn.2xlarge": {Region: "eu-west-3", Type: "r5dn.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.784}, "r5dn.4xlarge": {Region: "eu-west-3", Type: "r5dn.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.568}, "r5dn.8xlarge": {Region: "eu-west-3", Type: "r5dn.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 3.136}, "r5dn.12xlarge": {Region: "eu-west-3", Type: "r5dn.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.704}, "r5dn.16xlarge": {Region: "eu-west-3", Type: "r5dn.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 6.272}, "r5dn.24xlarge": {Region: "eu-west-3", Type: "r5dn.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 9.408}, "r5dn.metal": {Region: "eu-west-3", Type: "r5dn.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 9.408}, "r5n.large": {Region: "eu-west-3", Type: "r5n.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.175}, "r5n.xlarge": {Region: "eu-west-3", Type: "r5n.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.35}, "r5n.2xlarge": {Region: "eu-west-3", Type: "r5n.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.7}, "r5n.4xlarge": {Region: "eu-west-3", Type: "r5n.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.4}, "r5n.8xlarge": {Region: "eu-west-3", Type: "r5n.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.8}, "r5n.12xlarge": {Region: "eu-west-3", Type: "r5n.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.2}, "r5n.16xlarge": {Region: "eu-west-3", Type: "r5n.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.6}, "r5n.24xlarge": {Region: "eu-west-3", Type: "r5n.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.4}, "r5n.metal": {Region: "eu-west-3", Type: "r5n.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.4}, "r6g.medium": {Region: "eu-west-3", Type: "r6g.medium", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.059}, "r6g.large": {Region: "eu-west-3", Type: "r6g.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.118}, "r6g.xlarge": {Region: "eu-west-3", Type: "r6g.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.236}, "r6g.2xlarge": {Region: "eu-west-3", Type: "r6g.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.472}, "r6g.4xlarge": {Region: "eu-west-3", Type: "r6g.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.944}, "r6g.8xlarge": {Region: "eu-west-3", Type: "r6g.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.888}, "r6g.12xlarge": {Region: "eu-west-3", Type: "r6g.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.832}, "r6g.16xlarge": {Region: "eu-west-3", Type: "r6g.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.776}, "r6g.metal": {Region: "eu-west-3", Type: "r6g.metal", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.776}, "r6gd.medium": {Region: "eu-west-3", Type: "r6gd.medium", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0676}, "r6gd.large": {Region: "eu-west-3", Type: "r6gd.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.1352}, "r6gd.xlarge": {Region: "eu-west-3", Type: "r6gd.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.2704}, "r6gd.2xlarge": {Region: "eu-west-3", Type: "r6gd.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.5408}, "r6gd.4xlarge": {Region: "eu-west-3", Type: "r6gd.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.0816}, "r6gd.8xlarge": {Region: "eu-west-3", Type: "r6gd.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.1632}, "r6gd.12xlarge": {Region: "eu-west-3", Type: "r6gd.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.2448}, "r6gd.16xlarge": {Region: "eu-west-3", Type: "r6gd.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.3264}, "r6gd.metal": {Region: "eu-west-3", Type: "r6gd.metal", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.3264}, "r6i.large": {Region: "eu-west-3", Type: "r6i.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.148}, "r6i.xlarge": {Region: "eu-west-3", Type: "r6i.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.296}, "r6i.2xlarge": {Region: "eu-west-3", Type: "r6i.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.592}, "r6i.4xlarge": {Region: "eu-west-3", Type: "r6i.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.184}, "r6i.8xlarge": {Region: "eu-west-3", Type: "r6i.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.368}, "r6i.12xlarge": {Region: "eu-west-3", Type: "r6i.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.552}, "r6i.16xlarge": {Region: "eu-west-3", Type: "r6i.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.736}, "r6i.24xlarge": {Region: "eu-west-3", Type: "r6i.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.104}, "r6i.32xlarge": {Region: "eu-west-3", Type: "r6i.32xlarge", Memory: kresource.MustParse("1048576Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 9.472}, "r6i.metal": {Region: "eu-west-3", Type: "r6i.metal", Memory: kresource.MustParse("1048576Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 9.472}, "t2.nano": {Region: "eu-west-3", Type: "t2.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0066}, "t2.micro": {Region: "eu-west-3", Type: "t2.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0132}, "t2.small": {Region: "eu-west-3", Type: "t2.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0264}, "t2.medium": {Region: "eu-west-3", Type: "t2.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0528}, "t2.large": {Region: "eu-west-3", Type: "t2.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.1056}, "t2.xlarge": {Region: "eu-west-3", Type: "t2.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.2112}, "t2.2xlarge": {Region: "eu-west-3", Type: "t2.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.4224}, "t3.nano": {Region: "eu-west-3", Type: "t3.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0059}, "t3.micro": {Region: "eu-west-3", Type: "t3.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0118}, "t3.small": {Region: "eu-west-3", Type: "t3.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0236}, "t3.medium": {Region: "eu-west-3", Type: "t3.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0472}, "t3.large": {Region: "eu-west-3", Type: "t3.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0944}, "t3.xlarge": {Region: "eu-west-3", Type: "t3.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1888}, "t3.2xlarge": {Region: "eu-west-3", Type: "t3.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3776}, "t3a.nano": {Region: "eu-west-3", Type: "t3a.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0053}, "t3a.micro": {Region: "eu-west-3", Type: "t3a.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0106}, "t3a.small": {Region: "eu-west-3", Type: "t3a.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0212}, "t3a.medium": {Region: "eu-west-3", Type: "t3a.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0425}, "t3a.large": {Region: "eu-west-3", Type: "t3a.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.085}, "t3a.xlarge": {Region: "eu-west-3", Type: "t3a.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1699}, "t3a.2xlarge": {Region: "eu-west-3", Type: "t3a.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3398}, "t4g.nano": {Region: "eu-west-3", Type: "t4g.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0047}, "t4g.micro": {Region: "eu-west-3", Type: "t4g.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0094}, "t4g.small": {Region: "eu-west-3", Type: "t4g.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0188}, "t4g.medium": {Region: "eu-west-3", Type: "t4g.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0376}, "t4g.large": {Region: "eu-west-3", Type: "t4g.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0752}, "t4g.xlarge": {Region: "eu-west-3", Type: "t4g.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1504}, "t4g.2xlarge": {Region: "eu-west-3", Type: "t4g.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3008}, "u-6tb1.56xlarge": {Region: "eu-west-3", Type: "u-6tb1.56xlarge", Memory: kresource.MustParse("6291456Mi"), CPU: kresource.MustParse("224"), GPU: 0, Inf: 0, Price: 54.50589}, "u-6tb1.112xlarge": {Region: "eu-west-3", Type: "u-6tb1.112xlarge", Memory: kresource.MustParse("6291456Mi"), CPU: kresource.MustParse("448"), GPU: 0, Inf: 0, Price: 64.133}, "x1.16xlarge": {Region: "eu-west-3", Type: "x1.16xlarge", Memory: kresource.MustParse("999424Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 8.403}, "x1.32xlarge": {Region: "eu-west-3", Type: "x1.32xlarge", Memory: kresource.MustParse("1998848Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 16.806}, }, "me-south-1": { "c5.large": {Region: "me-south-1", Type: "c5.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.106}, "c5.xlarge": {Region: "me-south-1", Type: "c5.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.211}, "c5.2xlarge": {Region: "me-south-1", Type: "c5.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.422}, "c5.4xlarge": {Region: "me-south-1", Type: "c5.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.845}, "c5.9xlarge": {Region: "me-south-1", Type: "c5.9xlarge", Memory: kresource.MustParse("73728Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 1.901}, "c5.12xlarge": {Region: "me-south-1", Type: "c5.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.534}, "c5.18xlarge": {Region: "me-south-1", Type: "c5.18xlarge", Memory: kresource.MustParse("147456Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 3.802}, "c5.24xlarge": {Region: "me-south-1", Type: "c5.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.069}, "c5.metal": {Region: "me-south-1", Type: "c5.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.069}, "c5a.large": {Region: "me-south-1", Type: "c5a.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.095}, "c5a.xlarge": {Region: "me-south-1", Type: "c5a.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.19}, "c5a.2xlarge": {Region: "me-south-1", Type: "c5a.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.38}, "c5a.4xlarge": {Region: "me-south-1", Type: "c5a.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.76}, "c5a.8xlarge": {Region: "me-south-1", Type: "c5a.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.52}, "c5a.12xlarge": {Region: "me-south-1", Type: "c5a.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.28}, "c5a.16xlarge": {Region: "me-south-1", Type: "c5a.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.04}, "c5a.24xlarge": {Region: "me-south-1", Type: "c5a.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.56}, "c5ad.large": {Region: "me-south-1", Type: "c5ad.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.108}, "c5ad.xlarge": {Region: "me-south-1", Type: "c5ad.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.216}, "c5ad.2xlarge": {Region: "me-south-1", Type: "c5ad.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.432}, "c5ad.4xlarge": {Region: "me-south-1", Type: "c5ad.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.864}, "c5ad.8xlarge": {Region: "me-south-1", Type: "c5ad.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.728}, "c5ad.12xlarge": {Region: "me-south-1", Type: "c5ad.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.592}, "c5ad.16xlarge": {Region: "me-south-1", Type: "c5ad.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.456}, "c5ad.24xlarge": {Region: "me-south-1", Type: "c5ad.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.184}, "c5d.large": {Region: "me-south-1", Type: "c5d.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.12}, "c5d.xlarge": {Region: "me-south-1", Type: "c5d.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.24}, "c5d.2xlarge": {Region: "me-south-1", Type: "c5d.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.48}, "c5d.4xlarge": {Region: "me-south-1", Type: "c5d.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.959}, "c5d.9xlarge": {Region: "me-south-1", Type: "c5d.9xlarge", Memory: kresource.MustParse("73728Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 2.158}, "c5d.12xlarge": {Region: "me-south-1", Type: "c5d.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.878}, "c5d.18xlarge": {Region: "me-south-1", Type: "c5d.18xlarge", Memory: kresource.MustParse("147456Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 4.316}, "c5d.24xlarge": {Region: "me-south-1", Type: "c5d.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.755}, "c5d.metal": {Region: "me-south-1", Type: "c5d.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.755}, "c5n.large": {Region: "me-south-1", Type: "c5n.large", Memory: kresource.MustParse("5376Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.134}, "c5n.xlarge": {Region: "me-south-1", Type: "c5n.xlarge", Memory: kresource.MustParse("10752Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.268}, "c5n.2xlarge": {Region: "me-south-1", Type: "c5n.2xlarge", Memory: kresource.MustParse("21504Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.536}, "c5n.4xlarge": {Region: "me-south-1", Type: "c5n.4xlarge", Memory: kresource.MustParse("43008Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.072}, "c5n.9xlarge": {Region: "me-south-1", Type: "c5n.9xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 2.412}, "c5n.18xlarge": {Region: "me-south-1", Type: "c5n.18xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 4.824}, "c5n.metal": {Region: "me-south-1", Type: "c5n.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 4.824}, "c6g.medium": {Region: "me-south-1", Type: "c6g.medium", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0425}, "c6g.large": {Region: "me-south-1", Type: "c6g.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.085}, "c6g.xlarge": {Region: "me-south-1", Type: "c6g.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.17}, "c6g.2xlarge": {Region: "me-south-1", Type: "c6g.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.34}, "c6g.4xlarge": {Region: "me-south-1", Type: "c6g.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.68}, "c6g.8xlarge": {Region: "me-south-1", Type: "c6g.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.36}, "c6g.12xlarge": {Region: "me-south-1", Type: "c6g.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.04}, "c6g.16xlarge": {Region: "me-south-1", Type: "c6g.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.72}, "c6g.metal": {Region: "me-south-1", Type: "c6g.metal", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.72}, "d2.xlarge": {Region: "me-south-1", Type: "d2.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.809}, "d2.2xlarge": {Region: "me-south-1", Type: "d2.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.617}, "d2.4xlarge": {Region: "me-south-1", Type: "d2.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 3.234}, "d2.8xlarge": {Region: "me-south-1", Type: "d2.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 6.468}, "g4dn.xlarge": {Region: "me-south-1", Type: "g4dn.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 1, Inf: 0, Price: 0.645}, "g4dn.2xlarge": {Region: "me-south-1", Type: "g4dn.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 1, Inf: 0, Price: 0.922}, "g4dn.4xlarge": {Region: "me-south-1", Type: "g4dn.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 1, Inf: 0, Price: 1.477}, "g4dn.8xlarge": {Region: "me-south-1", Type: "g4dn.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 1, Inf: 0, Price: 2.669}, "g4dn.12xlarge": {Region: "me-south-1", Type: "g4dn.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 4, Inf: 0, Price: 4.798}, "g4dn.16xlarge": {Region: "me-south-1", Type: "g4dn.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 1, Inf: 0, Price: 5.338}, "g4dn.metal": {Region: "me-south-1", Type: "g4dn.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 8, Inf: 0, Price: 9.596}, "i3.large": {Region: "me-south-1", Type: "i3.large", Memory: kresource.MustParse("15616Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.189}, "i3.xlarge": {Region: "me-south-1", Type: "i3.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.378}, "i3.2xlarge": {Region: "me-south-1", Type: "i3.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.757}, "i3.4xlarge": {Region: "me-south-1", Type: "i3.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.514}, "i3.8xlarge": {Region: "me-south-1", Type: "i3.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 3.027}, "i3.16xlarge": {Region: "me-south-1", Type: "i3.16xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 6.054}, "i3en.large": {Region: "me-south-1", Type: "i3en.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.274}, "i3en.xlarge": {Region: "me-south-1", Type: "i3en.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.548}, "i3en.2xlarge": {Region: "me-south-1", Type: "i3en.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.095}, "i3en.3xlarge": {Region: "me-south-1", Type: "i3en.3xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("12"), GPU: 0, Inf: 0, Price: 1.643}, "i3en.6xlarge": {Region: "me-south-1", Type: "i3en.6xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 0, Price: 3.286}, "i3en.12xlarge": {Region: "me-south-1", Type: "i3en.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 6.571}, "i3en.24xlarge": {Region: "me-south-1", Type: "i3en.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 13.142}, "i3en.metal": {Region: "me-south-1", Type: "i3en.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 13.142}, "inf1.xlarge": {Region: "me-south-1", Type: "inf1.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 1, Price: 0.28}, "inf1.2xlarge": {Region: "me-south-1", Type: "inf1.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 1, Price: 0.444}, "inf1.6xlarge": {Region: "me-south-1", Type: "inf1.6xlarge", Memory: kresource.MustParse("49152Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 4, Price: 1.447}, "inf1.24xlarge": {Region: "me-south-1", Type: "inf1.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 16, Price: 5.786}, "m5.large": {Region: "me-south-1", Type: "m5.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.118}, "m5.xlarge": {Region: "me-south-1", Type: "m5.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.235}, "m5.2xlarge": {Region: "me-south-1", Type: "m5.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.471}, "m5.4xlarge": {Region: "me-south-1", Type: "m5.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.942}, "m5.8xlarge": {Region: "me-south-1", Type: "m5.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.883}, "m5.12xlarge": {Region: "me-south-1", Type: "m5.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.825}, "m5.16xlarge": {Region: "me-south-1", Type: "m5.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.766}, "m5.24xlarge": {Region: "me-south-1", Type: "m5.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.65}, "m5.metal": {Region: "me-south-1", Type: "m5.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.65}, "m5d.large": {Region: "me-south-1", Type: "m5d.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.139}, "m5d.xlarge": {Region: "me-south-1", Type: "m5d.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.277}, "m5d.2xlarge": {Region: "me-south-1", Type: "m5d.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.554}, "m5d.4xlarge": {Region: "me-south-1", Type: "m5d.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.109}, "m5d.8xlarge": {Region: "me-south-1", Type: "m5d.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.218}, "m5d.12xlarge": {Region: "me-south-1", Type: "m5d.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.326}, "m5d.16xlarge": {Region: "me-south-1", Type: "m5d.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.435}, "m5d.24xlarge": {Region: "me-south-1", Type: "m5d.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.653}, "m5d.metal": {Region: "me-south-1", Type: "m5d.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.653}, "m6g.medium": {Region: "me-south-1", Type: "m6g.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.047}, "m6g.large": {Region: "me-south-1", Type: "m6g.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.094}, "m6g.xlarge": {Region: "me-south-1", Type: "m6g.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.188}, "m6g.2xlarge": {Region: "me-south-1", Type: "m6g.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.376}, "m6g.4xlarge": {Region: "me-south-1", Type: "m6g.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.752}, "m6g.8xlarge": {Region: "me-south-1", Type: "m6g.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.504}, "m6g.12xlarge": {Region: "me-south-1", Type: "m6g.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.256}, "m6g.16xlarge": {Region: "me-south-1", Type: "m6g.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.008}, "m6g.metal": {Region: "me-south-1", Type: "m6g.metal", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.008}, "r5.large": {Region: "me-south-1", Type: "r5.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.155}, "r5.xlarge": {Region: "me-south-1", Type: "r5.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.31}, "r5.2xlarge": {Region: "me-south-1", Type: "r5.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.62}, "r5.4xlarge": {Region: "me-south-1", Type: "r5.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.241}, "r5.8xlarge": {Region: "me-south-1", Type: "r5.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.482}, "r5.12xlarge": {Region: "me-south-1", Type: "r5.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.722}, "r5.16xlarge": {Region: "me-south-1", Type: "r5.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.963}, "r5.24xlarge": {Region: "me-south-1", Type: "r5.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.445}, "r5.metal": {Region: "me-south-1", Type: "r5.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.445}, "r5d.large": {Region: "me-south-1", Type: "r5d.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.176}, "r5d.xlarge": {Region: "me-south-1", Type: "r5d.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.352}, "r5d.2xlarge": {Region: "me-south-1", Type: "r5d.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.704}, "r5d.4xlarge": {Region: "me-south-1", Type: "r5d.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.408}, "r5d.8xlarge": {Region: "me-south-1", Type: "r5d.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.816}, "r5d.12xlarge": {Region: "me-south-1", Type: "r5d.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.224}, "r5d.16xlarge": {Region: "me-south-1", Type: "r5d.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.632}, "r5d.24xlarge": {Region: "me-south-1", Type: "r5d.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.448}, "r5d.metal": {Region: "me-south-1", Type: "r5d.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.448}, "t3.nano": {Region: "me-south-1", Type: "t3.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0063}, "t3.micro": {Region: "me-south-1", Type: "t3.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0125}, "t3.small": {Region: "me-south-1", Type: "t3.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0251}, "t3.medium": {Region: "me-south-1", Type: "t3.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0502}, "t3.large": {Region: "me-south-1", Type: "t3.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.1003}, "t3.xlarge": {Region: "me-south-1", Type: "t3.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.2006}, "t3.2xlarge": {Region: "me-south-1", Type: "t3.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.4013}, }, "sa-east-1": { "c1.medium": {Region: "sa-east-1", Type: "c1.medium", Memory: kresource.MustParse("1740Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.179}, "c1.xlarge": {Region: "sa-east-1", Type: "c1.xlarge", Memory: kresource.MustParse("7168Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.718}, "c3.large": {Region: "sa-east-1", Type: "c3.large", Memory: kresource.MustParse("3840Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.163}, "c3.xlarge": {Region: "sa-east-1", Type: "c3.xlarge", Memory: kresource.MustParse("7680Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.325}, "c3.2xlarge": {Region: "sa-east-1", Type: "c3.2xlarge", Memory: kresource.MustParse("15360Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.65}, "c3.4xlarge": {Region: "sa-east-1", Type: "c3.4xlarge", Memory: kresource.MustParse("30720Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.3}, "c3.8xlarge": {Region: "sa-east-1", Type: "c3.8xlarge", Memory: kresource.MustParse("61440Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.6}, "c4.large": {Region: "sa-east-1", Type: "c4.large", Memory: kresource.MustParse("3840Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.155}, "c4.xlarge": {Region: "sa-east-1", Type: "c4.xlarge", Memory: kresource.MustParse("7680Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.309}, "c4.2xlarge": {Region: "sa-east-1", Type: "c4.2xlarge", Memory: kresource.MustParse("15360Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.618}, "c4.4xlarge": {Region: "sa-east-1", Type: "c4.4xlarge", Memory: kresource.MustParse("30720Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.235}, "c4.8xlarge": {Region: "sa-east-1", Type: "c4.8xlarge", Memory: kresource.MustParse("61440Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 2.47}, "c5.large": {Region: "sa-east-1", Type: "c5.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.131}, "c5.xlarge": {Region: "sa-east-1", Type: "c5.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.262}, "c5.2xlarge": {Region: "sa-east-1", Type: "c5.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.524}, "c5.4xlarge": {Region: "sa-east-1", Type: "c5.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.048}, "c5.9xlarge": {Region: "sa-east-1", Type: "c5.9xlarge", Memory: kresource.MustParse("73728Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 2.358}, "c5.12xlarge": {Region: "sa-east-1", Type: "c5.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.144}, "c5.18xlarge": {Region: "sa-east-1", Type: "c5.18xlarge", Memory: kresource.MustParse("147456Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 4.716}, "c5.24xlarge": {Region: "sa-east-1", Type: "c5.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.288}, "c5.metal": {Region: "sa-east-1", Type: "c5.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.288}, "c5a.large": {Region: "sa-east-1", Type: "c5a.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.118}, "c5a.xlarge": {Region: "sa-east-1", Type: "c5a.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.236}, "c5a.2xlarge": {Region: "sa-east-1", Type: "c5a.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.472}, "c5a.4xlarge": {Region: "sa-east-1", Type: "c5a.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.944}, "c5a.8xlarge": {Region: "sa-east-1", Type: "c5a.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.888}, "c5a.12xlarge": {Region: "sa-east-1", Type: "c5a.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.832}, "c5a.16xlarge": {Region: "sa-east-1", Type: "c5a.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.776}, "c5a.24xlarge": {Region: "sa-east-1", Type: "c5a.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.664}, "c5ad.large": {Region: "sa-east-1", Type: "c5ad.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.134}, "c5ad.xlarge": {Region: "sa-east-1", Type: "c5ad.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.268}, "c5ad.2xlarge": {Region: "sa-east-1", Type: "c5ad.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.536}, "c5ad.4xlarge": {Region: "sa-east-1", Type: "c5ad.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.072}, "c5ad.8xlarge": {Region: "sa-east-1", Type: "c5ad.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.144}, "c5ad.12xlarge": {Region: "sa-east-1", Type: "c5ad.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.216}, "c5ad.16xlarge": {Region: "sa-east-1", Type: "c5ad.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.288}, "c5ad.24xlarge": {Region: "sa-east-1", Type: "c5ad.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.432}, "c5d.large": {Region: "sa-east-1", Type: "c5d.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.149}, "c5d.xlarge": {Region: "sa-east-1", Type: "c5d.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.298}, "c5d.2xlarge": {Region: "sa-east-1", Type: "c5d.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.596}, "c5d.4xlarge": {Region: "sa-east-1", Type: "c5d.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.192}, "c5d.9xlarge": {Region: "sa-east-1", Type: "c5d.9xlarge", Memory: kresource.MustParse("73728Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 2.682}, "c5d.12xlarge": {Region: "sa-east-1", Type: "c5d.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.576}, "c5d.18xlarge": {Region: "sa-east-1", Type: "c5d.18xlarge", Memory: kresource.MustParse("147456Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 5.364}, "c5d.24xlarge": {Region: "sa-east-1", Type: "c5d.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.152}, "c5d.metal": {Region: "sa-east-1", Type: "c5d.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.152}, "c5n.large": {Region: "sa-east-1", Type: "c5n.large", Memory: kresource.MustParse("5376Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.166}, "c5n.xlarge": {Region: "sa-east-1", Type: "c5n.xlarge", Memory: kresource.MustParse("10752Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.332}, "c5n.2xlarge": {Region: "sa-east-1", Type: "c5n.2xlarge", Memory: kresource.MustParse("21504Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.664}, "c5n.4xlarge": {Region: "sa-east-1", Type: "c5n.4xlarge", Memory: kresource.MustParse("43008Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.328}, "c5n.9xlarge": {Region: "sa-east-1", Type: "c5n.9xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 2.988}, "c5n.18xlarge": {Region: "sa-east-1", Type: "c5n.18xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 5.976}, "c5n.metal": {Region: "sa-east-1", Type: "c5n.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 5.976}, "c6g.medium": {Region: "sa-east-1", Type: "c6g.medium", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0524}, "c6g.large": {Region: "sa-east-1", Type: "c6g.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.1048}, "c6g.xlarge": {Region: "sa-east-1", Type: "c6g.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.2096}, "c6g.2xlarge": {Region: "sa-east-1", Type: "c6g.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.4192}, "c6g.4xlarge": {Region: "sa-east-1", Type: "c6g.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.8384}, "c6g.8xlarge": {Region: "sa-east-1", Type: "c6g.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.6768}, "c6g.12xlarge": {Region: "sa-east-1", Type: "c6g.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.5152}, "c6g.16xlarge": {Region: "sa-east-1", Type: "c6g.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.3536}, "c6g.metal": {Region: "sa-east-1", Type: "c6g.metal", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.3536}, "c6gn.medium": {Region: "sa-east-1", Type: "c6gn.medium", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0665}, "c6gn.large": {Region: "sa-east-1", Type: "c6gn.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.133}, "c6gn.xlarge": {Region: "sa-east-1", Type: "c6gn.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.266}, "c6gn.2xlarge": {Region: "sa-east-1", Type: "c6gn.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.532}, "c6gn.4xlarge": {Region: "sa-east-1", Type: "c6gn.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.064}, "c6gn.8xlarge": {Region: "sa-east-1", Type: "c6gn.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.128}, "c6gn.12xlarge": {Region: "sa-east-1", Type: "c6gn.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.192}, "c6gn.16xlarge": {Region: "sa-east-1", Type: "c6gn.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.256}, "c6i.large": {Region: "sa-east-1", Type: "c6i.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.131}, "c6i.xlarge": {Region: "sa-east-1", Type: "c6i.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.262}, "c6i.2xlarge": {Region: "sa-east-1", Type: "c6i.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.524}, "c6i.4xlarge": {Region: "sa-east-1", Type: "c6i.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.048}, "c6i.8xlarge": {Region: "sa-east-1", Type: "c6i.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.096}, "c6i.12xlarge": {Region: "sa-east-1", Type: "c6i.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.144}, "c6i.16xlarge": {Region: "sa-east-1", Type: "c6i.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.192}, "c6i.24xlarge": {Region: "sa-east-1", Type: "c6i.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.288}, "c6i.32xlarge": {Region: "sa-east-1", Type: "c6i.32xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 8.384}, "c6i.metal": {Region: "sa-east-1", Type: "c6i.metal", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 8.384}, "g4dn.xlarge": {Region: "sa-east-1", Type: "g4dn.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 1, Inf: 0, Price: 0.894}, "g4dn.2xlarge": {Region: "sa-east-1", Type: "g4dn.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 1, Inf: 0, Price: 1.278}, "g4dn.4xlarge": {Region: "sa-east-1", Type: "g4dn.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 1, Inf: 0, Price: 2.046}, "g4dn.8xlarge": {Region: "sa-east-1", Type: "g4dn.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 1, Inf: 0, Price: 3.698}, "g4dn.12xlarge": {Region: "sa-east-1", Type: "g4dn.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 4, Inf: 0, Price: 6.649}, "g4dn.16xlarge": {Region: "sa-east-1", Type: "g4dn.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 1, Inf: 0, Price: 7.397}, "g4dn.metal": {Region: "sa-east-1", Type: "g4dn.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 8, Inf: 0, Price: 13.298}, "i3.large": {Region: "sa-east-1", Type: "i3.large", Memory: kresource.MustParse("15616Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.249}, "i3.xlarge": {Region: "sa-east-1", Type: "i3.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.498}, "i3.2xlarge": {Region: "sa-east-1", Type: "i3.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.996}, "i3.4xlarge": {Region: "sa-east-1", Type: "i3.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.992}, "i3.8xlarge": {Region: "sa-east-1", Type: "i3.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 3.984}, "i3.16xlarge": {Region: "sa-east-1", Type: "i3.16xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 7.968}, "i3.metal": {Region: "sa-east-1", Type: "i3.metal", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 7.968}, "i3en.large": {Region: "sa-east-1", Type: "i3en.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.36}, "i3en.xlarge": {Region: "sa-east-1", Type: "i3en.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.72}, "i3en.2xlarge": {Region: "sa-east-1", Type: "i3en.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.44}, "i3en.3xlarge": {Region: "sa-east-1", Type: "i3en.3xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("12"), GPU: 0, Inf: 0, Price: 2.16}, "i3en.6xlarge": {Region: "sa-east-1", Type: "i3en.6xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 0, Price: 4.32}, "i3en.12xlarge": {Region: "sa-east-1", Type: "i3en.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 8.64}, "i3en.24xlarge": {Region: "sa-east-1", Type: "i3en.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 17.28}, "i3en.metal": {Region: "sa-east-1", Type: "i3en.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 17.28}, "inf1.xlarge": {Region: "sa-east-1", Type: "inf1.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 1, Price: 0.377}, "inf1.2xlarge": {Region: "sa-east-1", Type: "inf1.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 1, Price: 0.598}, "inf1.6xlarge": {Region: "sa-east-1", Type: "inf1.6xlarge", Memory: kresource.MustParse("49152Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 4, Price: 1.95}, "inf1.24xlarge": {Region: "sa-east-1", Type: "inf1.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 16, Price: 7.8}, "m1.small": {Region: "sa-east-1", Type: "m1.small", Memory: kresource.MustParse("1740Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.058}, "m1.medium": {Region: "sa-east-1", Type: "m1.medium", Memory: kresource.MustParse("3840Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.117}, "m1.large": {Region: "sa-east-1", Type: "m1.large", Memory: kresource.MustParse("7680Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.233}, "m1.xlarge": {Region: "sa-east-1", Type: "m1.xlarge", Memory: kresource.MustParse("15360Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.467}, "m2.xlarge": {Region: "sa-east-1", Type: "m2.xlarge", Memory: kresource.MustParse("17510Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.323}, "m2.2xlarge": {Region: "sa-east-1", Type: "m2.2xlarge", Memory: kresource.MustParse("35020Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.645}, "m2.4xlarge": {Region: "sa-east-1", Type: "m2.4xlarge", Memory: kresource.MustParse("70041Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.291}, "m3.medium": {Region: "sa-east-1", Type: "m3.medium", Memory: kresource.MustParse("3840Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.095}, "m3.large": {Region: "sa-east-1", Type: "m3.large", Memory: kresource.MustParse("7680Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.19}, "m3.xlarge": {Region: "sa-east-1", Type: "m3.xlarge", Memory: kresource.MustParse("15360Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.381}, "m3.2xlarge": {Region: "sa-east-1", Type: "m3.2xlarge", Memory: kresource.MustParse("30720Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.761}, "m4.large": {Region: "sa-east-1", Type: "m4.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.159}, "m4.xlarge": {Region: "sa-east-1", Type: "m4.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.318}, "m4.2xlarge": {Region: "sa-east-1", Type: "m4.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.636}, "m4.4xlarge": {Region: "sa-east-1", Type: "m4.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.272}, "m4.10xlarge": {Region: "sa-east-1", Type: "m4.10xlarge", Memory: kresource.MustParse("163840Mi"), CPU: kresource.MustParse("40"), GPU: 0, Inf: 0, Price: 3.18}, "m4.16xlarge": {Region: "sa-east-1", Type: "m4.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.088}, "m5.large": {Region: "sa-east-1", Type: "m5.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.153}, "m5.xlarge": {Region: "sa-east-1", Type: "m5.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.306}, "m5.2xlarge": {Region: "sa-east-1", Type: "m5.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.612}, "m5.4xlarge": {Region: "sa-east-1", Type: "m5.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.224}, "m5.8xlarge": {Region: "sa-east-1", Type: "m5.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.448}, "m5.12xlarge": {Region: "sa-east-1", Type: "m5.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.672}, "m5.16xlarge": {Region: "sa-east-1", Type: "m5.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.896}, "m5.24xlarge": {Region: "sa-east-1", Type: "m5.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.344}, "m5.metal": {Region: "sa-east-1", Type: "m5.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.344}, "m5a.large": {Region: "sa-east-1", Type: "m5a.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.138}, "m5a.xlarge": {Region: "sa-east-1", Type: "m5a.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.276}, "m5a.2xlarge": {Region: "sa-east-1", Type: "m5a.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.552}, "m5a.4xlarge": {Region: "sa-east-1", Type: "m5a.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.104}, "m5a.8xlarge": {Region: "sa-east-1", Type: "m5a.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.208}, "m5a.12xlarge": {Region: "sa-east-1", Type: "m5a.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.312}, "m5a.16xlarge": {Region: "sa-east-1", Type: "m5a.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.416}, "m5a.24xlarge": {Region: "sa-east-1", Type: "m5a.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.624}, "m5ad.large": {Region: "sa-east-1", Type: "m5ad.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.165}, "m5ad.xlarge": {Region: "sa-east-1", Type: "m5ad.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.33}, "m5ad.2xlarge": {Region: "sa-east-1", Type: "m5ad.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.66}, "m5ad.4xlarge": {Region: "sa-east-1", Type: "m5ad.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.32}, "m5ad.8xlarge": {Region: "sa-east-1", Type: "m5ad.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.64}, "m5ad.12xlarge": {Region: "sa-east-1", Type: "m5ad.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.96}, "m5ad.16xlarge": {Region: "sa-east-1", Type: "m5ad.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.28}, "m5ad.24xlarge": {Region: "sa-east-1", Type: "m5ad.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.92}, "m5d.large": {Region: "sa-east-1", Type: "m5d.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.18}, "m5d.xlarge": {Region: "sa-east-1", Type: "m5d.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.36}, "m5d.2xlarge": {Region: "sa-east-1", Type: "m5d.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.72}, "m5d.4xlarge": {Region: "sa-east-1", Type: "m5d.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.44}, "m5d.8xlarge": {Region: "sa-east-1", Type: "m5d.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.88}, "m5d.12xlarge": {Region: "sa-east-1", Type: "m5d.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.32}, "m5d.16xlarge": {Region: "sa-east-1", Type: "m5d.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.76}, "m5d.24xlarge": {Region: "sa-east-1", Type: "m5d.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.64}, "m5d.metal": {Region: "sa-east-1", Type: "m5d.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.64}, "m5zn.large": {Region: "sa-east-1", Type: "m5zn.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.2632}, "m5zn.xlarge": {Region: "sa-east-1", Type: "m5zn.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.5265}, "m5zn.2xlarge": {Region: "sa-east-1", Type: "m5zn.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.053}, "m5zn.3xlarge": {Region: "sa-east-1", Type: "m5zn.3xlarge", Memory: kresource.MustParse("49152Mi"), CPU: kresource.MustParse("12"), GPU: 0, Inf: 0, Price: 1.5794}, "m5zn.6xlarge": {Region: "sa-east-1", Type: "m5zn.6xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 0, Price: 3.1589}, "m5zn.12xlarge": {Region: "sa-east-1", Type: "m5zn.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 6.3178}, "m5zn.metal": {Region: "sa-east-1", Type: "m5zn.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 6.3178}, "m6g.medium": {Region: "sa-east-1", Type: "m6g.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0612}, "m6g.large": {Region: "sa-east-1", Type: "m6g.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.1224}, "m6g.xlarge": {Region: "sa-east-1", Type: "m6g.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.2448}, "m6g.2xlarge": {Region: "sa-east-1", Type: "m6g.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.4896}, "m6g.4xlarge": {Region: "sa-east-1", Type: "m6g.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.9792}, "m6g.8xlarge": {Region: "sa-east-1", Type: "m6g.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.9584}, "m6g.12xlarge": {Region: "sa-east-1", Type: "m6g.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.9376}, "m6g.16xlarge": {Region: "sa-east-1", Type: "m6g.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.9168}, "m6g.metal": {Region: "sa-east-1", Type: "m6g.metal", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.9168}, "m6i.large": {Region: "sa-east-1", Type: "m6i.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.153}, "m6i.xlarge": {Region: "sa-east-1", Type: "m6i.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.306}, "m6i.2xlarge": {Region: "sa-east-1", Type: "m6i.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.612}, "m6i.4xlarge": {Region: "sa-east-1", Type: "m6i.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.224}, "m6i.8xlarge": {Region: "sa-east-1", Type: "m6i.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.448}, "m6i.12xlarge": {Region: "sa-east-1", Type: "m6i.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.672}, "m6i.16xlarge": {Region: "sa-east-1", Type: "m6i.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.896}, "m6i.24xlarge": {Region: "sa-east-1", Type: "m6i.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.344}, "m6i.32xlarge": {Region: "sa-east-1", Type: "m6i.32xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 9.792}, "m6i.metal": {Region: "sa-east-1", Type: "m6i.metal", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 9.792}, "r3.large": {Region: "sa-east-1", Type: "r3.large", Memory: kresource.MustParse("15616Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.35}, "r3.xlarge": {Region: "sa-east-1", Type: "r3.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.7}, "r3.2xlarge": {Region: "sa-east-1", Type: "r3.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.399}, "r3.4xlarge": {Region: "sa-east-1", Type: "r3.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 2.799}, "r3.8xlarge": {Region: "sa-east-1", Type: "r3.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 5.597}, "r4.large": {Region: "sa-east-1", Type: "r4.large", Memory: kresource.MustParse("15616Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.28}, "r4.xlarge": {Region: "sa-east-1", Type: "r4.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.56}, "r4.2xlarge": {Region: "sa-east-1", Type: "r4.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.12}, "r4.4xlarge": {Region: "sa-east-1", Type: "r4.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 2.24}, "r4.8xlarge": {Region: "sa-east-1", Type: "r4.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 4.48}, "r4.16xlarge": {Region: "sa-east-1", Type: "r4.16xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 8.96}, "r5.large": {Region: "sa-east-1", Type: "r5.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.201}, "r5.xlarge": {Region: "sa-east-1", Type: "r5.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.402}, "r5.2xlarge": {Region: "sa-east-1", Type: "r5.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.804}, "r5.4xlarge": {Region: "sa-east-1", Type: "r5.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.608}, "r5.8xlarge": {Region: "sa-east-1", Type: "r5.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 3.216}, "r5.12xlarge": {Region: "sa-east-1", Type: "r5.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.824}, "r5.16xlarge": {Region: "sa-east-1", Type: "r5.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 6.432}, "r5.24xlarge": {Region: "sa-east-1", Type: "r5.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 9.648}, "r5.metal": {Region: "sa-east-1", Type: "r5.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 9.648}, "r5a.large": {Region: "sa-east-1", Type: "r5a.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.181}, "r5a.xlarge": {Region: "sa-east-1", Type: "r5a.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.362}, "r5a.2xlarge": {Region: "sa-east-1", Type: "r5a.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.724}, "r5a.4xlarge": {Region: "sa-east-1", Type: "r5a.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.448}, "r5a.8xlarge": {Region: "sa-east-1", Type: "r5a.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.896}, "r5a.12xlarge": {Region: "sa-east-1", Type: "r5a.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.344}, "r5a.16xlarge": {Region: "sa-east-1", Type: "r5a.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.792}, "r5a.24xlarge": {Region: "sa-east-1", Type: "r5a.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.688}, "r5ad.large": {Region: "sa-east-1", Type: "r5ad.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.208}, "r5ad.xlarge": {Region: "sa-east-1", Type: "r5ad.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.416}, "r5ad.2xlarge": {Region: "sa-east-1", Type: "r5ad.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.832}, "r5ad.4xlarge": {Region: "sa-east-1", Type: "r5ad.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.664}, "r5ad.8xlarge": {Region: "sa-east-1", Type: "r5ad.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 3.328}, "r5ad.12xlarge": {Region: "sa-east-1", Type: "r5ad.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.992}, "r5ad.16xlarge": {Region: "sa-east-1", Type: "r5ad.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 6.656}, "r5ad.24xlarge": {Region: "sa-east-1", Type: "r5ad.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 9.984}, "r5d.large": {Region: "sa-east-1", Type: "r5d.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.228}, "r5d.xlarge": {Region: "sa-east-1", Type: "r5d.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.456}, "r5d.2xlarge": {Region: "sa-east-1", Type: "r5d.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.912}, "r5d.4xlarge": {Region: "sa-east-1", Type: "r5d.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.824}, "r5d.8xlarge": {Region: "sa-east-1", Type: "r5d.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 3.648}, "r5d.12xlarge": {Region: "sa-east-1", Type: "r5d.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 5.472}, "r5d.16xlarge": {Region: "sa-east-1", Type: "r5d.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 7.296}, "r5d.24xlarge": {Region: "sa-east-1", Type: "r5d.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 10.944}, "r5d.metal": {Region: "sa-east-1", Type: "r5d.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 10.944}, "r5n.large": {Region: "sa-east-1", Type: "r5n.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.236}, "r5n.xlarge": {Region: "sa-east-1", Type: "r5n.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.472}, "r5n.2xlarge": {Region: "sa-east-1", Type: "r5n.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.944}, "r5n.4xlarge": {Region: "sa-east-1", Type: "r5n.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.888}, "r5n.8xlarge": {Region: "sa-east-1", Type: "r5n.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 3.776}, "r5n.12xlarge": {Region: "sa-east-1", Type: "r5n.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 5.664}, "r5n.16xlarge": {Region: "sa-east-1", Type: "r5n.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 7.552}, "r5n.24xlarge": {Region: "sa-east-1", Type: "r5n.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 11.328}, "r5n.metal": {Region: "sa-east-1", Type: "r5n.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 11.328}, "r6g.medium": {Region: "sa-east-1", Type: "r6g.medium", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0804}, "r6g.large": {Region: "sa-east-1", Type: "r6g.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.1608}, "r6g.xlarge": {Region: "sa-east-1", Type: "r6g.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.3216}, "r6g.2xlarge": {Region: "sa-east-1", Type: "r6g.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.6432}, "r6g.4xlarge": {Region: "sa-east-1", Type: "r6g.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.2864}, "r6g.8xlarge": {Region: "sa-east-1", Type: "r6g.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.5728}, "r6g.12xlarge": {Region: "sa-east-1", Type: "r6g.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.8592}, "r6g.16xlarge": {Region: "sa-east-1", Type: "r6g.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.1456}, "r6g.metal": {Region: "sa-east-1", Type: "r6g.metal", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.1456}, "r6i.large": {Region: "sa-east-1", Type: "r6i.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.201}, "r6i.xlarge": {Region: "sa-east-1", Type: "r6i.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.402}, "r6i.2xlarge": {Region: "sa-east-1", Type: "r6i.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.804}, "r6i.4xlarge": {Region: "sa-east-1", Type: "r6i.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.608}, "r6i.8xlarge": {Region: "sa-east-1", Type: "r6i.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 3.216}, "r6i.12xlarge": {Region: "sa-east-1", Type: "r6i.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.824}, "r6i.16xlarge": {Region: "sa-east-1", Type: "r6i.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 6.432}, "r6i.24xlarge": {Region: "sa-east-1", Type: "r6i.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 9.648}, "r6i.32xlarge": {Region: "sa-east-1", Type: "r6i.32xlarge", Memory: kresource.MustParse("1048576Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 12.864}, "r6i.metal": {Region: "sa-east-1", Type: "r6i.metal", Memory: kresource.MustParse("1048576Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 12.864}, "t1.micro": {Region: "sa-east-1", Type: "t1.micro", Memory: kresource.MustParse("627Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.027}, "t2.nano": {Region: "sa-east-1", Type: "t2.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0093}, "t2.micro": {Region: "sa-east-1", Type: "t2.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0186}, "t2.small": {Region: "sa-east-1", Type: "t2.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0372}, "t2.medium": {Region: "sa-east-1", Type: "t2.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0744}, "t2.large": {Region: "sa-east-1", Type: "t2.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.1488}, "t2.xlarge": {Region: "sa-east-1", Type: "t2.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.2976}, "t2.2xlarge": {Region: "sa-east-1", Type: "t2.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.5952}, "t3.nano": {Region: "sa-east-1", Type: "t3.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0084}, "t3.micro": {Region: "sa-east-1", Type: "t3.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0168}, "t3.small": {Region: "sa-east-1", Type: "t3.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0336}, "t3.medium": {Region: "sa-east-1", Type: "t3.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0672}, "t3.large": {Region: "sa-east-1", Type: "t3.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.1344}, "t3.xlarge": {Region: "sa-east-1", Type: "t3.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.2688}, "t3.2xlarge": {Region: "sa-east-1", Type: "t3.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.5376}, "t3a.nano": {Region: "sa-east-1", Type: "t3a.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0076}, "t3a.micro": {Region: "sa-east-1", Type: "t3a.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0151}, "t3a.small": {Region: "sa-east-1", Type: "t3a.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0302}, "t3a.medium": {Region: "sa-east-1", Type: "t3a.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0605}, "t3a.large": {Region: "sa-east-1", Type: "t3a.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.121}, "t3a.xlarge": {Region: "sa-east-1", Type: "t3a.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.2419}, "t3a.2xlarge": {Region: "sa-east-1", Type: "t3a.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.4838}, "t4g.nano": {Region: "sa-east-1", Type: "t4g.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0067}, "t4g.micro": {Region: "sa-east-1", Type: "t4g.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0134}, "t4g.small": {Region: "sa-east-1", Type: "t4g.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0268}, "t4g.medium": {Region: "sa-east-1", Type: "t4g.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0536}, "t4g.large": {Region: "sa-east-1", Type: "t4g.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.1072}, "t4g.xlarge": {Region: "sa-east-1", Type: "t4g.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.2144}, "t4g.2xlarge": {Region: "sa-east-1", Type: "t4g.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.4288}, "x1.16xlarge": {Region: "sa-east-1", Type: "x1.16xlarge", Memory: kresource.MustParse("999424Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 13.005}, "x1.32xlarge": {Region: "sa-east-1", Type: "x1.32xlarge", Memory: kresource.MustParse("1998848Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 26.01}, "x1e.xlarge": {Region: "sa-east-1", Type: "x1e.xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 1.626}, "x1e.2xlarge": {Region: "sa-east-1", Type: "x1e.2xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 3.251}, "x1e.4xlarge": {Region: "sa-east-1", Type: "x1e.4xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 6.502}, "x1e.8xlarge": {Region: "sa-east-1", Type: "x1e.8xlarge", Memory: kresource.MustParse("999424Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 13.005}, "x1e.16xlarge": {Region: "sa-east-1", Type: "x1e.16xlarge", Memory: kresource.MustParse("1998848Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 26.01}, "x1e.32xlarge": {Region: "sa-east-1", Type: "x1e.32xlarge", Memory: kresource.MustParse("3997696Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 52.019}, }, "us-east-1": { "a1.medium": {Region: "us-east-1", Type: "a1.medium", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0255}, "a1.large": {Region: "us-east-1", Type: "a1.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.051}, "a1.xlarge": {Region: "us-east-1", Type: "a1.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.102}, "a1.2xlarge": {Region: "us-east-1", Type: "a1.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.204}, "a1.4xlarge": {Region: "us-east-1", Type: "a1.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.408}, "a1.metal": {Region: "us-east-1", Type: "a1.metal", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.408}, "c1.medium": {Region: "us-east-1", Type: "c1.medium", Memory: kresource.MustParse("1740Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.13}, "c1.xlarge": {Region: "us-east-1", Type: "c1.xlarge", Memory: kresource.MustParse("7168Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.52}, "c3.large": {Region: "us-east-1", Type: "c3.large", Memory: kresource.MustParse("3840Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.105}, "c3.xlarge": {Region: "us-east-1", Type: "c3.xlarge", Memory: kresource.MustParse("7680Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.21}, "c3.2xlarge": {Region: "us-east-1", Type: "c3.2xlarge", Memory: kresource.MustParse("15360Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.42}, "c3.4xlarge": {Region: "us-east-1", Type: "c3.4xlarge", Memory: kresource.MustParse("30720Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.84}, "c3.8xlarge": {Region: "us-east-1", Type: "c3.8xlarge", Memory: kresource.MustParse("61440Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.68}, "c4.large": {Region: "us-east-1", Type: "c4.large", Memory: kresource.MustParse("3840Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.1}, "c4.xlarge": {Region: "us-east-1", Type: "c4.xlarge", Memory: kresource.MustParse("7680Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.199}, "c4.2xlarge": {Region: "us-east-1", Type: "c4.2xlarge", Memory: kresource.MustParse("15360Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.398}, "c4.4xlarge": {Region: "us-east-1", Type: "c4.4xlarge", Memory: kresource.MustParse("30720Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.796}, "c4.8xlarge": {Region: "us-east-1", Type: "c4.8xlarge", Memory: kresource.MustParse("61440Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 1.591}, "c5.large": {Region: "us-east-1", Type: "c5.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.085}, "c5.xlarge": {Region: "us-east-1", Type: "c5.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.17}, "c5.2xlarge": {Region: "us-east-1", Type: "c5.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.34}, "c5.4xlarge": {Region: "us-east-1", Type: "c5.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.68}, "c5.9xlarge": {Region: "us-east-1", Type: "c5.9xlarge", Memory: kresource.MustParse("73728Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 1.53}, "c5.12xlarge": {Region: "us-east-1", Type: "c5.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.04}, "c5.18xlarge": {Region: "us-east-1", Type: "c5.18xlarge", Memory: kresource.MustParse("147456Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 3.06}, "c5.24xlarge": {Region: "us-east-1", Type: "c5.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.08}, "c5.metal": {Region: "us-east-1", Type: "c5.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.08}, "c5a.large": {Region: "us-east-1", Type: "c5a.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.077}, "c5a.xlarge": {Region: "us-east-1", Type: "c5a.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.154}, "c5a.2xlarge": {Region: "us-east-1", Type: "c5a.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.308}, "c5a.4xlarge": {Region: "us-east-1", Type: "c5a.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.616}, "c5a.8xlarge": {Region: "us-east-1", Type: "c5a.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.232}, "c5a.12xlarge": {Region: "us-east-1", Type: "c5a.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 1.848}, "c5a.16xlarge": {Region: "us-east-1", Type: "c5a.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.464}, "c5a.24xlarge": {Region: "us-east-1", Type: "c5a.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 3.696}, "c5ad.large": {Region: "us-east-1", Type: "c5ad.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.086}, "c5ad.xlarge": {Region: "us-east-1", Type: "c5ad.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.172}, "c5ad.2xlarge": {Region: "us-east-1", Type: "c5ad.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.344}, "c5ad.4xlarge": {Region: "us-east-1", Type: "c5ad.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.688}, "c5ad.8xlarge": {Region: "us-east-1", Type: "c5ad.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.376}, "c5ad.12xlarge": {Region: "us-east-1", Type: "c5ad.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.064}, "c5ad.16xlarge": {Region: "us-east-1", Type: "c5ad.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.752}, "c5ad.24xlarge": {Region: "us-east-1", Type: "c5ad.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.128}, "c5d.large": {Region: "us-east-1", Type: "c5d.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.096}, "c5d.xlarge": {Region: "us-east-1", Type: "c5d.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.192}, "c5d.2xlarge": {Region: "us-east-1", Type: "c5d.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.384}, "c5d.4xlarge": {Region: "us-east-1", Type: "c5d.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.768}, "c5d.9xlarge": {Region: "us-east-1", Type: "c5d.9xlarge", Memory: kresource.MustParse("73728Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 1.728}, "c5d.12xlarge": {Region: "us-east-1", Type: "c5d.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.304}, "c5d.18xlarge": {Region: "us-east-1", Type: "c5d.18xlarge", Memory: kresource.MustParse("147456Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 3.456}, "c5d.24xlarge": {Region: "us-east-1", Type: "c5d.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.608}, "c5d.metal": {Region: "us-east-1", Type: "c5d.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.608}, "c5n.large": {Region: "us-east-1", Type: "c5n.large", Memory: kresource.MustParse("5376Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.108}, "c5n.xlarge": {Region: "us-east-1", Type: "c5n.xlarge", Memory: kresource.MustParse("10752Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.216}, "c5n.2xlarge": {Region: "us-east-1", Type: "c5n.2xlarge", Memory: kresource.MustParse("21504Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.432}, "c5n.4xlarge": {Region: "us-east-1", Type: "c5n.4xlarge", Memory: kresource.MustParse("43008Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.864}, "c5n.9xlarge": {Region: "us-east-1", Type: "c5n.9xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 1.944}, "c5n.18xlarge": {Region: "us-east-1", Type: "c5n.18xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 3.888}, "c5n.metal": {Region: "us-east-1", Type: "c5n.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 3.888}, "c6a.large": {Region: "us-east-1", Type: "c6a.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0765}, "c6a.xlarge": {Region: "us-east-1", Type: "c6a.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.153}, "c6a.2xlarge": {Region: "us-east-1", Type: "c6a.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.306}, "c6a.4xlarge": {Region: "us-east-1", Type: "c6a.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.612}, "c6a.8xlarge": {Region: "us-east-1", Type: "c6a.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.224}, "c6a.12xlarge": {Region: "us-east-1", Type: "c6a.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 1.836}, "c6a.16xlarge": {Region: "us-east-1", Type: "c6a.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.448}, "c6a.24xlarge": {Region: "us-east-1", Type: "c6a.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 3.672}, "c6a.32xlarge": {Region: "us-east-1", Type: "c6a.32xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 4.896}, "c6a.48xlarge": {Region: "us-east-1", Type: "c6a.48xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("192"), GPU: 0, Inf: 0, Price: 7.344}, "c6a.metal": {Region: "us-east-1", Type: "c6a.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("192"), GPU: 0, Inf: 0, Price: 7.344}, "c6g.medium": {Region: "us-east-1", Type: "c6g.medium", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.034}, "c6g.large": {Region: "us-east-1", Type: "c6g.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.068}, "c6g.xlarge": {Region: "us-east-1", Type: "c6g.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.136}, "c6g.2xlarge": {Region: "us-east-1", Type: "c6g.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.272}, "c6g.4xlarge": {Region: "us-east-1", Type: "c6g.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.544}, "c6g.8xlarge": {Region: "us-east-1", Type: "c6g.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.088}, "c6g.12xlarge": {Region: "us-east-1", Type: "c6g.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 1.632}, "c6g.16xlarge": {Region: "us-east-1", Type: "c6g.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.176}, "c6g.metal": {Region: "us-east-1", Type: "c6g.metal", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.176}, "c6gd.medium": {Region: "us-east-1", Type: "c6gd.medium", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0384}, "c6gd.large": {Region: "us-east-1", Type: "c6gd.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0768}, "c6gd.xlarge": {Region: "us-east-1", Type: "c6gd.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1536}, "c6gd.2xlarge": {Region: "us-east-1", Type: "c6gd.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3072}, "c6gd.4xlarge": {Region: "us-east-1", Type: "c6gd.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.6144}, "c6gd.8xlarge": {Region: "us-east-1", Type: "c6gd.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.2288}, "c6gd.12xlarge": {Region: "us-east-1", Type: "c6gd.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 1.8432}, "c6gd.16xlarge": {Region: "us-east-1", Type: "c6gd.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.4576}, "c6gd.metal": {Region: "us-east-1", Type: "c6gd.metal", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.4576}, "c6gn.medium": {Region: "us-east-1", Type: "c6gn.medium", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0432}, "c6gn.large": {Region: "us-east-1", Type: "c6gn.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0864}, "c6gn.xlarge": {Region: "us-east-1", Type: "c6gn.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1728}, "c6gn.2xlarge": {Region: "us-east-1", Type: "c6gn.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3456}, "c6gn.4xlarge": {Region: "us-east-1", Type: "c6gn.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.6912}, "c6gn.8xlarge": {Region: "us-east-1", Type: "c6gn.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.3824}, "c6gn.12xlarge": {Region: "us-east-1", Type: "c6gn.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.0736}, "c6gn.16xlarge": {Region: "us-east-1", Type: "c6gn.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.7648}, "c6i.large": {Region: "us-east-1", Type: "c6i.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.085}, "c6i.xlarge": {Region: "us-east-1", Type: "c6i.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.17}, "c6i.2xlarge": {Region: "us-east-1", Type: "c6i.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.34}, "c6i.4xlarge": {Region: "us-east-1", Type: "c6i.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.68}, "c6i.8xlarge": {Region: "us-east-1", Type: "c6i.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.36}, "c6i.12xlarge": {Region: "us-east-1", Type: "c6i.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.04}, "c6i.16xlarge": {Region: "us-east-1", Type: "c6i.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.72}, "c6i.24xlarge": {Region: "us-east-1", Type: "c6i.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.08}, "c6i.32xlarge": {Region: "us-east-1", Type: "c6i.32xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 5.44}, "c6i.metal": {Region: "us-east-1", Type: "c6i.metal", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 5.44}, "cc2.8xlarge": {Region: "us-east-1", Type: "cc2.8xlarge", Memory: kresource.MustParse("61952Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.0}, "cr1.8xlarge": {Region: "us-east-1", Type: "cr1.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 3.5}, "d2.xlarge": {Region: "us-east-1", Type: "d2.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.69}, "d2.2xlarge": {Region: "us-east-1", Type: "d2.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.38}, "d2.4xlarge": {Region: "us-east-1", Type: "d2.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 2.76}, "d2.8xlarge": {Region: "us-east-1", Type: "d2.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 5.52}, "d3.xlarge": {Region: "us-east-1", Type: "d3.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.499}, "d3.2xlarge": {Region: "us-east-1", Type: "d3.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.999}, "d3.4xlarge": {Region: "us-east-1", Type: "d3.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.998}, "d3.8xlarge": {Region: "us-east-1", Type: "d3.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 3.99552}, "d3en.xlarge": {Region: "us-east-1", Type: "d3en.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.526}, "d3en.2xlarge": {Region: "us-east-1", Type: "d3en.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.051}, "d3en.4xlarge": {Region: "us-east-1", Type: "d3en.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 2.103}, "d3en.6xlarge": {Region: "us-east-1", Type: "d3en.6xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 0, Price: 3.154}, "d3en.8xlarge": {Region: "us-east-1", Type: "d3en.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 4.20576}, "d3en.12xlarge": {Region: "us-east-1", Type: "d3en.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 6.30864}, "dl1.24xlarge": {Region: "us-east-1", Type: "dl1.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 13.10904}, "f1.2xlarge": {Region: "us-east-1", Type: "f1.2xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.65}, "f1.4xlarge": {Region: "us-east-1", Type: "f1.4xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 3.3}, "f1.16xlarge": {Region: "us-east-1", Type: "f1.16xlarge", Memory: kresource.MustParse("999424Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 13.2}, "g2.2xlarge": {Region: "us-east-1", Type: "g2.2xlarge", Memory: kresource.MustParse("15360Mi"), CPU: kresource.MustParse("8"), GPU: 1, Inf: 0, Price: 0.65}, "g2.8xlarge": {Region: "us-east-1", Type: "g2.8xlarge", Memory: kresource.MustParse("61440Mi"), CPU: kresource.MustParse("32"), GPU: 4, Inf: 0, Price: 2.6}, "g3.4xlarge": {Region: "us-east-1", Type: "g3.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 1, Inf: 0, Price: 1.14}, "g3.8xlarge": {Region: "us-east-1", Type: "g3.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 2, Inf: 0, Price: 2.28}, "g3.16xlarge": {Region: "us-east-1", Type: "g3.16xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("64"), GPU: 4, Inf: 0, Price: 4.56}, "g3s.xlarge": {Region: "us-east-1", Type: "g3s.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 1, Inf: 0, Price: 0.75}, "g4ad.xlarge": {Region: "us-east-1", Type: "g4ad.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 1, Inf: 0, Price: 0.37853}, "g4ad.2xlarge": {Region: "us-east-1", Type: "g4ad.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 1, Inf: 0, Price: 0.54117}, "g4ad.4xlarge": {Region: "us-east-1", Type: "g4ad.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 1, Inf: 0, Price: 0.867}, "g4ad.8xlarge": {Region: "us-east-1", Type: "g4ad.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 2, Inf: 0, Price: 1.734}, "g4ad.16xlarge": {Region: "us-east-1", Type: "g4ad.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 4, Inf: 0, Price: 3.468}, "g4dn.xlarge": {Region: "us-east-1", Type: "g4dn.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 1, Inf: 0, Price: 0.526}, "g4dn.2xlarge": {Region: "us-east-1", Type: "g4dn.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 1, Inf: 0, Price: 0.752}, "g4dn.4xlarge": {Region: "us-east-1", Type: "g4dn.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 1, Inf: 0, Price: 1.204}, "g4dn.8xlarge": {Region: "us-east-1", Type: "g4dn.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 1, Inf: 0, Price: 2.176}, "g4dn.12xlarge": {Region: "us-east-1", Type: "g4dn.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 4, Inf: 0, Price: 3.912}, "g4dn.16xlarge": {Region: "us-east-1", Type: "g4dn.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 1, Inf: 0, Price: 4.352}, "g4dn.metal": {Region: "us-east-1", Type: "g4dn.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 8, Inf: 0, Price: 7.824}, "g5.xlarge": {Region: "us-east-1", Type: "g5.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 1, Inf: 0, Price: 1.006}, "g5.2xlarge": {Region: "us-east-1", Type: "g5.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 1, Inf: 0, Price: 1.212}, "g5.4xlarge": {Region: "us-east-1", Type: "g5.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 1, Inf: 0, Price: 1.624}, "g5.8xlarge": {Region: "us-east-1", Type: "g5.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 1, Inf: 0, Price: 2.448}, "g5.12xlarge": {Region: "us-east-1", Type: "g5.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 4, Inf: 0, Price: 5.672}, "g5.16xlarge": {Region: "us-east-1", Type: "g5.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 1, Inf: 0, Price: 4.096}, "g5.24xlarge": {Region: "us-east-1", Type: "g5.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 4, Inf: 0, Price: 8.144}, "g5.48xlarge": {Region: "us-east-1", Type: "g5.48xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("192"), GPU: 8, Inf: 0, Price: 16.288}, "g5g.xlarge": {Region: "us-east-1", Type: "g5g.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 1, Inf: 0, Price: 0.42}, "g5g.2xlarge": {Region: "us-east-1", Type: "g5g.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 1, Inf: 0, Price: 0.556}, "g5g.4xlarge": {Region: "us-east-1", Type: "g5g.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 1, Inf: 0, Price: 0.828}, "g5g.8xlarge": {Region: "us-east-1", Type: "g5g.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 1, Inf: 0, Price: 1.372}, "g5g.16xlarge": {Region: "us-east-1", Type: "g5g.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 2, Inf: 0, Price: 2.744}, "g5g.metal": {Region: "us-east-1", Type: "g5g.metal", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 2, Inf: 0, Price: 2.744}, "h1.2xlarge": {Region: "us-east-1", Type: "h1.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.468}, "h1.4xlarge": {Region: "us-east-1", Type: "h1.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.936}, "h1.8xlarge": {Region: "us-east-1", Type: "h1.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.872}, "h1.16xlarge": {Region: "us-east-1", Type: "h1.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.744}, "hs1.8xlarge": {Region: "us-east-1", Type: "hs1.8xlarge", Memory: kresource.MustParse("119808Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 4.6}, "i2.xlarge": {Region: "us-east-1", Type: "i2.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.853}, "i2.2xlarge": {Region: "us-east-1", Type: "i2.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.705}, "i2.4xlarge": {Region: "us-east-1", Type: "i2.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 3.41}, "i2.8xlarge": {Region: "us-east-1", Type: "i2.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 6.82}, "i3.large": {Region: "us-east-1", Type: "i3.large", Memory: kresource.MustParse("15616Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.156}, "i3.xlarge": {Region: "us-east-1", Type: "i3.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.312}, "i3.2xlarge": {Region: "us-east-1", Type: "i3.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.624}, "i3.4xlarge": {Region: "us-east-1", Type: "i3.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.248}, "i3.8xlarge": {Region: "us-east-1", Type: "i3.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.496}, "i3.16xlarge": {Region: "us-east-1", Type: "i3.16xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.992}, "i3.metal": {Region: "us-east-1", Type: "i3.metal", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.992}, "i3en.large": {Region: "us-east-1", Type: "i3en.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.226}, "i3en.xlarge": {Region: "us-east-1", Type: "i3en.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.452}, "i3en.2xlarge": {Region: "us-east-1", Type: "i3en.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.904}, "i3en.3xlarge": {Region: "us-east-1", Type: "i3en.3xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("12"), GPU: 0, Inf: 0, Price: 1.356}, "i3en.6xlarge": {Region: "us-east-1", Type: "i3en.6xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 0, Price: 2.712}, "i3en.12xlarge": {Region: "us-east-1", Type: "i3en.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 5.424}, "i3en.24xlarge": {Region: "us-east-1", Type: "i3en.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 10.848}, "i3en.metal": {Region: "us-east-1", Type: "i3en.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 10.848}, "im4gn.large": {Region: "us-east-1", Type: "im4gn.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.1819}, "im4gn.xlarge": {Region: "us-east-1", Type: "im4gn.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.36379}, "im4gn.2xlarge": {Region: "us-east-1", Type: "im4gn.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.72758}, "im4gn.4xlarge": {Region: "us-east-1", Type: "im4gn.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.45517}, "im4gn.8xlarge": {Region: "us-east-1", Type: "im4gn.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.91034}, "im4gn.16xlarge": {Region: "us-east-1", Type: "im4gn.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.82067}, "inf1.xlarge": {Region: "us-east-1", Type: "inf1.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 1, Price: 0.228}, "inf1.2xlarge": {Region: "us-east-1", Type: "inf1.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 1, Price: 0.362}, "inf1.6xlarge": {Region: "us-east-1", Type: "inf1.6xlarge", Memory: kresource.MustParse("49152Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 4, Price: 1.18}, "inf1.24xlarge": {Region: "us-east-1", Type: "inf1.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 16, Price: 4.721}, "is4gen.medium": {Region: "us-east-1", Type: "is4gen.medium", Memory: kresource.MustParse("6144Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.14408}, "is4gen.large": {Region: "us-east-1", Type: "is4gen.large", Memory: kresource.MustParse("12288Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.28815}, "is4gen.xlarge": {Region: "us-east-1", Type: "is4gen.xlarge", Memory: kresource.MustParse("24576Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.5763}, "is4gen.2xlarge": {Region: "us-east-1", Type: "is4gen.2xlarge", Memory: kresource.MustParse("49152Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.1526}, "is4gen.4xlarge": {Region: "us-east-1", Type: "is4gen.4xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 2.3052}, "is4gen.8xlarge": {Region: "us-east-1", Type: "is4gen.8xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 4.6104}, "m1.small": {Region: "us-east-1", Type: "m1.small", Memory: kresource.MustParse("1740Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.044}, "m1.medium": {Region: "us-east-1", Type: "m1.medium", Memory: kresource.MustParse("3840Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.087}, "m1.large": {Region: "us-east-1", Type: "m1.large", Memory: kresource.MustParse("7680Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.175}, "m1.xlarge": {Region: "us-east-1", Type: "m1.xlarge", Memory: kresource.MustParse("15360Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.35}, "m2.xlarge": {Region: "us-east-1", Type: "m2.xlarge", Memory: kresource.MustParse("17510Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.245}, "m2.2xlarge": {Region: "us-east-1", Type: "m2.2xlarge", Memory: kresource.MustParse("35020Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.49}, "m2.4xlarge": {Region: "us-east-1", Type: "m2.4xlarge", Memory: kresource.MustParse("70041Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.98}, "m3.medium": {Region: "us-east-1", Type: "m3.medium", Memory: kresource.MustParse("3840Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.067}, "m3.large": {Region: "us-east-1", Type: "m3.large", Memory: kresource.MustParse("7680Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.133}, "m3.xlarge": {Region: "us-east-1", Type: "m3.xlarge", Memory: kresource.MustParse("15360Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.266}, "m3.2xlarge": {Region: "us-east-1", Type: "m3.2xlarge", Memory: kresource.MustParse("30720Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.532}, "m4.large": {Region: "us-east-1", Type: "m4.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.1}, "m4.xlarge": {Region: "us-east-1", Type: "m4.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.2}, "m4.2xlarge": {Region: "us-east-1", Type: "m4.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.4}, "m4.4xlarge": {Region: "us-east-1", Type: "m4.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.8}, "m4.10xlarge": {Region: "us-east-1", Type: "m4.10xlarge", Memory: kresource.MustParse("163840Mi"), CPU: kresource.MustParse("40"), GPU: 0, Inf: 0, Price: 2.0}, "m4.16xlarge": {Region: "us-east-1", Type: "m4.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.2}, "m5.large": {Region: "us-east-1", Type: "m5.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.096}, "m5.xlarge": {Region: "us-east-1", Type: "m5.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.192}, "m5.2xlarge": {Region: "us-east-1", Type: "m5.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.384}, "m5.4xlarge": {Region: "us-east-1", Type: "m5.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.768}, "m5.8xlarge": {Region: "us-east-1", Type: "m5.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.536}, "m5.12xlarge": {Region: "us-east-1", Type: "m5.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.304}, "m5.16xlarge": {Region: "us-east-1", Type: "m5.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.072}, "m5.24xlarge": {Region: "us-east-1", Type: "m5.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.608}, "m5.metal": {Region: "us-east-1", Type: "m5.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.608}, "m5a.large": {Region: "us-east-1", Type: "m5a.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.086}, "m5a.xlarge": {Region: "us-east-1", Type: "m5a.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.172}, "m5a.2xlarge": {Region: "us-east-1", Type: "m5a.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.344}, "m5a.4xlarge": {Region: "us-east-1", Type: "m5a.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.688}, "m5a.8xlarge": {Region: "us-east-1", Type: "m5a.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.376}, "m5a.12xlarge": {Region: "us-east-1", Type: "m5a.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.064}, "m5a.16xlarge": {Region: "us-east-1", Type: "m5a.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.752}, "m5a.24xlarge": {Region: "us-east-1", Type: "m5a.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.128}, "m5ad.large": {Region: "us-east-1", Type: "m5ad.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.103}, "m5ad.xlarge": {Region: "us-east-1", Type: "m5ad.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.206}, "m5ad.2xlarge": {Region: "us-east-1", Type: "m5ad.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.412}, "m5ad.4xlarge": {Region: "us-east-1", Type: "m5ad.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.824}, "m5ad.8xlarge": {Region: "us-east-1", Type: "m5ad.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.648}, "m5ad.12xlarge": {Region: "us-east-1", Type: "m5ad.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.472}, "m5ad.16xlarge": {Region: "us-east-1", Type: "m5ad.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.296}, "m5ad.24xlarge": {Region: "us-east-1", Type: "m5ad.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.944}, "m5d.large": {Region: "us-east-1", Type: "m5d.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.113}, "m5d.xlarge": {Region: "us-east-1", Type: "m5d.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.226}, "m5d.2xlarge": {Region: "us-east-1", Type: "m5d.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.452}, "m5d.4xlarge": {Region: "us-east-1", Type: "m5d.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.904}, "m5d.8xlarge": {Region: "us-east-1", Type: "m5d.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.808}, "m5d.12xlarge": {Region: "us-east-1", Type: "m5d.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.712}, "m5d.16xlarge": {Region: "us-east-1", Type: "m5d.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.616}, "m5d.24xlarge": {Region: "us-east-1", Type: "m5d.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.424}, "m5d.metal": {Region: "us-east-1", Type: "m5d.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.424}, "m5dn.large": {Region: "us-east-1", Type: "m5dn.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.136}, "m5dn.xlarge": {Region: "us-east-1", Type: "m5dn.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.272}, "m5dn.2xlarge": {Region: "us-east-1", Type: "m5dn.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.544}, "m5dn.4xlarge": {Region: "us-east-1", Type: "m5dn.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.088}, "m5dn.8xlarge": {Region: "us-east-1", Type: "m5dn.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.176}, "m5dn.12xlarge": {Region: "us-east-1", Type: "m5dn.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.264}, "m5dn.16xlarge": {Region: "us-east-1", Type: "m5dn.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.352}, "m5dn.24xlarge": {Region: "us-east-1", Type: "m5dn.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.528}, "m5dn.metal": {Region: "us-east-1", Type: "m5dn.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.528}, "m5n.large": {Region: "us-east-1", Type: "m5n.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.119}, "m5n.xlarge": {Region: "us-east-1", Type: "m5n.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.238}, "m5n.2xlarge": {Region: "us-east-1", Type: "m5n.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.476}, "m5n.4xlarge": {Region: "us-east-1", Type: "m5n.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.952}, "m5n.8xlarge": {Region: "us-east-1", Type: "m5n.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.904}, "m5n.12xlarge": {Region: "us-east-1", Type: "m5n.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.856}, "m5n.16xlarge": {Region: "us-east-1", Type: "m5n.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.808}, "m5n.24xlarge": {Region: "us-east-1", Type: "m5n.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.712}, "m5n.metal": {Region: "us-east-1", Type: "m5n.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.712}, "m5zn.large": {Region: "us-east-1", Type: "m5zn.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.1652}, "m5zn.xlarge": {Region: "us-east-1", Type: "m5zn.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.3303}, "m5zn.2xlarge": {Region: "us-east-1", Type: "m5zn.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.6607}, "m5zn.3xlarge": {Region: "us-east-1", Type: "m5zn.3xlarge", Memory: kresource.MustParse("49152Mi"), CPU: kresource.MustParse("12"), GPU: 0, Inf: 0, Price: 0.991}, "m5zn.6xlarge": {Region: "us-east-1", Type: "m5zn.6xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 0, Price: 1.982}, "m5zn.12xlarge": {Region: "us-east-1", Type: "m5zn.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.9641}, "m5zn.metal": {Region: "us-east-1", Type: "m5zn.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.9641}, "m6a.large": {Region: "us-east-1", Type: "m6a.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0864}, "m6a.xlarge": {Region: "us-east-1", Type: "m6a.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1728}, "m6a.2xlarge": {Region: "us-east-1", Type: "m6a.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3456}, "m6a.4xlarge": {Region: "us-east-1", Type: "m6a.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.6912}, "m6a.8xlarge": {Region: "us-east-1", Type: "m6a.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.3824}, "m6a.12xlarge": {Region: "us-east-1", Type: "m6a.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.0736}, "m6a.16xlarge": {Region: "us-east-1", Type: "m6a.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.7648}, "m6a.24xlarge": {Region: "us-east-1", Type: "m6a.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.1472}, "m6a.32xlarge": {Region: "us-east-1", Type: "m6a.32xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 5.5296}, "m6a.48xlarge": {Region: "us-east-1", Type: "m6a.48xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("192"), GPU: 0, Inf: 0, Price: 8.2944}, "m6a.metal": {Region: "us-east-1", Type: "m6a.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("192"), GPU: 0, Inf: 0, Price: 8.2944}, "m6g.medium": {Region: "us-east-1", Type: "m6g.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0385}, "m6g.large": {Region: "us-east-1", Type: "m6g.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.077}, "m6g.xlarge": {Region: "us-east-1", Type: "m6g.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.154}, "m6g.2xlarge": {Region: "us-east-1", Type: "m6g.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.308}, "m6g.4xlarge": {Region: "us-east-1", Type: "m6g.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.616}, "m6g.8xlarge": {Region: "us-east-1", Type: "m6g.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.232}, "m6g.12xlarge": {Region: "us-east-1", Type: "m6g.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 1.848}, "m6g.16xlarge": {Region: "us-east-1", Type: "m6g.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.464}, "m6g.metal": {Region: "us-east-1", Type: "m6g.metal", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.464}, "m6gd.medium": {Region: "us-east-1", Type: "m6gd.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0452}, "m6gd.large": {Region: "us-east-1", Type: "m6gd.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0904}, "m6gd.xlarge": {Region: "us-east-1", Type: "m6gd.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1808}, "m6gd.2xlarge": {Region: "us-east-1", Type: "m6gd.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3616}, "m6gd.4xlarge": {Region: "us-east-1", Type: "m6gd.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.7232}, "m6gd.8xlarge": {Region: "us-east-1", Type: "m6gd.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.4464}, "m6gd.12xlarge": {Region: "us-east-1", Type: "m6gd.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.1696}, "m6gd.16xlarge": {Region: "us-east-1", Type: "m6gd.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.8928}, "m6gd.metal": {Region: "us-east-1", Type: "m6gd.metal", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.8928}, "m6i.large": {Region: "us-east-1", Type: "m6i.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.096}, "m6i.xlarge": {Region: "us-east-1", Type: "m6i.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.192}, "m6i.2xlarge": {Region: "us-east-1", Type: "m6i.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.384}, "m6i.4xlarge": {Region: "us-east-1", Type: "m6i.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.768}, "m6i.8xlarge": {Region: "us-east-1", Type: "m6i.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.536}, "m6i.12xlarge": {Region: "us-east-1", Type: "m6i.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.304}, "m6i.16xlarge": {Region: "us-east-1", Type: "m6i.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.072}, "m6i.24xlarge": {Region: "us-east-1", Type: "m6i.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.608}, "m6i.32xlarge": {Region: "us-east-1", Type: "m6i.32xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 6.144}, "m6i.metal": {Region: "us-east-1", Type: "m6i.metal", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 6.144}, "p2.xlarge": {Region: "us-east-1", Type: "p2.xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("4"), GPU: 1, Inf: 0, Price: 0.9}, "p2.8xlarge": {Region: "us-east-1", Type: "p2.8xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("32"), GPU: 8, Inf: 0, Price: 7.2}, "p2.16xlarge": {Region: "us-east-1", Type: "p2.16xlarge", Memory: kresource.MustParse("749568Mi"), CPU: kresource.MustParse("64"), GPU: 16, Inf: 0, Price: 14.4}, "p3.2xlarge": {Region: "us-east-1", Type: "p3.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 1, Inf: 0, Price: 3.06}, "p3.8xlarge": {Region: "us-east-1", Type: "p3.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 4, Inf: 0, Price: 12.24}, "p3.16xlarge": {Region: "us-east-1", Type: "p3.16xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("64"), GPU: 8, Inf: 0, Price: 24.48}, "p3dn.24xlarge": {Region: "us-east-1", Type: "p3dn.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 8, Inf: 0, Price: 31.212}, "p4d.24xlarge": {Region: "us-east-1", Type: "p4d.24xlarge", Memory: kresource.MustParse("1179648Mi"), CPU: kresource.MustParse("96"), GPU: 8, Inf: 0, Price: 32.7726}, "r3.large": {Region: "us-east-1", Type: "r3.large", Memory: kresource.MustParse("15616Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.166}, "r3.xlarge": {Region: "us-east-1", Type: "r3.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.333}, "r3.2xlarge": {Region: "us-east-1", Type: "r3.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.665}, "r3.4xlarge": {Region: "us-east-1", Type: "r3.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.33}, "r3.8xlarge": {Region: "us-east-1", Type: "r3.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.66}, "r4.large": {Region: "us-east-1", Type: "r4.large", Memory: kresource.MustParse("15616Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.133}, "r4.xlarge": {Region: "us-east-1", Type: "r4.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.266}, "r4.2xlarge": {Region: "us-east-1", Type: "r4.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.532}, "r4.4xlarge": {Region: "us-east-1", Type: "r4.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.064}, "r4.8xlarge": {Region: "us-east-1", Type: "r4.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.128}, "r4.16xlarge": {Region: "us-east-1", Type: "r4.16xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.256}, "r5.large": {Region: "us-east-1", Type: "r5.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.126}, "r5.xlarge": {Region: "us-east-1", Type: "r5.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.252}, "r5.2xlarge": {Region: "us-east-1", Type: "r5.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.504}, "r5.4xlarge": {Region: "us-east-1", Type: "r5.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.008}, "r5.8xlarge": {Region: "us-east-1", Type: "r5.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.016}, "r5.12xlarge": {Region: "us-east-1", Type: "r5.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.024}, "r5.16xlarge": {Region: "us-east-1", Type: "r5.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.032}, "r5.24xlarge": {Region: "us-east-1", Type: "r5.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.048}, "r5.metal": {Region: "us-east-1", Type: "r5.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.048}, "r5a.large": {Region: "us-east-1", Type: "r5a.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.113}, "r5a.xlarge": {Region: "us-east-1", Type: "r5a.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.226}, "r5a.2xlarge": {Region: "us-east-1", Type: "r5a.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.452}, "r5a.4xlarge": {Region: "us-east-1", Type: "r5a.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.904}, "r5a.8xlarge": {Region: "us-east-1", Type: "r5a.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.808}, "r5a.12xlarge": {Region: "us-east-1", Type: "r5a.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.712}, "r5a.16xlarge": {Region: "us-east-1", Type: "r5a.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.616}, "r5a.24xlarge": {Region: "us-east-1", Type: "r5a.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.424}, "r5ad.large": {Region: "us-east-1", Type: "r5ad.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.131}, "r5ad.xlarge": {Region: "us-east-1", Type: "r5ad.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.262}, "r5ad.2xlarge": {Region: "us-east-1", Type: "r5ad.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.524}, "r5ad.4xlarge": {Region: "us-east-1", Type: "r5ad.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.048}, "r5ad.8xlarge": {Region: "us-east-1", Type: "r5ad.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.096}, "r5ad.12xlarge": {Region: "us-east-1", Type: "r5ad.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.144}, "r5ad.16xlarge": {Region: "us-east-1", Type: "r5ad.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.192}, "r5ad.24xlarge": {Region: "us-east-1", Type: "r5ad.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.288}, "r5b.large": {Region: "us-east-1", Type: "r5b.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.149}, "r5b.xlarge": {Region: "us-east-1", Type: "r5b.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.298}, "r5b.2xlarge": {Region: "us-east-1", Type: "r5b.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.596}, "r5b.4xlarge": {Region: "us-east-1", Type: "r5b.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.192}, "r5b.8xlarge": {Region: "us-east-1", Type: "r5b.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.384}, "r5b.12xlarge": {Region: "us-east-1", Type: "r5b.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.576}, "r5b.16xlarge": {Region: "us-east-1", Type: "r5b.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.768}, "r5b.24xlarge": {Region: "us-east-1", Type: "r5b.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.152}, "r5b.metal": {Region: "us-east-1", Type: "r5b.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.152}, "r5d.large": {Region: "us-east-1", Type: "r5d.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.144}, "r5d.xlarge": {Region: "us-east-1", Type: "r5d.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.288}, "r5d.2xlarge": {Region: "us-east-1", Type: "r5d.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.576}, "r5d.4xlarge": {Region: "us-east-1", Type: "r5d.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.152}, "r5d.8xlarge": {Region: "us-east-1", Type: "r5d.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.304}, "r5d.12xlarge": {Region: "us-east-1", Type: "r5d.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.456}, "r5d.16xlarge": {Region: "us-east-1", Type: "r5d.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.608}, "r5d.24xlarge": {Region: "us-east-1", Type: "r5d.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.912}, "r5d.metal": {Region: "us-east-1", Type: "r5d.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.912}, "r5dn.large": {Region: "us-east-1", Type: "r5dn.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.167}, "r5dn.xlarge": {Region: "us-east-1", Type: "r5dn.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.334}, "r5dn.2xlarge": {Region: "us-east-1", Type: "r5dn.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.668}, "r5dn.4xlarge": {Region: "us-east-1", Type: "r5dn.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.336}, "r5dn.8xlarge": {Region: "us-east-1", Type: "r5dn.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.672}, "r5dn.12xlarge": {Region: "us-east-1", Type: "r5dn.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.008}, "r5dn.16xlarge": {Region: "us-east-1", Type: "r5dn.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.344}, "r5dn.24xlarge": {Region: "us-east-1", Type: "r5dn.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.016}, "r5dn.metal": {Region: "us-east-1", Type: "r5dn.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.016}, "r5n.large": {Region: "us-east-1", Type: "r5n.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.149}, "r5n.xlarge": {Region: "us-east-1", Type: "r5n.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.298}, "r5n.2xlarge": {Region: "us-east-1", Type: "r5n.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.596}, "r5n.4xlarge": {Region: "us-east-1", Type: "r5n.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.192}, "r5n.8xlarge": {Region: "us-east-1", Type: "r5n.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.384}, "r5n.12xlarge": {Region: "us-east-1", Type: "r5n.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.576}, "r5n.16xlarge": {Region: "us-east-1", Type: "r5n.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.768}, "r5n.24xlarge": {Region: "us-east-1", Type: "r5n.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.152}, "r5n.metal": {Region: "us-east-1", Type: "r5n.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.152}, "r6g.medium": {Region: "us-east-1", Type: "r6g.medium", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0504}, "r6g.large": {Region: "us-east-1", Type: "r6g.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.1008}, "r6g.xlarge": {Region: "us-east-1", Type: "r6g.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.2016}, "r6g.2xlarge": {Region: "us-east-1", Type: "r6g.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.4032}, "r6g.4xlarge": {Region: "us-east-1", Type: "r6g.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.8064}, "r6g.8xlarge": {Region: "us-east-1", Type: "r6g.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.6128}, "r6g.12xlarge": {Region: "us-east-1", Type: "r6g.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.4192}, "r6g.16xlarge": {Region: "us-east-1", Type: "r6g.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.2256}, "r6g.metal": {Region: "us-east-1", Type: "r6g.metal", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.2256}, "r6gd.medium": {Region: "us-east-1", Type: "r6gd.medium", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0576}, "r6gd.large": {Region: "us-east-1", Type: "r6gd.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.1152}, "r6gd.xlarge": {Region: "us-east-1", Type: "r6gd.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.2304}, "r6gd.2xlarge": {Region: "us-east-1", Type: "r6gd.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.4608}, "r6gd.4xlarge": {Region: "us-east-1", Type: "r6gd.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.9216}, "r6gd.8xlarge": {Region: "us-east-1", Type: "r6gd.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.8432}, "r6gd.12xlarge": {Region: "us-east-1", Type: "r6gd.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.7648}, "r6gd.16xlarge": {Region: "us-east-1", Type: "r6gd.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.6864}, "r6gd.metal": {Region: "us-east-1", Type: "r6gd.metal", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.6864}, "r6i.large": {Region: "us-east-1", Type: "r6i.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.126}, "r6i.xlarge": {Region: "us-east-1", Type: "r6i.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.252}, "r6i.2xlarge": {Region: "us-east-1", Type: "r6i.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.504}, "r6i.4xlarge": {Region: "us-east-1", Type: "r6i.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.008}, "r6i.8xlarge": {Region: "us-east-1", Type: "r6i.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.016}, "r6i.12xlarge": {Region: "us-east-1", Type: "r6i.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.024}, "r6i.16xlarge": {Region: "us-east-1", Type: "r6i.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.032}, "r6i.24xlarge": {Region: "us-east-1", Type: "r6i.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.048}, "r6i.32xlarge": {Region: "us-east-1", Type: "r6i.32xlarge", Memory: kresource.MustParse("1048576Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 8.064}, "r6i.metal": {Region: "us-east-1", Type: "r6i.metal", Memory: kresource.MustParse("1048576Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 8.064}, "t1.micro": {Region: "us-east-1", Type: "t1.micro", Memory: kresource.MustParse("627Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.02}, "t2.nano": {Region: "us-east-1", Type: "t2.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0058}, "t2.micro": {Region: "us-east-1", Type: "t2.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0116}, "t2.small": {Region: "us-east-1", Type: "t2.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.023}, "t2.medium": {Region: "us-east-1", Type: "t2.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0464}, "t2.large": {Region: "us-east-1", Type: "t2.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0928}, "t2.xlarge": {Region: "us-east-1", Type: "t2.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1856}, "t2.2xlarge": {Region: "us-east-1", Type: "t2.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3712}, "t3.nano": {Region: "us-east-1", Type: "t3.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0052}, "t3.micro": {Region: "us-east-1", Type: "t3.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0104}, "t3.small": {Region: "us-east-1", Type: "t3.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0208}, "t3.medium": {Region: "us-east-1", Type: "t3.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0416}, "t3.large": {Region: "us-east-1", Type: "t3.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0832}, "t3.xlarge": {Region: "us-east-1", Type: "t3.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1664}, "t3.2xlarge": {Region: "us-east-1", Type: "t3.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3328}, "t3a.nano": {Region: "us-east-1", Type: "t3a.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0047}, "t3a.micro": {Region: "us-east-1", Type: "t3a.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0094}, "t3a.small": {Region: "us-east-1", Type: "t3a.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0188}, "t3a.medium": {Region: "us-east-1", Type: "t3a.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0376}, "t3a.large": {Region: "us-east-1", Type: "t3a.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0752}, "t3a.xlarge": {Region: "us-east-1", Type: "t3a.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1504}, "t3a.2xlarge": {Region: "us-east-1", Type: "t3a.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3008}, "t4g.nano": {Region: "us-east-1", Type: "t4g.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0042}, "t4g.micro": {Region: "us-east-1", Type: "t4g.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0084}, "t4g.small": {Region: "us-east-1", Type: "t4g.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0168}, "t4g.medium": {Region: "us-east-1", Type: "t4g.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0336}, "t4g.large": {Region: "us-east-1", Type: "t4g.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0672}, "t4g.xlarge": {Region: "us-east-1", Type: "t4g.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1344}, "t4g.2xlarge": {Region: "us-east-1", Type: "t4g.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.2688}, "u-12tb1.112xlarge": {Region: "us-east-1", Type: "u-12tb1.112xlarge", Memory: kresource.MustParse("12582912Mi"), CPU: kresource.MustParse("448"), GPU: 0, Inf: 0, Price: 109.2}, "u-3tb1.56xlarge": {Region: "us-east-1", Type: "u-3tb1.56xlarge", Memory: kresource.MustParse("3145728Mi"), CPU: kresource.MustParse("224"), GPU: 0, Inf: 0, Price: 27.3}, "u-6tb1.56xlarge": {Region: "us-east-1", Type: "u-6tb1.56xlarge", Memory: kresource.MustParse("6291456Mi"), CPU: kresource.MustParse("224"), GPU: 0, Inf: 0, Price: 46.40391}, "u-6tb1.112xlarge": {Region: "us-east-1", Type: "u-6tb1.112xlarge", Memory: kresource.MustParse("6291456Mi"), CPU: kresource.MustParse("448"), GPU: 0, Inf: 0, Price: 54.6}, "u-9tb1.112xlarge": {Region: "us-east-1", Type: "u-9tb1.112xlarge", Memory: kresource.MustParse("9437184Mi"), CPU: kresource.MustParse("448"), GPU: 0, Inf: 0, Price: 81.9}, "vt1.3xlarge": {Region: "us-east-1", Type: "vt1.3xlarge", Memory: kresource.MustParse("24576Mi"), CPU: kresource.MustParse("12"), GPU: 0, Inf: 0, Price: 0.65}, "vt1.6xlarge": {Region: "us-east-1", Type: "vt1.6xlarge", Memory: kresource.MustParse("49152Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 0, Price: 1.3}, "vt1.24xlarge": {Region: "us-east-1", Type: "vt1.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.2}, "x1.16xlarge": {Region: "us-east-1", Type: "x1.16xlarge", Memory: kresource.MustParse("999424Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 6.669}, "x1.32xlarge": {Region: "us-east-1", Type: "x1.32xlarge", Memory: kresource.MustParse("1998848Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 13.338}, "x1e.xlarge": {Region: "us-east-1", Type: "x1e.xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.834}, "x1e.2xlarge": {Region: "us-east-1", Type: "x1e.2xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.668}, "x1e.4xlarge": {Region: "us-east-1", Type: "x1e.4xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 3.336}, "x1e.8xlarge": {Region: "us-east-1", Type: "x1e.8xlarge", Memory: kresource.MustParse("999424Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 6.672}, "x1e.16xlarge": {Region: "us-east-1", Type: "x1e.16xlarge", Memory: kresource.MustParse("1998848Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 13.344}, "x1e.32xlarge": {Region: "us-east-1", Type: "x1e.32xlarge", Memory: kresource.MustParse("3997696Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 26.688}, "x2gd.medium": {Region: "us-east-1", Type: "x2gd.medium", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0835}, "x2gd.large": {Region: "us-east-1", Type: "x2gd.large", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.167}, "x2gd.xlarge": {Region: "us-east-1", Type: "x2gd.xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.334}, "x2gd.2xlarge": {Region: "us-east-1", Type: "x2gd.2xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.668}, "x2gd.4xlarge": {Region: "us-east-1", Type: "x2gd.4xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.336}, "x2gd.8xlarge": {Region: "us-east-1", Type: "x2gd.8xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.672}, "x2gd.12xlarge": {Region: "us-east-1", Type: "x2gd.12xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.008}, "x2gd.16xlarge": {Region: "us-east-1", Type: "x2gd.16xlarge", Memory: kresource.MustParse("1048576Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.344}, "x2gd.metal": {Region: "us-east-1", Type: "x2gd.metal", Memory: kresource.MustParse("1048576Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.344}, "x2idn.16xlarge": {Region: "us-east-1", Type: "x2idn.16xlarge", Memory: kresource.MustParse("1048576Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 6.669}, "x2idn.24xlarge": {Region: "us-east-1", Type: "x2idn.24xlarge", Memory: kresource.MustParse("1572864Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 10.0035}, "x2idn.32xlarge": {Region: "us-east-1", Type: "x2idn.32xlarge", Memory: kresource.MustParse("2097152Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 13.338}, "x2iedn.xlarge": {Region: "us-east-1", Type: "x2iedn.xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.83363}, "x2iedn.2xlarge": {Region: "us-east-1", Type: "x2iedn.2xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.66725}, "x2iedn.4xlarge": {Region: "us-east-1", Type: "x2iedn.4xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 3.3345}, "x2iedn.8xlarge": {Region: "us-east-1", Type: "x2iedn.8xlarge", Memory: kresource.MustParse("1048576Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 6.669}, "x2iedn.16xlarge": {Region: "us-east-1", Type: "x2iedn.16xlarge", Memory: kresource.MustParse("2097152Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 13.338}, "x2iedn.24xlarge": {Region: "us-east-1", Type: "x2iedn.24xlarge", Memory: kresource.MustParse("3145728Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 20.007}, "x2iedn.32xlarge": {Region: "us-east-1", Type: "x2iedn.32xlarge", Memory: kresource.MustParse("4194304Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 26.676}, "x2iezn.2xlarge": {Region: "us-east-1", Type: "x2iezn.2xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.668}, "x2iezn.4xlarge": {Region: "us-east-1", Type: "x2iezn.4xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 3.336}, "x2iezn.6xlarge": {Region: "us-east-1", Type: "x2iezn.6xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 0, Price: 5.004}, "x2iezn.8xlarge": {Region: "us-east-1", Type: "x2iezn.8xlarge", Memory: kresource.MustParse("1048576Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 6.672}, "x2iezn.12xlarge": {Region: "us-east-1", Type: "x2iezn.12xlarge", Memory: kresource.MustParse("1572864Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 10.008}, "x2iezn.metal": {Region: "us-east-1", Type: "x2iezn.metal", Memory: kresource.MustParse("1572864Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 10.008}, "z1d.large": {Region: "us-east-1", Type: "z1d.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.186}, "z1d.xlarge": {Region: "us-east-1", Type: "z1d.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.372}, "z1d.2xlarge": {Region: "us-east-1", Type: "z1d.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.744}, "z1d.3xlarge": {Region: "us-east-1", Type: "z1d.3xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("12"), GPU: 0, Inf: 0, Price: 1.116}, "z1d.6xlarge": {Region: "us-east-1", Type: "z1d.6xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 0, Price: 2.232}, "z1d.12xlarge": {Region: "us-east-1", Type: "z1d.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.464}, "z1d.metal": {Region: "us-east-1", Type: "z1d.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.464}, }, "us-east-2": { "a1.medium": {Region: "us-east-2", Type: "a1.medium", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0255}, "a1.large": {Region: "us-east-2", Type: "a1.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.051}, "a1.xlarge": {Region: "us-east-2", Type: "a1.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.102}, "a1.2xlarge": {Region: "us-east-2", Type: "a1.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.204}, "a1.4xlarge": {Region: "us-east-2", Type: "a1.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.408}, "a1.metal": {Region: "us-east-2", Type: "a1.metal", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.408}, "c4.large": {Region: "us-east-2", Type: "c4.large", Memory: kresource.MustParse("3840Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.1}, "c4.xlarge": {Region: "us-east-2", Type: "c4.xlarge", Memory: kresource.MustParse("7680Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.199}, "c4.2xlarge": {Region: "us-east-2", Type: "c4.2xlarge", Memory: kresource.MustParse("15360Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.398}, "c4.4xlarge": {Region: "us-east-2", Type: "c4.4xlarge", Memory: kresource.MustParse("30720Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.796}, "c4.8xlarge": {Region: "us-east-2", Type: "c4.8xlarge", Memory: kresource.MustParse("61440Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 1.591}, "c5.large": {Region: "us-east-2", Type: "c5.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.085}, "c5.xlarge": {Region: "us-east-2", Type: "c5.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.17}, "c5.2xlarge": {Region: "us-east-2", Type: "c5.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.34}, "c5.4xlarge": {Region: "us-east-2", Type: "c5.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.68}, "c5.9xlarge": {Region: "us-east-2", Type: "c5.9xlarge", Memory: kresource.MustParse("73728Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 1.53}, "c5.12xlarge": {Region: "us-east-2", Type: "c5.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.04}, "c5.18xlarge": {Region: "us-east-2", Type: "c5.18xlarge", Memory: kresource.MustParse("147456Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 3.06}, "c5.24xlarge": {Region: "us-east-2", Type: "c5.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.08}, "c5.metal": {Region: "us-east-2", Type: "c5.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.08}, "c5a.large": {Region: "us-east-2", Type: "c5a.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.077}, "c5a.xlarge": {Region: "us-east-2", Type: "c5a.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.154}, "c5a.2xlarge": {Region: "us-east-2", Type: "c5a.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.308}, "c5a.4xlarge": {Region: "us-east-2", Type: "c5a.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.616}, "c5a.8xlarge": {Region: "us-east-2", Type: "c5a.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.232}, "c5a.12xlarge": {Region: "us-east-2", Type: "c5a.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 1.848}, "c5a.16xlarge": {Region: "us-east-2", Type: "c5a.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.464}, "c5a.24xlarge": {Region: "us-east-2", Type: "c5a.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 3.696}, "c5ad.large": {Region: "us-east-2", Type: "c5ad.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.086}, "c5ad.xlarge": {Region: "us-east-2", Type: "c5ad.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.172}, "c5ad.2xlarge": {Region: "us-east-2", Type: "c5ad.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.344}, "c5ad.4xlarge": {Region: "us-east-2", Type: "c5ad.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.688}, "c5ad.8xlarge": {Region: "us-east-2", Type: "c5ad.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.376}, "c5ad.12xlarge": {Region: "us-east-2", Type: "c5ad.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.064}, "c5ad.16xlarge": {Region: "us-east-2", Type: "c5ad.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.752}, "c5ad.24xlarge": {Region: "us-east-2", Type: "c5ad.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.128}, "c5d.large": {Region: "us-east-2", Type: "c5d.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.096}, "c5d.xlarge": {Region: "us-east-2", Type: "c5d.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.192}, "c5d.2xlarge": {Region: "us-east-2", Type: "c5d.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.384}, "c5d.4xlarge": {Region: "us-east-2", Type: "c5d.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.768}, "c5d.9xlarge": {Region: "us-east-2", Type: "c5d.9xlarge", Memory: kresource.MustParse("73728Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 1.728}, "c5d.12xlarge": {Region: "us-east-2", Type: "c5d.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.304}, "c5d.18xlarge": {Region: "us-east-2", Type: "c5d.18xlarge", Memory: kresource.MustParse("147456Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 3.456}, "c5d.24xlarge": {Region: "us-east-2", Type: "c5d.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.608}, "c5d.metal": {Region: "us-east-2", Type: "c5d.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.608}, "c5n.large": {Region: "us-east-2", Type: "c5n.large", Memory: kresource.MustParse("5376Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.108}, "c5n.xlarge": {Region: "us-east-2", Type: "c5n.xlarge", Memory: kresource.MustParse("10752Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.216}, "c5n.2xlarge": {Region: "us-east-2", Type: "c5n.2xlarge", Memory: kresource.MustParse("21504Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.432}, "c5n.4xlarge": {Region: "us-east-2", Type: "c5n.4xlarge", Memory: kresource.MustParse("43008Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.864}, "c5n.9xlarge": {Region: "us-east-2", Type: "c5n.9xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 1.944}, "c5n.18xlarge": {Region: "us-east-2", Type: "c5n.18xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 3.888}, "c5n.metal": {Region: "us-east-2", Type: "c5n.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 3.888}, "c6g.medium": {Region: "us-east-2", Type: "c6g.medium", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.034}, "c6g.large": {Region: "us-east-2", Type: "c6g.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.068}, "c6g.xlarge": {Region: "us-east-2", Type: "c6g.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.136}, "c6g.2xlarge": {Region: "us-east-2", Type: "c6g.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.272}, "c6g.4xlarge": {Region: "us-east-2", Type: "c6g.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.544}, "c6g.8xlarge": {Region: "us-east-2", Type: "c6g.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.088}, "c6g.12xlarge": {Region: "us-east-2", Type: "c6g.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 1.632}, "c6g.16xlarge": {Region: "us-east-2", Type: "c6g.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.176}, "c6g.metal": {Region: "us-east-2", Type: "c6g.metal", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.176}, "c6gd.medium": {Region: "us-east-2", Type: "c6gd.medium", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0384}, "c6gd.large": {Region: "us-east-2", Type: "c6gd.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0768}, "c6gd.xlarge": {Region: "us-east-2", Type: "c6gd.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1536}, "c6gd.2xlarge": {Region: "us-east-2", Type: "c6gd.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3072}, "c6gd.4xlarge": {Region: "us-east-2", Type: "c6gd.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.6144}, "c6gd.8xlarge": {Region: "us-east-2", Type: "c6gd.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.2288}, "c6gd.12xlarge": {Region: "us-east-2", Type: "c6gd.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 1.8432}, "c6gd.16xlarge": {Region: "us-east-2", Type: "c6gd.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.4576}, "c6gd.metal": {Region: "us-east-2", Type: "c6gd.metal", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.4576}, "c6gn.medium": {Region: "us-east-2", Type: "c6gn.medium", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0432}, "c6gn.large": {Region: "us-east-2", Type: "c6gn.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0864}, "c6gn.xlarge": {Region: "us-east-2", Type: "c6gn.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1728}, "c6gn.2xlarge": {Region: "us-east-2", Type: "c6gn.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3456}, "c6gn.4xlarge": {Region: "us-east-2", Type: "c6gn.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.6912}, "c6gn.8xlarge": {Region: "us-east-2", Type: "c6gn.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.3824}, "c6gn.12xlarge": {Region: "us-east-2", Type: "c6gn.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.0736}, "c6gn.16xlarge": {Region: "us-east-2", Type: "c6gn.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.7648}, "c6i.large": {Region: "us-east-2", Type: "c6i.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.085}, "c6i.xlarge": {Region: "us-east-2", Type: "c6i.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.17}, "c6i.2xlarge": {Region: "us-east-2", Type: "c6i.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.34}, "c6i.4xlarge": {Region: "us-east-2", Type: "c6i.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.68}, "c6i.8xlarge": {Region: "us-east-2", Type: "c6i.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.36}, "c6i.12xlarge": {Region: "us-east-2", Type: "c6i.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.04}, "c6i.16xlarge": {Region: "us-east-2", Type: "c6i.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.72}, "c6i.24xlarge": {Region: "us-east-2", Type: "c6i.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.08}, "c6i.32xlarge": {Region: "us-east-2", Type: "c6i.32xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 5.44}, "c6i.metal": {Region: "us-east-2", Type: "c6i.metal", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 5.44}, "d2.xlarge": {Region: "us-east-2", Type: "d2.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.69}, "d2.2xlarge": {Region: "us-east-2", Type: "d2.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.38}, "d2.4xlarge": {Region: "us-east-2", Type: "d2.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 2.76}, "d2.8xlarge": {Region: "us-east-2", Type: "d2.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 5.52}, "d3.xlarge": {Region: "us-east-2", Type: "d3.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.499}, "d3.2xlarge": {Region: "us-east-2", Type: "d3.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.999}, "d3.4xlarge": {Region: "us-east-2", Type: "d3.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.998}, "d3.8xlarge": {Region: "us-east-2", Type: "d3.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 3.99552}, "g3.4xlarge": {Region: "us-east-2", Type: "g3.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 1, Inf: 0, Price: 1.14}, "g3.8xlarge": {Region: "us-east-2", Type: "g3.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 2, Inf: 0, Price: 2.28}, "g3.16xlarge": {Region: "us-east-2", Type: "g3.16xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("64"), GPU: 4, Inf: 0, Price: 4.56}, "g3s.xlarge": {Region: "us-east-2", Type: "g3s.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 1, Inf: 0, Price: 0.75}, "g4ad.xlarge": {Region: "us-east-2", Type: "g4ad.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 1, Inf: 0, Price: 0.37853}, "g4ad.2xlarge": {Region: "us-east-2", Type: "g4ad.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 1, Inf: 0, Price: 0.54117}, "g4ad.4xlarge": {Region: "us-east-2", Type: "g4ad.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 1, Inf: 0, Price: 0.867}, "g4ad.8xlarge": {Region: "us-east-2", Type: "g4ad.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 2, Inf: 0, Price: 1.734}, "g4ad.16xlarge": {Region: "us-east-2", Type: "g4ad.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 4, Inf: 0, Price: 3.468}, "g4dn.xlarge": {Region: "us-east-2", Type: "g4dn.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 1, Inf: 0, Price: 0.526}, "g4dn.2xlarge": {Region: "us-east-2", Type: "g4dn.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 1, Inf: 0, Price: 0.752}, "g4dn.4xlarge": {Region: "us-east-2", Type: "g4dn.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 1, Inf: 0, Price: 1.204}, "g4dn.8xlarge": {Region: "us-east-2", Type: "g4dn.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 1, Inf: 0, Price: 2.176}, "g4dn.12xlarge": {Region: "us-east-2", Type: "g4dn.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 4, Inf: 0, Price: 3.912}, "g4dn.16xlarge": {Region: "us-east-2", Type: "g4dn.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 1, Inf: 0, Price: 4.352}, "g4dn.metal": {Region: "us-east-2", Type: "g4dn.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 8, Inf: 0, Price: 7.824}, "h1.2xlarge": {Region: "us-east-2", Type: "h1.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.468}, "h1.4xlarge": {Region: "us-east-2", Type: "h1.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.936}, "h1.8xlarge": {Region: "us-east-2", Type: "h1.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.872}, "h1.16xlarge": {Region: "us-east-2", Type: "h1.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.744}, "hpc6a.48xlarge": {Region: "us-east-2", Type: "hpc6a.48xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 2.88}, "i2.xlarge": {Region: "us-east-2", Type: "i2.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.853}, "i2.2xlarge": {Region: "us-east-2", Type: "i2.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.705}, "i2.4xlarge": {Region: "us-east-2", Type: "i2.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 3.41}, "i2.8xlarge": {Region: "us-east-2", Type: "i2.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 6.82}, "i3.large": {Region: "us-east-2", Type: "i3.large", Memory: kresource.MustParse("15616Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.156}, "i3.xlarge": {Region: "us-east-2", Type: "i3.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.312}, "i3.2xlarge": {Region: "us-east-2", Type: "i3.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.624}, "i3.4xlarge": {Region: "us-east-2", Type: "i3.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.248}, "i3.8xlarge": {Region: "us-east-2", Type: "i3.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.496}, "i3.16xlarge": {Region: "us-east-2", Type: "i3.16xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.992}, "i3.metal": {Region: "us-east-2", Type: "i3.metal", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.992}, "i3en.large": {Region: "us-east-2", Type: "i3en.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.226}, "i3en.xlarge": {Region: "us-east-2", Type: "i3en.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.452}, "i3en.2xlarge": {Region: "us-east-2", Type: "i3en.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.904}, "i3en.3xlarge": {Region: "us-east-2", Type: "i3en.3xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("12"), GPU: 0, Inf: 0, Price: 1.356}, "i3en.6xlarge": {Region: "us-east-2", Type: "i3en.6xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 0, Price: 2.712}, "i3en.12xlarge": {Region: "us-east-2", Type: "i3en.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 5.424}, "i3en.24xlarge": {Region: "us-east-2", Type: "i3en.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 10.848}, "i3en.metal": {Region: "us-east-2", Type: "i3en.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 10.848}, "im4gn.large": {Region: "us-east-2", Type: "im4gn.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.1819}, "im4gn.xlarge": {Region: "us-east-2", Type: "im4gn.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.36379}, "im4gn.2xlarge": {Region: "us-east-2", Type: "im4gn.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.72758}, "im4gn.4xlarge": {Region: "us-east-2", Type: "im4gn.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.45517}, "im4gn.8xlarge": {Region: "us-east-2", Type: "im4gn.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.91034}, "im4gn.16xlarge": {Region: "us-east-2", Type: "im4gn.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.82067}, "inf1.xlarge": {Region: "us-east-2", Type: "inf1.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 1, Price: 0.228}, "inf1.2xlarge": {Region: "us-east-2", Type: "inf1.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 1, Price: 0.362}, "inf1.6xlarge": {Region: "us-east-2", Type: "inf1.6xlarge", Memory: kresource.MustParse("49152Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 4, Price: 1.18}, "inf1.24xlarge": {Region: "us-east-2", Type: "inf1.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 16, Price: 4.721}, "is4gen.medium": {Region: "us-east-2", Type: "is4gen.medium", Memory: kresource.MustParse("6144Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.14408}, "is4gen.large": {Region: "us-east-2", Type: "is4gen.large", Memory: kresource.MustParse("12288Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.28815}, "is4gen.xlarge": {Region: "us-east-2", Type: "is4gen.xlarge", Memory: kresource.MustParse("24576Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.5763}, "is4gen.2xlarge": {Region: "us-east-2", Type: "is4gen.2xlarge", Memory: kresource.MustParse("49152Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.1526}, "is4gen.4xlarge": {Region: "us-east-2", Type: "is4gen.4xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 2.3052}, "is4gen.8xlarge": {Region: "us-east-2", Type: "is4gen.8xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 4.6104}, "m4.large": {Region: "us-east-2", Type: "m4.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.1}, "m4.xlarge": {Region: "us-east-2", Type: "m4.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.2}, "m4.2xlarge": {Region: "us-east-2", Type: "m4.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.4}, "m4.4xlarge": {Region: "us-east-2", Type: "m4.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.8}, "m4.10xlarge": {Region: "us-east-2", Type: "m4.10xlarge", Memory: kresource.MustParse("163840Mi"), CPU: kresource.MustParse("40"), GPU: 0, Inf: 0, Price: 2.0}, "m4.16xlarge": {Region: "us-east-2", Type: "m4.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.2}, "m5.large": {Region: "us-east-2", Type: "m5.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.096}, "m5.xlarge": {Region: "us-east-2", Type: "m5.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.192}, "m5.2xlarge": {Region: "us-east-2", Type: "m5.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.384}, "m5.4xlarge": {Region: "us-east-2", Type: "m5.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.768}, "m5.8xlarge": {Region: "us-east-2", Type: "m5.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.536}, "m5.12xlarge": {Region: "us-east-2", Type: "m5.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.304}, "m5.16xlarge": {Region: "us-east-2", Type: "m5.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.072}, "m5.24xlarge": {Region: "us-east-2", Type: "m5.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.608}, "m5.metal": {Region: "us-east-2", Type: "m5.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.608}, "m5a.large": {Region: "us-east-2", Type: "m5a.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.086}, "m5a.xlarge": {Region: "us-east-2", Type: "m5a.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.172}, "m5a.2xlarge": {Region: "us-east-2", Type: "m5a.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.344}, "m5a.4xlarge": {Region: "us-east-2", Type: "m5a.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.688}, "m5a.8xlarge": {Region: "us-east-2", Type: "m5a.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.376}, "m5a.12xlarge": {Region: "us-east-2", Type: "m5a.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.064}, "m5a.16xlarge": {Region: "us-east-2", Type: "m5a.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.752}, "m5a.24xlarge": {Region: "us-east-2", Type: "m5a.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.128}, "m5ad.large": {Region: "us-east-2", Type: "m5ad.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.103}, "m5ad.xlarge": {Region: "us-east-2", Type: "m5ad.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.206}, "m5ad.2xlarge": {Region: "us-east-2", Type: "m5ad.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.412}, "m5ad.4xlarge": {Region: "us-east-2", Type: "m5ad.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.824}, "m5ad.8xlarge": {Region: "us-east-2", Type: "m5ad.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.648}, "m5ad.12xlarge": {Region: "us-east-2", Type: "m5ad.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.472}, "m5ad.16xlarge": {Region: "us-east-2", Type: "m5ad.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.296}, "m5ad.24xlarge": {Region: "us-east-2", Type: "m5ad.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.944}, "m5d.large": {Region: "us-east-2", Type: "m5d.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.113}, "m5d.xlarge": {Region: "us-east-2", Type: "m5d.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.226}, "m5d.2xlarge": {Region: "us-east-2", Type: "m5d.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.452}, "m5d.4xlarge": {Region: "us-east-2", Type: "m5d.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.904}, "m5d.8xlarge": {Region: "us-east-2", Type: "m5d.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.808}, "m5d.12xlarge": {Region: "us-east-2", Type: "m5d.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.712}, "m5d.16xlarge": {Region: "us-east-2", Type: "m5d.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.616}, "m5d.24xlarge": {Region: "us-east-2", Type: "m5d.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.424}, "m5d.metal": {Region: "us-east-2", Type: "m5d.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.424}, "m5dn.large": {Region: "us-east-2", Type: "m5dn.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.136}, "m5dn.xlarge": {Region: "us-east-2", Type: "m5dn.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.272}, "m5dn.2xlarge": {Region: "us-east-2", Type: "m5dn.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.544}, "m5dn.4xlarge": {Region: "us-east-2", Type: "m5dn.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.088}, "m5dn.8xlarge": {Region: "us-east-2", Type: "m5dn.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.176}, "m5dn.12xlarge": {Region: "us-east-2", Type: "m5dn.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.264}, "m5dn.16xlarge": {Region: "us-east-2", Type: "m5dn.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.352}, "m5dn.24xlarge": {Region: "us-east-2", Type: "m5dn.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.528}, "m5dn.metal": {Region: "us-east-2", Type: "m5dn.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.528}, "m5n.large": {Region: "us-east-2", Type: "m5n.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.119}, "m5n.xlarge": {Region: "us-east-2", Type: "m5n.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.238}, "m5n.2xlarge": {Region: "us-east-2", Type: "m5n.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.476}, "m5n.4xlarge": {Region: "us-east-2", Type: "m5n.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.952}, "m5n.8xlarge": {Region: "us-east-2", Type: "m5n.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.904}, "m5n.12xlarge": {Region: "us-east-2", Type: "m5n.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.856}, "m5n.16xlarge": {Region: "us-east-2", Type: "m5n.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.808}, "m5n.24xlarge": {Region: "us-east-2", Type: "m5n.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.712}, "m5n.metal": {Region: "us-east-2", Type: "m5n.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.712}, "m5zn.large": {Region: "us-east-2", Type: "m5zn.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.1652}, "m5zn.xlarge": {Region: "us-east-2", Type: "m5zn.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.3303}, "m5zn.2xlarge": {Region: "us-east-2", Type: "m5zn.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.6607}, "m5zn.3xlarge": {Region: "us-east-2", Type: "m5zn.3xlarge", Memory: kresource.MustParse("49152Mi"), CPU: kresource.MustParse("12"), GPU: 0, Inf: 0, Price: 0.991}, "m5zn.6xlarge": {Region: "us-east-2", Type: "m5zn.6xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 0, Price: 1.982}, "m5zn.12xlarge": {Region: "us-east-2", Type: "m5zn.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.9641}, "m5zn.metal": {Region: "us-east-2", Type: "m5zn.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.9641}, "m6g.medium": {Region: "us-east-2", Type: "m6g.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0385}, "m6g.large": {Region: "us-east-2", Type: "m6g.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.077}, "m6g.xlarge": {Region: "us-east-2", Type: "m6g.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.154}, "m6g.2xlarge": {Region: "us-east-2", Type: "m6g.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.308}, "m6g.4xlarge": {Region: "us-east-2", Type: "m6g.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.616}, "m6g.8xlarge": {Region: "us-east-2", Type: "m6g.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.232}, "m6g.12xlarge": {Region: "us-east-2", Type: "m6g.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 1.848}, "m6g.16xlarge": {Region: "us-east-2", Type: "m6g.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.464}, "m6g.metal": {Region: "us-east-2", Type: "m6g.metal", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.464}, "m6gd.medium": {Region: "us-east-2", Type: "m6gd.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0452}, "m6gd.large": {Region: "us-east-2", Type: "m6gd.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0904}, "m6gd.xlarge": {Region: "us-east-2", Type: "m6gd.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1808}, "m6gd.2xlarge": {Region: "us-east-2", Type: "m6gd.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3616}, "m6gd.4xlarge": {Region: "us-east-2", Type: "m6gd.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.7232}, "m6gd.8xlarge": {Region: "us-east-2", Type: "m6gd.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.4464}, "m6gd.12xlarge": {Region: "us-east-2", Type: "m6gd.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.1696}, "m6gd.16xlarge": {Region: "us-east-2", Type: "m6gd.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.8928}, "m6gd.metal": {Region: "us-east-2", Type: "m6gd.metal", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.8928}, "m6i.large": {Region: "us-east-2", Type: "m6i.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.096}, "m6i.xlarge": {Region: "us-east-2", Type: "m6i.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.192}, "m6i.2xlarge": {Region: "us-east-2", Type: "m6i.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.384}, "m6i.4xlarge": {Region: "us-east-2", Type: "m6i.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.768}, "m6i.8xlarge": {Region: "us-east-2", Type: "m6i.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.536}, "m6i.12xlarge": {Region: "us-east-2", Type: "m6i.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.304}, "m6i.16xlarge": {Region: "us-east-2", Type: "m6i.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.072}, "m6i.24xlarge": {Region: "us-east-2", Type: "m6i.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.608}, "m6i.32xlarge": {Region: "us-east-2", Type: "m6i.32xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 6.144}, "m6i.metal": {Region: "us-east-2", Type: "m6i.metal", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 6.144}, "p2.xlarge": {Region: "us-east-2", Type: "p2.xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("4"), GPU: 1, Inf: 0, Price: 0.9}, "p2.8xlarge": {Region: "us-east-2", Type: "p2.8xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("32"), GPU: 8, Inf: 0, Price: 7.2}, "p2.16xlarge": {Region: "us-east-2", Type: "p2.16xlarge", Memory: kresource.MustParse("749568Mi"), CPU: kresource.MustParse("64"), GPU: 16, Inf: 0, Price: 14.4}, "p3.2xlarge": {Region: "us-east-2", Type: "p3.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 1, Inf: 0, Price: 3.06}, "p3.8xlarge": {Region: "us-east-2", Type: "p3.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 4, Inf: 0, Price: 12.24}, "p3.16xlarge": {Region: "us-east-2", Type: "p3.16xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("64"), GPU: 8, Inf: 0, Price: 24.48}, "p4d.24xlarge": {Region: "us-east-2", Type: "p4d.24xlarge", Memory: kresource.MustParse("1179648Mi"), CPU: kresource.MustParse("96"), GPU: 8, Inf: 0, Price: 32.7726}, "r3.large": {Region: "us-east-2", Type: "r3.large", Memory: kresource.MustParse("15616Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.166}, "r3.xlarge": {Region: "us-east-2", Type: "r3.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.332}, "r3.2xlarge": {Region: "us-east-2", Type: "r3.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.664}, "r3.4xlarge": {Region: "us-east-2", Type: "r3.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.328}, "r3.8xlarge": {Region: "us-east-2", Type: "r3.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.656}, "r4.large": {Region: "us-east-2", Type: "r4.large", Memory: kresource.MustParse("15616Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.133}, "r4.xlarge": {Region: "us-east-2", Type: "r4.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.266}, "r4.2xlarge": {Region: "us-east-2", Type: "r4.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.532}, "r4.4xlarge": {Region: "us-east-2", Type: "r4.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.064}, "r4.8xlarge": {Region: "us-east-2", Type: "r4.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.128}, "r4.16xlarge": {Region: "us-east-2", Type: "r4.16xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.256}, "r5.large": {Region: "us-east-2", Type: "r5.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.126}, "r5.xlarge": {Region: "us-east-2", Type: "r5.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.252}, "r5.2xlarge": {Region: "us-east-2", Type: "r5.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.504}, "r5.4xlarge": {Region: "us-east-2", Type: "r5.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.008}, "r5.8xlarge": {Region: "us-east-2", Type: "r5.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.016}, "r5.12xlarge": {Region: "us-east-2", Type: "r5.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.024}, "r5.16xlarge": {Region: "us-east-2", Type: "r5.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.032}, "r5.24xlarge": {Region: "us-east-2", Type: "r5.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.048}, "r5.metal": {Region: "us-east-2", Type: "r5.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.048}, "r5a.large": {Region: "us-east-2", Type: "r5a.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.113}, "r5a.xlarge": {Region: "us-east-2", Type: "r5a.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.226}, "r5a.2xlarge": {Region: "us-east-2", Type: "r5a.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.452}, "r5a.4xlarge": {Region: "us-east-2", Type: "r5a.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.904}, "r5a.8xlarge": {Region: "us-east-2", Type: "r5a.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.808}, "r5a.12xlarge": {Region: "us-east-2", Type: "r5a.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.712}, "r5a.16xlarge": {Region: "us-east-2", Type: "r5a.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.616}, "r5a.24xlarge": {Region: "us-east-2", Type: "r5a.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.424}, "r5ad.large": {Region: "us-east-2", Type: "r5ad.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.131}, "r5ad.xlarge": {Region: "us-east-2", Type: "r5ad.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.262}, "r5ad.2xlarge": {Region: "us-east-2", Type: "r5ad.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.524}, "r5ad.4xlarge": {Region: "us-east-2", Type: "r5ad.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.048}, "r5ad.8xlarge": {Region: "us-east-2", Type: "r5ad.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.096}, "r5ad.12xlarge": {Region: "us-east-2", Type: "r5ad.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.144}, "r5ad.16xlarge": {Region: "us-east-2", Type: "r5ad.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.192}, "r5ad.24xlarge": {Region: "us-east-2", Type: "r5ad.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.288}, "r5b.large": {Region: "us-east-2", Type: "r5b.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.149}, "r5b.xlarge": {Region: "us-east-2", Type: "r5b.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.298}, "r5b.2xlarge": {Region: "us-east-2", Type: "r5b.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.596}, "r5b.4xlarge": {Region: "us-east-2", Type: "r5b.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.192}, "r5b.8xlarge": {Region: "us-east-2", Type: "r5b.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.384}, "r5b.12xlarge": {Region: "us-east-2", Type: "r5b.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.576}, "r5b.16xlarge": {Region: "us-east-2", Type: "r5b.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.768}, "r5b.24xlarge": {Region: "us-east-2", Type: "r5b.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.152}, "r5b.metal": {Region: "us-east-2", Type: "r5b.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.152}, "r5d.large": {Region: "us-east-2", Type: "r5d.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.144}, "r5d.xlarge": {Region: "us-east-2", Type: "r5d.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.288}, "r5d.2xlarge": {Region: "us-east-2", Type: "r5d.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.576}, "r5d.4xlarge": {Region: "us-east-2", Type: "r5d.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.152}, "r5d.8xlarge": {Region: "us-east-2", Type: "r5d.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.304}, "r5d.12xlarge": {Region: "us-east-2", Type: "r5d.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.456}, "r5d.16xlarge": {Region: "us-east-2", Type: "r5d.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.608}, "r5d.24xlarge": {Region: "us-east-2", Type: "r5d.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.912}, "r5d.metal": {Region: "us-east-2", Type: "r5d.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.912}, "r5dn.large": {Region: "us-east-2", Type: "r5dn.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.167}, "r5dn.xlarge": {Region: "us-east-2", Type: "r5dn.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.334}, "r5dn.2xlarge": {Region: "us-east-2", Type: "r5dn.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.668}, "r5dn.4xlarge": {Region: "us-east-2", Type: "r5dn.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.336}, "r5dn.8xlarge": {Region: "us-east-2", Type: "r5dn.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.672}, "r5dn.12xlarge": {Region: "us-east-2", Type: "r5dn.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.008}, "r5dn.16xlarge": {Region: "us-east-2", Type: "r5dn.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.344}, "r5dn.24xlarge": {Region: "us-east-2", Type: "r5dn.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.016}, "r5dn.metal": {Region: "us-east-2", Type: "r5dn.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.016}, "r5n.large": {Region: "us-east-2", Type: "r5n.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.149}, "r5n.xlarge": {Region: "us-east-2", Type: "r5n.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.298}, "r5n.2xlarge": {Region: "us-east-2", Type: "r5n.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.596}, "r5n.4xlarge": {Region: "us-east-2", Type: "r5n.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.192}, "r5n.8xlarge": {Region: "us-east-2", Type: "r5n.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.384}, "r5n.12xlarge": {Region: "us-east-2", Type: "r5n.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.576}, "r5n.16xlarge": {Region: "us-east-2", Type: "r5n.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.768}, "r5n.24xlarge": {Region: "us-east-2", Type: "r5n.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.152}, "r5n.metal": {Region: "us-east-2", Type: "r5n.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.152}, "r6g.medium": {Region: "us-east-2", Type: "r6g.medium", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0504}, "r6g.large": {Region: "us-east-2", Type: "r6g.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.1008}, "r6g.xlarge": {Region: "us-east-2", Type: "r6g.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.2016}, "r6g.2xlarge": {Region: "us-east-2", Type: "r6g.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.4032}, "r6g.4xlarge": {Region: "us-east-2", Type: "r6g.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.8064}, "r6g.8xlarge": {Region: "us-east-2", Type: "r6g.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.6128}, "r6g.12xlarge": {Region: "us-east-2", Type: "r6g.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.4192}, "r6g.16xlarge": {Region: "us-east-2", Type: "r6g.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.2256}, "r6g.metal": {Region: "us-east-2", Type: "r6g.metal", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.2256}, "r6gd.medium": {Region: "us-east-2", Type: "r6gd.medium", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0576}, "r6gd.large": {Region: "us-east-2", Type: "r6gd.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.1152}, "r6gd.xlarge": {Region: "us-east-2", Type: "r6gd.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.2304}, "r6gd.2xlarge": {Region: "us-east-2", Type: "r6gd.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.4608}, "r6gd.4xlarge": {Region: "us-east-2", Type: "r6gd.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.9216}, "r6gd.8xlarge": {Region: "us-east-2", Type: "r6gd.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.8432}, "r6gd.12xlarge": {Region: "us-east-2", Type: "r6gd.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.7648}, "r6gd.16xlarge": {Region: "us-east-2", Type: "r6gd.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.6864}, "r6gd.metal": {Region: "us-east-2", Type: "r6gd.metal", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.6864}, "r6i.large": {Region: "us-east-2", Type: "r6i.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.126}, "r6i.xlarge": {Region: "us-east-2", Type: "r6i.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.252}, "r6i.2xlarge": {Region: "us-east-2", Type: "r6i.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.504}, "r6i.4xlarge": {Region: "us-east-2", Type: "r6i.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.008}, "r6i.8xlarge": {Region: "us-east-2", Type: "r6i.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.016}, "r6i.12xlarge": {Region: "us-east-2", Type: "r6i.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.024}, "r6i.16xlarge": {Region: "us-east-2", Type: "r6i.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.032}, "r6i.24xlarge": {Region: "us-east-2", Type: "r6i.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.048}, "r6i.32xlarge": {Region: "us-east-2", Type: "r6i.32xlarge", Memory: kresource.MustParse("1048576Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 8.064}, "r6i.metal": {Region: "us-east-2", Type: "r6i.metal", Memory: kresource.MustParse("1048576Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 8.064}, "t2.nano": {Region: "us-east-2", Type: "t2.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0058}, "t2.micro": {Region: "us-east-2", Type: "t2.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0116}, "t2.small": {Region: "us-east-2", Type: "t2.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.023}, "t2.medium": {Region: "us-east-2", Type: "t2.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0464}, "t2.large": {Region: "us-east-2", Type: "t2.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0928}, "t2.xlarge": {Region: "us-east-2", Type: "t2.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1856}, "t2.2xlarge": {Region: "us-east-2", Type: "t2.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3712}, "t3.nano": {Region: "us-east-2", Type: "t3.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0052}, "t3.micro": {Region: "us-east-2", Type: "t3.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0104}, "t3.small": {Region: "us-east-2", Type: "t3.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0208}, "t3.medium": {Region: "us-east-2", Type: "t3.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0416}, "t3.large": {Region: "us-east-2", Type: "t3.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0832}, "t3.xlarge": {Region: "us-east-2", Type: "t3.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1664}, "t3.2xlarge": {Region: "us-east-2", Type: "t3.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3328}, "t3a.nano": {Region: "us-east-2", Type: "t3a.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0047}, "t3a.micro": {Region: "us-east-2", Type: "t3a.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0094}, "t3a.small": {Region: "us-east-2", Type: "t3a.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0188}, "t3a.medium": {Region: "us-east-2", Type: "t3a.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0376}, "t3a.large": {Region: "us-east-2", Type: "t3a.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0752}, "t3a.xlarge": {Region: "us-east-2", Type: "t3a.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1504}, "t3a.2xlarge": {Region: "us-east-2", Type: "t3a.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3008}, "t4g.nano": {Region: "us-east-2", Type: "t4g.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0042}, "t4g.micro": {Region: "us-east-2", Type: "t4g.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0084}, "t4g.small": {Region: "us-east-2", Type: "t4g.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0168}, "t4g.medium": {Region: "us-east-2", Type: "t4g.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0336}, "t4g.large": {Region: "us-east-2", Type: "t4g.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0672}, "t4g.xlarge": {Region: "us-east-2", Type: "t4g.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1344}, "t4g.2xlarge": {Region: "us-east-2", Type: "t4g.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.2688}, "u-6tb1.56xlarge": {Region: "us-east-2", Type: "u-6tb1.56xlarge", Memory: kresource.MustParse("6291456Mi"), CPU: kresource.MustParse("224"), GPU: 0, Inf: 0, Price: 46.40391}, "u-6tb1.112xlarge": {Region: "us-east-2", Type: "u-6tb1.112xlarge", Memory: kresource.MustParse("6291456Mi"), CPU: kresource.MustParse("448"), GPU: 0, Inf: 0, Price: 54.6}, "x1.16xlarge": {Region: "us-east-2", Type: "x1.16xlarge", Memory: kresource.MustParse("999424Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 6.669}, "x1.32xlarge": {Region: "us-east-2", Type: "x1.32xlarge", Memory: kresource.MustParse("1998848Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 13.338}, "x1e.xlarge": {Region: "us-east-2", Type: "x1e.xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.834}, "x1e.2xlarge": {Region: "us-east-2", Type: "x1e.2xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.668}, "x1e.4xlarge": {Region: "us-east-2", Type: "x1e.4xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 3.336}, "x1e.8xlarge": {Region: "us-east-2", Type: "x1e.8xlarge", Memory: kresource.MustParse("999424Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 6.672}, "x1e.16xlarge": {Region: "us-east-2", Type: "x1e.16xlarge", Memory: kresource.MustParse("1998848Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 13.344}, "x1e.32xlarge": {Region: "us-east-2", Type: "x1e.32xlarge", Memory: kresource.MustParse("3997696Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 26.688}, "x2gd.medium": {Region: "us-east-2", Type: "x2gd.medium", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0835}, "x2gd.large": {Region: "us-east-2", Type: "x2gd.large", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.167}, "x2gd.xlarge": {Region: "us-east-2", Type: "x2gd.xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.334}, "x2gd.2xlarge": {Region: "us-east-2", Type: "x2gd.2xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.668}, "x2gd.4xlarge": {Region: "us-east-2", Type: "x2gd.4xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.336}, "x2gd.8xlarge": {Region: "us-east-2", Type: "x2gd.8xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.672}, "x2gd.12xlarge": {Region: "us-east-2", Type: "x2gd.12xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.008}, "x2gd.16xlarge": {Region: "us-east-2", Type: "x2gd.16xlarge", Memory: kresource.MustParse("1048576Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.344}, "x2gd.metal": {Region: "us-east-2", Type: "x2gd.metal", Memory: kresource.MustParse("1048576Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.344}, "x2iedn.xlarge": {Region: "us-east-2", Type: "x2iedn.xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.83363}, "x2iedn.2xlarge": {Region: "us-east-2", Type: "x2iedn.2xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.66725}, "x2iedn.4xlarge": {Region: "us-east-2", Type: "x2iedn.4xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 3.3345}, "x2iedn.8xlarge": {Region: "us-east-2", Type: "x2iedn.8xlarge", Memory: kresource.MustParse("1048576Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 6.669}, "x2iedn.16xlarge": {Region: "us-east-2", Type: "x2iedn.16xlarge", Memory: kresource.MustParse("2097152Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 13.338}, "x2iedn.24xlarge": {Region: "us-east-2", Type: "x2iedn.24xlarge", Memory: kresource.MustParse("3145728Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 20.007}, "x2iedn.32xlarge": {Region: "us-east-2", Type: "x2iedn.32xlarge", Memory: kresource.MustParse("4194304Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 26.676}, "z1d.large": {Region: "us-east-2", Type: "z1d.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.186}, "z1d.xlarge": {Region: "us-east-2", Type: "z1d.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.372}, "z1d.2xlarge": {Region: "us-east-2", Type: "z1d.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.744}, "z1d.3xlarge": {Region: "us-east-2", Type: "z1d.3xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("12"), GPU: 0, Inf: 0, Price: 1.116}, "z1d.6xlarge": {Region: "us-east-2", Type: "z1d.6xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 0, Price: 2.232}, "z1d.12xlarge": {Region: "us-east-2", Type: "z1d.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.464}, "z1d.metal": {Region: "us-east-2", Type: "z1d.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.464}, }, "us-gov-east-1": { "c5.large": {Region: "us-gov-east-1", Type: "c5.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.102}, "c5.xlarge": {Region: "us-gov-east-1", Type: "c5.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.204}, "c5.2xlarge": {Region: "us-gov-east-1", Type: "c5.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.408}, "c5.4xlarge": {Region: "us-gov-east-1", Type: "c5.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.816}, "c5.9xlarge": {Region: "us-gov-east-1", Type: "c5.9xlarge", Memory: kresource.MustParse("73728Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 1.836}, "c5.12xlarge": {Region: "us-gov-east-1", Type: "c5.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.448}, "c5.18xlarge": {Region: "us-gov-east-1", Type: "c5.18xlarge", Memory: kresource.MustParse("147456Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 3.672}, "c5.24xlarge": {Region: "us-gov-east-1", Type: "c5.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.896}, "c5.metal": {Region: "us-gov-east-1", Type: "c5.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.896}, "c5a.large": {Region: "us-gov-east-1", Type: "c5a.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.092}, "c5a.xlarge": {Region: "us-gov-east-1", Type: "c5a.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.184}, "c5a.2xlarge": {Region: "us-gov-east-1", Type: "c5a.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.368}, "c5a.4xlarge": {Region: "us-gov-east-1", Type: "c5a.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.736}, "c5a.8xlarge": {Region: "us-gov-east-1", Type: "c5a.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.472}, "c5a.12xlarge": {Region: "us-gov-east-1", Type: "c5a.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.208}, "c5a.16xlarge": {Region: "us-gov-east-1", Type: "c5a.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.944}, "c5a.24xlarge": {Region: "us-gov-east-1", Type: "c5a.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.416}, "c5d.large": {Region: "us-gov-east-1", Type: "c5d.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.116}, "c5d.xlarge": {Region: "us-gov-east-1", Type: "c5d.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.232}, "c5d.2xlarge": {Region: "us-gov-east-1", Type: "c5d.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.464}, "c5d.4xlarge": {Region: "us-gov-east-1", Type: "c5d.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.928}, "c5d.9xlarge": {Region: "us-gov-east-1", Type: "c5d.9xlarge", Memory: kresource.MustParse("73728Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 2.088}, "c5d.18xlarge": {Region: "us-gov-east-1", Type: "c5d.18xlarge", Memory: kresource.MustParse("147456Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 4.176}, "c5n.large": {Region: "us-gov-east-1", Type: "c5n.large", Memory: kresource.MustParse("5376Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.13}, "c5n.xlarge": {Region: "us-gov-east-1", Type: "c5n.xlarge", Memory: kresource.MustParse("10752Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.26}, "c5n.2xlarge": {Region: "us-gov-east-1", Type: "c5n.2xlarge", Memory: kresource.MustParse("21504Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.52}, "c5n.4xlarge": {Region: "us-gov-east-1", Type: "c5n.4xlarge", Memory: kresource.MustParse("43008Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.04}, "c5n.9xlarge": {Region: "us-gov-east-1", Type: "c5n.9xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 2.34}, "c5n.18xlarge": {Region: "us-gov-east-1", Type: "c5n.18xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 4.68}, "c5n.metal": {Region: "us-gov-east-1", Type: "c5n.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 4.68}, "c6g.medium": {Region: "us-gov-east-1", Type: "c6g.medium", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0408}, "c6g.large": {Region: "us-gov-east-1", Type: "c6g.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0816}, "c6g.xlarge": {Region: "us-gov-east-1", Type: "c6g.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1632}, "c6g.2xlarge": {Region: "us-gov-east-1", Type: "c6g.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3264}, "c6g.4xlarge": {Region: "us-gov-east-1", Type: "c6g.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.6528}, "c6g.8xlarge": {Region: "us-gov-east-1", Type: "c6g.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.3056}, "c6g.12xlarge": {Region: "us-gov-east-1", Type: "c6g.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 1.9584}, "c6g.16xlarge": {Region: "us-gov-east-1", Type: "c6g.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.6112}, "c6g.metal": {Region: "us-gov-east-1", Type: "c6g.metal", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.6112}, "d2.xlarge": {Region: "us-gov-east-1", Type: "d2.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.828}, "d2.2xlarge": {Region: "us-gov-east-1", Type: "d2.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.656}, "d2.4xlarge": {Region: "us-gov-east-1", Type: "d2.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 3.312}, "d2.8xlarge": {Region: "us-gov-east-1", Type: "d2.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 6.624}, "g4dn.xlarge": {Region: "us-gov-east-1", Type: "g4dn.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 1, Inf: 0, Price: 0.663}, "g4dn.2xlarge": {Region: "us-gov-east-1", Type: "g4dn.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 1, Inf: 0, Price: 0.948}, "g4dn.4xlarge": {Region: "us-gov-east-1", Type: "g4dn.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 1, Inf: 0, Price: 1.518}, "g4dn.8xlarge": {Region: "us-gov-east-1", Type: "g4dn.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 1, Inf: 0, Price: 2.743}, "g4dn.12xlarge": {Region: "us-gov-east-1", Type: "g4dn.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 4, Inf: 0, Price: 4.931}, "g4dn.16xlarge": {Region: "us-gov-east-1", Type: "g4dn.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 1, Inf: 0, Price: 5.486}, "i3.large": {Region: "us-gov-east-1", Type: "i3.large", Memory: kresource.MustParse("15616Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.188}, "i3.xlarge": {Region: "us-gov-east-1", Type: "i3.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.376}, "i3.2xlarge": {Region: "us-gov-east-1", Type: "i3.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.752}, "i3.4xlarge": {Region: "us-gov-east-1", Type: "i3.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.504}, "i3.8xlarge": {Region: "us-gov-east-1", Type: "i3.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 3.008}, "i3.16xlarge": {Region: "us-gov-east-1", Type: "i3.16xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 6.016}, "i3.metal": {Region: "us-gov-east-1", Type: "i3.metal", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 6.016}, "i3en.large": {Region: "us-gov-east-1", Type: "i3en.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.273}, "i3en.xlarge": {Region: "us-gov-east-1", Type: "i3en.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.546}, "i3en.2xlarge": {Region: "us-gov-east-1", Type: "i3en.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.092}, "i3en.3xlarge": {Region: "us-gov-east-1", Type: "i3en.3xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("12"), GPU: 0, Inf: 0, Price: 1.638}, "i3en.6xlarge": {Region: "us-gov-east-1", Type: "i3en.6xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 0, Price: 3.276}, "i3en.12xlarge": {Region: "us-gov-east-1", Type: "i3en.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 6.552}, "i3en.24xlarge": {Region: "us-gov-east-1", Type: "i3en.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 13.104}, "i3en.metal": {Region: "us-gov-east-1", Type: "i3en.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 13.104}, "inf1.xlarge": {Region: "us-gov-east-1", Type: "inf1.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 1, Price: 0.288}, "inf1.2xlarge": {Region: "us-gov-east-1", Type: "inf1.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 1, Price: 0.456}, "inf1.6xlarge": {Region: "us-gov-east-1", Type: "inf1.6xlarge", Memory: kresource.MustParse("49152Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 4, Price: 1.488}, "inf1.24xlarge": {Region: "us-gov-east-1", Type: "inf1.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 16, Price: 5.953}, "m5.large": {Region: "us-gov-east-1", Type: "m5.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.121}, "m5.xlarge": {Region: "us-gov-east-1", Type: "m5.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.242}, "m5.2xlarge": {Region: "us-gov-east-1", Type: "m5.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.484}, "m5.4xlarge": {Region: "us-gov-east-1", Type: "m5.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.968}, "m5.8xlarge": {Region: "us-gov-east-1", Type: "m5.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.936}, "m5.12xlarge": {Region: "us-gov-east-1", Type: "m5.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.904}, "m5.16xlarge": {Region: "us-gov-east-1", Type: "m5.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.872}, "m5.24xlarge": {Region: "us-gov-east-1", Type: "m5.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.808}, "m5.metal": {Region: "us-gov-east-1", Type: "m5.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.808}, "m5a.large": {Region: "us-gov-east-1", Type: "m5a.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.109}, "m5a.xlarge": {Region: "us-gov-east-1", Type: "m5a.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.218}, "m5a.2xlarge": {Region: "us-gov-east-1", Type: "m5a.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.436}, "m5a.4xlarge": {Region: "us-gov-east-1", Type: "m5a.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.872}, "m5a.8xlarge": {Region: "us-gov-east-1", Type: "m5a.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.744}, "m5a.12xlarge": {Region: "us-gov-east-1", Type: "m5a.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.616}, "m5a.16xlarge": {Region: "us-gov-east-1", Type: "m5a.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.488}, "m5a.24xlarge": {Region: "us-gov-east-1", Type: "m5a.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.232}, "m5d.large": {Region: "us-gov-east-1", Type: "m5d.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.143}, "m5d.xlarge": {Region: "us-gov-east-1", Type: "m5d.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.286}, "m5d.2xlarge": {Region: "us-gov-east-1", Type: "m5d.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.572}, "m5d.4xlarge": {Region: "us-gov-east-1", Type: "m5d.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.144}, "m5d.8xlarge": {Region: "us-gov-east-1", Type: "m5d.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.288}, "m5d.12xlarge": {Region: "us-gov-east-1", Type: "m5d.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.432}, "m5d.16xlarge": {Region: "us-gov-east-1", Type: "m5d.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.576}, "m5d.24xlarge": {Region: "us-gov-east-1", Type: "m5d.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.864}, "m5d.metal": {Region: "us-gov-east-1", Type: "m5d.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.864}, "m5dn.large": {Region: "us-gov-east-1", Type: "m5dn.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.171}, "m5dn.xlarge": {Region: "us-gov-east-1", Type: "m5dn.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.342}, "m5dn.2xlarge": {Region: "us-gov-east-1", Type: "m5dn.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.684}, "m5dn.4xlarge": {Region: "us-gov-east-1", Type: "m5dn.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.368}, "m5dn.8xlarge": {Region: "us-gov-east-1", Type: "m5dn.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.736}, "m5dn.12xlarge": {Region: "us-gov-east-1", Type: "m5dn.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.104}, "m5dn.16xlarge": {Region: "us-gov-east-1", Type: "m5dn.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.472}, "m5dn.24xlarge": {Region: "us-gov-east-1", Type: "m5dn.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.208}, "m5dn.metal": {Region: "us-gov-east-1", Type: "m5dn.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.208}, "m5n.large": {Region: "us-gov-east-1", Type: "m5n.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.149}, "m5n.xlarge": {Region: "us-gov-east-1", Type: "m5n.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.298}, "m5n.2xlarge": {Region: "us-gov-east-1", Type: "m5n.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.596}, "m5n.4xlarge": {Region: "us-gov-east-1", Type: "m5n.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.192}, "m5n.8xlarge": {Region: "us-gov-east-1", Type: "m5n.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.384}, "m5n.12xlarge": {Region: "us-gov-east-1", Type: "m5n.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.576}, "m5n.16xlarge": {Region: "us-gov-east-1", Type: "m5n.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.768}, "m5n.24xlarge": {Region: "us-gov-east-1", Type: "m5n.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.152}, "m5n.metal": {Region: "us-gov-east-1", Type: "m5n.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.152}, "m6g.medium": {Region: "us-gov-east-1", Type: "m6g.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0484}, "m6g.large": {Region: "us-gov-east-1", Type: "m6g.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0968}, "m6g.xlarge": {Region: "us-gov-east-1", Type: "m6g.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1936}, "m6g.2xlarge": {Region: "us-gov-east-1", Type: "m6g.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3872}, "m6g.4xlarge": {Region: "us-gov-east-1", Type: "m6g.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.7744}, "m6g.8xlarge": {Region: "us-gov-east-1", Type: "m6g.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.5488}, "m6g.12xlarge": {Region: "us-gov-east-1", Type: "m6g.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.3232}, "m6g.16xlarge": {Region: "us-gov-east-1", Type: "m6g.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.0976}, "m6g.metal": {Region: "us-gov-east-1", Type: "m6g.metal", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.0976}, "p3dn.24xlarge": {Region: "us-gov-east-1", Type: "p3dn.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 8, Inf: 0, Price: 37.454}, "r5.large": {Region: "us-gov-east-1", Type: "r5.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.151}, "r5.xlarge": {Region: "us-gov-east-1", Type: "r5.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.302}, "r5.2xlarge": {Region: "us-gov-east-1", Type: "r5.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.604}, "r5.4xlarge": {Region: "us-gov-east-1", Type: "r5.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.208}, "r5.8xlarge": {Region: "us-gov-east-1", Type: "r5.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.416}, "r5.12xlarge": {Region: "us-gov-east-1", Type: "r5.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.624}, "r5.16xlarge": {Region: "us-gov-east-1", Type: "r5.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.832}, "r5.24xlarge": {Region: "us-gov-east-1", Type: "r5.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.248}, "r5.metal": {Region: "us-gov-east-1", Type: "r5.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.248}, "r5a.large": {Region: "us-gov-east-1", Type: "r5a.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.136}, "r5a.xlarge": {Region: "us-gov-east-1", Type: "r5a.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.272}, "r5a.2xlarge": {Region: "us-gov-east-1", Type: "r5a.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.544}, "r5a.4xlarge": {Region: "us-gov-east-1", Type: "r5a.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.088}, "r5a.8xlarge": {Region: "us-gov-east-1", Type: "r5a.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.176}, "r5a.12xlarge": {Region: "us-gov-east-1", Type: "r5a.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.264}, "r5a.16xlarge": {Region: "us-gov-east-1", Type: "r5a.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.352}, "r5a.24xlarge": {Region: "us-gov-east-1", Type: "r5a.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.528}, "r5d.large": {Region: "us-gov-east-1", Type: "r5d.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.173}, "r5d.xlarge": {Region: "us-gov-east-1", Type: "r5d.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.346}, "r5d.2xlarge": {Region: "us-gov-east-1", Type: "r5d.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.692}, "r5d.4xlarge": {Region: "us-gov-east-1", Type: "r5d.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.384}, "r5d.8xlarge": {Region: "us-gov-east-1", Type: "r5d.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.768}, "r5d.12xlarge": {Region: "us-gov-east-1", Type: "r5d.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.152}, "r5d.16xlarge": {Region: "us-gov-east-1", Type: "r5d.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.536}, "r5d.24xlarge": {Region: "us-gov-east-1", Type: "r5d.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.304}, "r5d.metal": {Region: "us-gov-east-1", Type: "r5d.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.304}, "r5dn.large": {Region: "us-gov-east-1", Type: "r5dn.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.201}, "r5dn.xlarge": {Region: "us-gov-east-1", Type: "r5dn.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.402}, "r5dn.2xlarge": {Region: "us-gov-east-1", Type: "r5dn.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.804}, "r5dn.4xlarge": {Region: "us-gov-east-1", Type: "r5dn.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.608}, "r5dn.8xlarge": {Region: "us-gov-east-1", Type: "r5dn.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 3.216}, "r5dn.12xlarge": {Region: "us-gov-east-1", Type: "r5dn.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.824}, "r5dn.16xlarge": {Region: "us-gov-east-1", Type: "r5dn.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 6.432}, "r5dn.24xlarge": {Region: "us-gov-east-1", Type: "r5dn.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 9.648}, "r5dn.metal": {Region: "us-gov-east-1", Type: "r5dn.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 9.648}, "r5n.large": {Region: "us-gov-east-1", Type: "r5n.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.179}, "r5n.xlarge": {Region: "us-gov-east-1", Type: "r5n.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.358}, "r5n.2xlarge": {Region: "us-gov-east-1", Type: "r5n.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.716}, "r5n.4xlarge": {Region: "us-gov-east-1", Type: "r5n.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.432}, "r5n.8xlarge": {Region: "us-gov-east-1", Type: "r5n.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.864}, "r5n.12xlarge": {Region: "us-gov-east-1", Type: "r5n.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.296}, "r5n.16xlarge": {Region: "us-gov-east-1", Type: "r5n.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.728}, "r5n.24xlarge": {Region: "us-gov-east-1", Type: "r5n.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.592}, "r5n.metal": {Region: "us-gov-east-1", Type: "r5n.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.592}, "r6g.medium": {Region: "us-gov-east-1", Type: "r6g.medium", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0604}, "r6g.large": {Region: "us-gov-east-1", Type: "r6g.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.1208}, "r6g.xlarge": {Region: "us-gov-east-1", Type: "r6g.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.2416}, "r6g.2xlarge": {Region: "us-gov-east-1", Type: "r6g.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.4832}, "r6g.4xlarge": {Region: "us-gov-east-1", Type: "r6g.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.9664}, "r6g.8xlarge": {Region: "us-gov-east-1", Type: "r6g.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.9328}, "r6g.12xlarge": {Region: "us-gov-east-1", Type: "r6g.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.8992}, "r6g.16xlarge": {Region: "us-gov-east-1", Type: "r6g.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.8656}, "r6g.metal": {Region: "us-gov-east-1", Type: "r6g.metal", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.8656}, "t3.nano": {Region: "us-gov-east-1", Type: "t3.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0061}, "t3.micro": {Region: "us-gov-east-1", Type: "t3.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0122}, "t3.small": {Region: "us-gov-east-1", Type: "t3.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0244}, "t3.medium": {Region: "us-gov-east-1", Type: "t3.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0488}, "t3.large": {Region: "us-gov-east-1", Type: "t3.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0976}, "t3.xlarge": {Region: "us-gov-east-1", Type: "t3.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1952}, "t3.2xlarge": {Region: "us-gov-east-1", Type: "t3.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3904}, "t3a.nano": {Region: "us-gov-east-1", Type: "t3a.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0055}, "t3a.micro": {Region: "us-gov-east-1", Type: "t3a.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.011}, "t3a.small": {Region: "us-gov-east-1", Type: "t3a.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.022}, "t3a.medium": {Region: "us-gov-east-1", Type: "t3a.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0439}, "t3a.large": {Region: "us-gov-east-1", Type: "t3a.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0878}, "t3a.xlarge": {Region: "us-gov-east-1", Type: "t3a.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1757}, "t3a.2xlarge": {Region: "us-gov-east-1", Type: "t3a.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3514}, "t4g.nano": {Region: "us-gov-east-1", Type: "t4g.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0049}, "t4g.micro": {Region: "us-gov-east-1", Type: "t4g.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0098}, "t4g.small": {Region: "us-gov-east-1", Type: "t4g.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0196}, "t4g.medium": {Region: "us-gov-east-1", Type: "t4g.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0392}, "t4g.large": {Region: "us-gov-east-1", Type: "t4g.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0784}, "t4g.xlarge": {Region: "us-gov-east-1", Type: "t4g.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1568}, "t4g.2xlarge": {Region: "us-gov-east-1", Type: "t4g.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3136}, "u-6tb1.56xlarge": {Region: "us-gov-east-1", Type: "u-6tb1.56xlarge", Memory: kresource.MustParse("6291456Mi"), CPU: kresource.MustParse("224"), GPU: 0, Inf: 0, Price: 55.61075}, "u-6tb1.112xlarge": {Region: "us-gov-east-1", Type: "u-6tb1.112xlarge", Memory: kresource.MustParse("6291456Mi"), CPU: kresource.MustParse("448"), GPU: 0, Inf: 0, Price: 65.433}, "x1.16xlarge": {Region: "us-gov-east-1", Type: "x1.16xlarge", Memory: kresource.MustParse("999424Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 8.003}, "x1.32xlarge": {Region: "us-gov-east-1", Type: "x1.32xlarge", Memory: kresource.MustParse("1998848Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 16.006}, "x1e.xlarge": {Region: "us-gov-east-1", Type: "x1e.xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 1.0}, "x1e.2xlarge": {Region: "us-gov-east-1", Type: "x1e.2xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 2.0}, "x1e.4xlarge": {Region: "us-gov-east-1", Type: "x1e.4xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 4.0}, "x1e.8xlarge": {Region: "us-gov-east-1", Type: "x1e.8xlarge", Memory: kresource.MustParse("999424Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 8.0}, "x1e.16xlarge": {Region: "us-gov-east-1", Type: "x1e.16xlarge", Memory: kresource.MustParse("1998848Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 16.0}, "x1e.32xlarge": {Region: "us-gov-east-1", Type: "x1e.32xlarge", Memory: kresource.MustParse("3997696Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 32.0}, }, "us-gov-west-1": { "c1.medium": {Region: "us-gov-west-1", Type: "c1.medium", Memory: kresource.MustParse("1740Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.157}, "c1.xlarge": {Region: "us-gov-west-1", Type: "c1.xlarge", Memory: kresource.MustParse("7168Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.628}, "c3.large": {Region: "us-gov-west-1", Type: "c3.large", Memory: kresource.MustParse("3840Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.126}, "c3.xlarge": {Region: "us-gov-west-1", Type: "c3.xlarge", Memory: kresource.MustParse("7680Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.252}, "c3.2xlarge": {Region: "us-gov-west-1", Type: "c3.2xlarge", Memory: kresource.MustParse("15360Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.504}, "c3.4xlarge": {Region: "us-gov-west-1", Type: "c3.4xlarge", Memory: kresource.MustParse("30720Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.008}, "c3.8xlarge": {Region: "us-gov-west-1", Type: "c3.8xlarge", Memory: kresource.MustParse("61440Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.016}, "c4.large": {Region: "us-gov-west-1", Type: "c4.large", Memory: kresource.MustParse("3840Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.12}, "c4.xlarge": {Region: "us-gov-west-1", Type: "c4.xlarge", Memory: kresource.MustParse("7680Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.239}, "c4.2xlarge": {Region: "us-gov-west-1", Type: "c4.2xlarge", Memory: kresource.MustParse("15360Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.479}, "c4.4xlarge": {Region: "us-gov-west-1", Type: "c4.4xlarge", Memory: kresource.MustParse("30720Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.958}, "c4.8xlarge": {Region: "us-gov-west-1", Type: "c4.8xlarge", Memory: kresource.MustParse("61440Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 1.915}, "c5.large": {Region: "us-gov-west-1", Type: "c5.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.102}, "c5.xlarge": {Region: "us-gov-west-1", Type: "c5.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.204}, "c5.2xlarge": {Region: "us-gov-west-1", Type: "c5.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.408}, "c5.4xlarge": {Region: "us-gov-west-1", Type: "c5.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.816}, "c5.9xlarge": {Region: "us-gov-west-1", Type: "c5.9xlarge", Memory: kresource.MustParse("73728Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 1.836}, "c5.12xlarge": {Region: "us-gov-west-1", Type: "c5.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.448}, "c5.18xlarge": {Region: "us-gov-west-1", Type: "c5.18xlarge", Memory: kresource.MustParse("147456Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 3.672}, "c5.24xlarge": {Region: "us-gov-west-1", Type: "c5.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.896}, "c5.metal": {Region: "us-gov-west-1", Type: "c5.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.896}, "c5a.large": {Region: "us-gov-west-1", Type: "c5a.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.092}, "c5a.xlarge": {Region: "us-gov-west-1", Type: "c5a.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.184}, "c5a.2xlarge": {Region: "us-gov-west-1", Type: "c5a.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.368}, "c5a.4xlarge": {Region: "us-gov-west-1", Type: "c5a.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.736}, "c5a.8xlarge": {Region: "us-gov-west-1", Type: "c5a.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.472}, "c5a.12xlarge": {Region: "us-gov-west-1", Type: "c5a.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.208}, "c5a.16xlarge": {Region: "us-gov-west-1", Type: "c5a.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.944}, "c5a.24xlarge": {Region: "us-gov-west-1", Type: "c5a.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.416}, "c5d.large": {Region: "us-gov-west-1", Type: "c5d.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.116}, "c5d.xlarge": {Region: "us-gov-west-1", Type: "c5d.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.232}, "c5d.2xlarge": {Region: "us-gov-west-1", Type: "c5d.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.464}, "c5d.4xlarge": {Region: "us-gov-west-1", Type: "c5d.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.928}, "c5d.9xlarge": {Region: "us-gov-west-1", Type: "c5d.9xlarge", Memory: kresource.MustParse("73728Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 2.088}, "c5d.12xlarge": {Region: "us-gov-west-1", Type: "c5d.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.784}, "c5d.18xlarge": {Region: "us-gov-west-1", Type: "c5d.18xlarge", Memory: kresource.MustParse("147456Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 4.176}, "c5d.24xlarge": {Region: "us-gov-west-1", Type: "c5d.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.568}, "c5d.metal": {Region: "us-gov-west-1", Type: "c5d.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.568}, "c5n.large": {Region: "us-gov-west-1", Type: "c5n.large", Memory: kresource.MustParse("5376Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.13}, "c5n.xlarge": {Region: "us-gov-west-1", Type: "c5n.xlarge", Memory: kresource.MustParse("10752Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.26}, "c5n.2xlarge": {Region: "us-gov-west-1", Type: "c5n.2xlarge", Memory: kresource.MustParse("21504Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.52}, "c5n.4xlarge": {Region: "us-gov-west-1", Type: "c5n.4xlarge", Memory: kresource.MustParse("43008Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.04}, "c5n.9xlarge": {Region: "us-gov-west-1", Type: "c5n.9xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 2.34}, "c5n.18xlarge": {Region: "us-gov-west-1", Type: "c5n.18xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 4.68}, "c5n.metal": {Region: "us-gov-west-1", Type: "c5n.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 4.68}, "c6g.medium": {Region: "us-gov-west-1", Type: "c6g.medium", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0408}, "c6g.large": {Region: "us-gov-west-1", Type: "c6g.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0816}, "c6g.xlarge": {Region: "us-gov-west-1", Type: "c6g.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1632}, "c6g.2xlarge": {Region: "us-gov-west-1", Type: "c6g.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3264}, "c6g.4xlarge": {Region: "us-gov-west-1", Type: "c6g.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.6528}, "c6g.8xlarge": {Region: "us-gov-west-1", Type: "c6g.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.3056}, "c6g.12xlarge": {Region: "us-gov-west-1", Type: "c6g.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 1.9584}, "c6g.16xlarge": {Region: "us-gov-west-1", Type: "c6g.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.6112}, "c6g.metal": {Region: "us-gov-west-1", Type: "c6g.metal", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.6112}, "cc2.8xlarge": {Region: "us-gov-west-1", Type: "cc2.8xlarge", Memory: kresource.MustParse("61952Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.25}, "d2.xlarge": {Region: "us-gov-west-1", Type: "d2.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.828}, "d2.2xlarge": {Region: "us-gov-west-1", Type: "d2.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.656}, "d2.4xlarge": {Region: "us-gov-west-1", Type: "d2.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 3.312}, "d2.8xlarge": {Region: "us-gov-west-1", Type: "d2.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 6.624}, "d3.xlarge": {Region: "us-gov-west-1", Type: "d3.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.598}, "d3.2xlarge": {Region: "us-gov-west-1", Type: "d3.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.197}, "d3.4xlarge": {Region: "us-gov-west-1", Type: "d3.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 2.394}, "d3.8xlarge": {Region: "us-gov-west-1", Type: "d3.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 4.78776}, "f1.2xlarge": {Region: "us-gov-west-1", Type: "f1.2xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.98}, "f1.4xlarge": {Region: "us-gov-west-1", Type: "f1.4xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 3.96}, "f1.16xlarge": {Region: "us-gov-west-1", Type: "f1.16xlarge", Memory: kresource.MustParse("999424Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 15.84}, "g3.4xlarge": {Region: "us-gov-west-1", Type: "g3.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 1, Inf: 0, Price: 1.32}, "g3.8xlarge": {Region: "us-gov-west-1", Type: "g3.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 2, Inf: 0, Price: 2.64}, "g3.16xlarge": {Region: "us-gov-west-1", Type: "g3.16xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("64"), GPU: 4, Inf: 0, Price: 5.28}, "g3s.xlarge": {Region: "us-gov-west-1", Type: "g3s.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 1, Inf: 0, Price: 0.868}, "g4dn.xlarge": {Region: "us-gov-west-1", Type: "g4dn.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 1, Inf: 0, Price: 0.663}, "g4dn.2xlarge": {Region: "us-gov-west-1", Type: "g4dn.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 1, Inf: 0, Price: 0.948}, "g4dn.4xlarge": {Region: "us-gov-west-1", Type: "g4dn.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 1, Inf: 0, Price: 1.518}, "g4dn.8xlarge": {Region: "us-gov-west-1", Type: "g4dn.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 1, Inf: 0, Price: 2.743}, "g4dn.12xlarge": {Region: "us-gov-west-1", Type: "g4dn.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 4, Inf: 0, Price: 4.931}, "g4dn.16xlarge": {Region: "us-gov-west-1", Type: "g4dn.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 1, Inf: 0, Price: 5.486}, "g4dn.metal": {Region: "us-gov-west-1", Type: "g4dn.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 8, Inf: 0, Price: 9.862}, "hpc6a.48xlarge": {Region: "us-gov-west-1", Type: "hpc6a.48xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 3.467}, "hs1.8xlarge": {Region: "us-gov-west-1", Type: "hs1.8xlarge", Memory: kresource.MustParse("119808Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 5.52}, "i2.xlarge": {Region: "us-gov-west-1", Type: "i2.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 1.023}, "i2.2xlarge": {Region: "us-gov-west-1", Type: "i2.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 2.046}, "i2.4xlarge": {Region: "us-gov-west-1", Type: "i2.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 4.092}, "i2.8xlarge": {Region: "us-gov-west-1", Type: "i2.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 8.184}, "i3.large": {Region: "us-gov-west-1", Type: "i3.large", Memory: kresource.MustParse("15616Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.188}, "i3.xlarge": {Region: "us-gov-west-1", Type: "i3.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.376}, "i3.2xlarge": {Region: "us-gov-west-1", Type: "i3.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.752}, "i3.4xlarge": {Region: "us-gov-west-1", Type: "i3.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.504}, "i3.8xlarge": {Region: "us-gov-west-1", Type: "i3.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 3.008}, "i3.16xlarge": {Region: "us-gov-west-1", Type: "i3.16xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 6.016}, "i3.metal": {Region: "us-gov-west-1", Type: "i3.metal", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 6.016}, "i3en.large": {Region: "us-gov-west-1", Type: "i3en.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.273}, "i3en.xlarge": {Region: "us-gov-west-1", Type: "i3en.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.546}, "i3en.2xlarge": {Region: "us-gov-west-1", Type: "i3en.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.092}, "i3en.3xlarge": {Region: "us-gov-west-1", Type: "i3en.3xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("12"), GPU: 0, Inf: 0, Price: 1.638}, "i3en.6xlarge": {Region: "us-gov-west-1", Type: "i3en.6xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 0, Price: 3.276}, "i3en.12xlarge": {Region: "us-gov-west-1", Type: "i3en.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 6.552}, "i3en.24xlarge": {Region: "us-gov-west-1", Type: "i3en.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 13.104}, "i3en.metal": {Region: "us-gov-west-1", Type: "i3en.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 13.104}, "i3p.16xlarge": {Region: "us-gov-west-1", Type: "i3p.16xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 6.016}, "inf1.xlarge": {Region: "us-gov-west-1", Type: "inf1.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 1, Price: 0.288}, "inf1.2xlarge": {Region: "us-gov-west-1", Type: "inf1.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 1, Price: 0.456}, "inf1.6xlarge": {Region: "us-gov-west-1", Type: "inf1.6xlarge", Memory: kresource.MustParse("49152Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 4, Price: 1.488}, "inf1.24xlarge": {Region: "us-gov-west-1", Type: "inf1.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 16, Price: 5.953}, "m1.small": {Region: "us-gov-west-1", Type: "m1.small", Memory: kresource.MustParse("1740Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.053}, "m1.medium": {Region: "us-gov-west-1", Type: "m1.medium", Memory: kresource.MustParse("3840Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.106}, "m1.large": {Region: "us-gov-west-1", Type: "m1.large", Memory: kresource.MustParse("7680Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.211}, "m1.xlarge": {Region: "us-gov-west-1", Type: "m1.xlarge", Memory: kresource.MustParse("15360Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.423}, "m2.xlarge": {Region: "us-gov-west-1", Type: "m2.xlarge", Memory: kresource.MustParse("17510Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.293}, "m2.2xlarge": {Region: "us-gov-west-1", Type: "m2.2xlarge", Memory: kresource.MustParse("35020Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.586}, "m2.4xlarge": {Region: "us-gov-west-1", Type: "m2.4xlarge", Memory: kresource.MustParse("70041Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.171}, "m3.medium": {Region: "us-gov-west-1", Type: "m3.medium", Memory: kresource.MustParse("3840Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.084}, "m3.large": {Region: "us-gov-west-1", Type: "m3.large", Memory: kresource.MustParse("7680Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.168}, "m3.xlarge": {Region: "us-gov-west-1", Type: "m3.xlarge", Memory: kresource.MustParse("15360Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.336}, "m3.2xlarge": {Region: "us-gov-west-1", Type: "m3.2xlarge", Memory: kresource.MustParse("30720Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.672}, "m4.large": {Region: "us-gov-west-1", Type: "m4.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.126}, "m4.xlarge": {Region: "us-gov-west-1", Type: "m4.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.252}, "m4.2xlarge": {Region: "us-gov-west-1", Type: "m4.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.504}, "m4.4xlarge": {Region: "us-gov-west-1", Type: "m4.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.008}, "m4.10xlarge": {Region: "us-gov-west-1", Type: "m4.10xlarge", Memory: kresource.MustParse("163840Mi"), CPU: kresource.MustParse("40"), GPU: 0, Inf: 0, Price: 2.52}, "m4.16xlarge": {Region: "us-gov-west-1", Type: "m4.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.032}, "m5.large": {Region: "us-gov-west-1", Type: "m5.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.121}, "m5.xlarge": {Region: "us-gov-west-1", Type: "m5.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.242}, "m5.2xlarge": {Region: "us-gov-west-1", Type: "m5.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.484}, "m5.4xlarge": {Region: "us-gov-west-1", Type: "m5.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.968}, "m5.8xlarge": {Region: "us-gov-west-1", Type: "m5.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.936}, "m5.12xlarge": {Region: "us-gov-west-1", Type: "m5.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.904}, "m5.16xlarge": {Region: "us-gov-west-1", Type: "m5.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.872}, "m5.24xlarge": {Region: "us-gov-west-1", Type: "m5.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.808}, "m5.metal": {Region: "us-gov-west-1", Type: "m5.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.808}, "m5a.large": {Region: "us-gov-west-1", Type: "m5a.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.109}, "m5a.xlarge": {Region: "us-gov-west-1", Type: "m5a.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.218}, "m5a.2xlarge": {Region: "us-gov-west-1", Type: "m5a.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.436}, "m5a.4xlarge": {Region: "us-gov-west-1", Type: "m5a.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.872}, "m5a.8xlarge": {Region: "us-gov-west-1", Type: "m5a.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.744}, "m5a.12xlarge": {Region: "us-gov-west-1", Type: "m5a.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.616}, "m5a.16xlarge": {Region: "us-gov-west-1", Type: "m5a.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.488}, "m5a.24xlarge": {Region: "us-gov-west-1", Type: "m5a.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.232}, "m5ad.large": {Region: "us-gov-west-1", Type: "m5ad.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.131}, "m5ad.xlarge": {Region: "us-gov-west-1", Type: "m5ad.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.262}, "m5ad.2xlarge": {Region: "us-gov-west-1", Type: "m5ad.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.524}, "m5ad.4xlarge": {Region: "us-gov-west-1", Type: "m5ad.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.048}, "m5ad.8xlarge": {Region: "us-gov-west-1", Type: "m5ad.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.096}, "m5ad.12xlarge": {Region: "us-gov-west-1", Type: "m5ad.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.144}, "m5ad.16xlarge": {Region: "us-gov-west-1", Type: "m5ad.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.192}, "m5ad.24xlarge": {Region: "us-gov-west-1", Type: "m5ad.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.288}, "m5d.large": {Region: "us-gov-west-1", Type: "m5d.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.143}, "m5d.xlarge": {Region: "us-gov-west-1", Type: "m5d.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.286}, "m5d.2xlarge": {Region: "us-gov-west-1", Type: "m5d.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.572}, "m5d.4xlarge": {Region: "us-gov-west-1", Type: "m5d.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.144}, "m5d.8xlarge": {Region: "us-gov-west-1", Type: "m5d.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.288}, "m5d.12xlarge": {Region: "us-gov-west-1", Type: "m5d.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.432}, "m5d.16xlarge": {Region: "us-gov-west-1", Type: "m5d.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.576}, "m5d.24xlarge": {Region: "us-gov-west-1", Type: "m5d.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.864}, "m5d.metal": {Region: "us-gov-west-1", Type: "m5d.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.864}, "m5dn.large": {Region: "us-gov-west-1", Type: "m5dn.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.171}, "m5dn.xlarge": {Region: "us-gov-west-1", Type: "m5dn.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.342}, "m5dn.2xlarge": {Region: "us-gov-west-1", Type: "m5dn.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.684}, "m5dn.4xlarge": {Region: "us-gov-west-1", Type: "m5dn.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.368}, "m5dn.8xlarge": {Region: "us-gov-west-1", Type: "m5dn.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.736}, "m5dn.12xlarge": {Region: "us-gov-west-1", Type: "m5dn.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.104}, "m5dn.16xlarge": {Region: "us-gov-west-1", Type: "m5dn.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.472}, "m5dn.24xlarge": {Region: "us-gov-west-1", Type: "m5dn.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.208}, "m5dn.metal": {Region: "us-gov-west-1", Type: "m5dn.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.208}, "m5n.large": {Region: "us-gov-west-1", Type: "m5n.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.149}, "m5n.xlarge": {Region: "us-gov-west-1", Type: "m5n.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.298}, "m5n.2xlarge": {Region: "us-gov-west-1", Type: "m5n.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.596}, "m5n.4xlarge": {Region: "us-gov-west-1", Type: "m5n.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.192}, "m5n.8xlarge": {Region: "us-gov-west-1", Type: "m5n.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.384}, "m5n.12xlarge": {Region: "us-gov-west-1", Type: "m5n.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.576}, "m5n.16xlarge": {Region: "us-gov-west-1", Type: "m5n.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.768}, "m5n.24xlarge": {Region: "us-gov-west-1", Type: "m5n.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.152}, "m5n.metal": {Region: "us-gov-west-1", Type: "m5n.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.152}, "m6g.medium": {Region: "us-gov-west-1", Type: "m6g.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0484}, "m6g.large": {Region: "us-gov-west-1", Type: "m6g.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0968}, "m6g.xlarge": {Region: "us-gov-west-1", Type: "m6g.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1936}, "m6g.2xlarge": {Region: "us-gov-west-1", Type: "m6g.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3872}, "m6g.4xlarge": {Region: "us-gov-west-1", Type: "m6g.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.7744}, "m6g.8xlarge": {Region: "us-gov-west-1", Type: "m6g.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.5488}, "m6g.12xlarge": {Region: "us-gov-west-1", Type: "m6g.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.3232}, "m6g.16xlarge": {Region: "us-gov-west-1", Type: "m6g.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.0976}, "m6g.metal": {Region: "us-gov-west-1", Type: "m6g.metal", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.0976}, "p2.xlarge": {Region: "us-gov-west-1", Type: "p2.xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("4"), GPU: 1, Inf: 0, Price: 1.08}, "p2.8xlarge": {Region: "us-gov-west-1", Type: "p2.8xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("32"), GPU: 8, Inf: 0, Price: 8.64}, "p2.16xlarge": {Region: "us-gov-west-1", Type: "p2.16xlarge", Memory: kresource.MustParse("749568Mi"), CPU: kresource.MustParse("64"), GPU: 16, Inf: 0, Price: 17.28}, "p3.2xlarge": {Region: "us-gov-west-1", Type: "p3.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 1, Inf: 0, Price: 3.672}, "p3.8xlarge": {Region: "us-gov-west-1", Type: "p3.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 4, Inf: 0, Price: 14.688}, "p3.16xlarge": {Region: "us-gov-west-1", Type: "p3.16xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("64"), GPU: 8, Inf: 0, Price: 29.376}, "p3dn.24xlarge": {Region: "us-gov-west-1", Type: "p3dn.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 8, Inf: 0, Price: 37.454}, "r3.large": {Region: "us-gov-west-1", Type: "r3.large", Memory: kresource.MustParse("15616Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.2}, "r3.xlarge": {Region: "us-gov-west-1", Type: "r3.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.399}, "r3.2xlarge": {Region: "us-gov-west-1", Type: "r3.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.798}, "r3.4xlarge": {Region: "us-gov-west-1", Type: "r3.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.596}, "r3.8xlarge": {Region: "us-gov-west-1", Type: "r3.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 3.192}, "r4.large": {Region: "us-gov-west-1", Type: "r4.large", Memory: kresource.MustParse("15616Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.1596}, "r4.xlarge": {Region: "us-gov-west-1", Type: "r4.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.3192}, "r4.2xlarge": {Region: "us-gov-west-1", Type: "r4.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.6384}, "r4.4xlarge": {Region: "us-gov-west-1", Type: "r4.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.2768}, "r4.8xlarge": {Region: "us-gov-west-1", Type: "r4.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.5536}, "r4.16xlarge": {Region: "us-gov-west-1", Type: "r4.16xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.1072}, "r5.large": {Region: "us-gov-west-1", Type: "r5.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.151}, "r5.xlarge": {Region: "us-gov-west-1", Type: "r5.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.302}, "r5.2xlarge": {Region: "us-gov-west-1", Type: "r5.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.604}, "r5.4xlarge": {Region: "us-gov-west-1", Type: "r5.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.208}, "r5.8xlarge": {Region: "us-gov-west-1", Type: "r5.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.416}, "r5.12xlarge": {Region: "us-gov-west-1", Type: "r5.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.624}, "r5.16xlarge": {Region: "us-gov-west-1", Type: "r5.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.832}, "r5.24xlarge": {Region: "us-gov-west-1", Type: "r5.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.248}, "r5.metal": {Region: "us-gov-west-1", Type: "r5.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.248}, "r5a.large": {Region: "us-gov-west-1", Type: "r5a.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.136}, "r5a.xlarge": {Region: "us-gov-west-1", Type: "r5a.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.272}, "r5a.2xlarge": {Region: "us-gov-west-1", Type: "r5a.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.544}, "r5a.4xlarge": {Region: "us-gov-west-1", Type: "r5a.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.088}, "r5a.8xlarge": {Region: "us-gov-west-1", Type: "r5a.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.176}, "r5a.12xlarge": {Region: "us-gov-west-1", Type: "r5a.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.264}, "r5a.16xlarge": {Region: "us-gov-west-1", Type: "r5a.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.352}, "r5a.24xlarge": {Region: "us-gov-west-1", Type: "r5a.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.528}, "r5ad.large": {Region: "us-gov-west-1", Type: "r5ad.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.158}, "r5ad.xlarge": {Region: "us-gov-west-1", Type: "r5ad.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.316}, "r5ad.2xlarge": {Region: "us-gov-west-1", Type: "r5ad.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.632}, "r5ad.4xlarge": {Region: "us-gov-west-1", Type: "r5ad.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.264}, "r5ad.8xlarge": {Region: "us-gov-west-1", Type: "r5ad.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.528}, "r5ad.12xlarge": {Region: "us-gov-west-1", Type: "r5ad.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.792}, "r5ad.16xlarge": {Region: "us-gov-west-1", Type: "r5ad.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.056}, "r5ad.24xlarge": {Region: "us-gov-west-1", Type: "r5ad.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.584}, "r5d.large": {Region: "us-gov-west-1", Type: "r5d.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.173}, "r5d.xlarge": {Region: "us-gov-west-1", Type: "r5d.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.346}, "r5d.2xlarge": {Region: "us-gov-west-1", Type: "r5d.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.692}, "r5d.4xlarge": {Region: "us-gov-west-1", Type: "r5d.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.384}, "r5d.8xlarge": {Region: "us-gov-west-1", Type: "r5d.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.768}, "r5d.12xlarge": {Region: "us-gov-west-1", Type: "r5d.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.152}, "r5d.16xlarge": {Region: "us-gov-west-1", Type: "r5d.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.536}, "r5d.24xlarge": {Region: "us-gov-west-1", Type: "r5d.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.304}, "r5d.metal": {Region: "us-gov-west-1", Type: "r5d.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.304}, "r5dn.large": {Region: "us-gov-west-1", Type: "r5dn.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.201}, "r5dn.xlarge": {Region: "us-gov-west-1", Type: "r5dn.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.402}, "r5dn.2xlarge": {Region: "us-gov-west-1", Type: "r5dn.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.804}, "r5dn.4xlarge": {Region: "us-gov-west-1", Type: "r5dn.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.608}, "r5dn.8xlarge": {Region: "us-gov-west-1", Type: "r5dn.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 3.216}, "r5dn.12xlarge": {Region: "us-gov-west-1", Type: "r5dn.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.824}, "r5dn.16xlarge": {Region: "us-gov-west-1", Type: "r5dn.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 6.432}, "r5dn.24xlarge": {Region: "us-gov-west-1", Type: "r5dn.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 9.648}, "r5dn.metal": {Region: "us-gov-west-1", Type: "r5dn.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 9.648}, "r5n.large": {Region: "us-gov-west-1", Type: "r5n.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.179}, "r5n.xlarge": {Region: "us-gov-west-1", Type: "r5n.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.358}, "r5n.2xlarge": {Region: "us-gov-west-1", Type: "r5n.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.716}, "r5n.4xlarge": {Region: "us-gov-west-1", Type: "r5n.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.432}, "r5n.8xlarge": {Region: "us-gov-west-1", Type: "r5n.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.864}, "r5n.12xlarge": {Region: "us-gov-west-1", Type: "r5n.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.296}, "r5n.16xlarge": {Region: "us-gov-west-1", Type: "r5n.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.728}, "r5n.24xlarge": {Region: "us-gov-west-1", Type: "r5n.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.592}, "r5n.metal": {Region: "us-gov-west-1", Type: "r5n.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.592}, "r6g.medium": {Region: "us-gov-west-1", Type: "r6g.medium", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0604}, "r6g.large": {Region: "us-gov-west-1", Type: "r6g.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.1208}, "r6g.xlarge": {Region: "us-gov-west-1", Type: "r6g.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.2416}, "r6g.2xlarge": {Region: "us-gov-west-1", Type: "r6g.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.4832}, "r6g.4xlarge": {Region: "us-gov-west-1", Type: "r6g.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.9664}, "r6g.8xlarge": {Region: "us-gov-west-1", Type: "r6g.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.9328}, "r6g.12xlarge": {Region: "us-gov-west-1", Type: "r6g.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.8992}, "r6g.16xlarge": {Region: "us-gov-west-1", Type: "r6g.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.8656}, "r6g.metal": {Region: "us-gov-west-1", Type: "r6g.metal", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.8656}, "t1.micro": {Region: "us-gov-west-1", Type: "t1.micro", Memory: kresource.MustParse("627Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.024}, "t2.nano": {Region: "us-gov-west-1", Type: "t2.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0068}, "t2.micro": {Region: "us-gov-west-1", Type: "t2.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0136}, "t2.small": {Region: "us-gov-west-1", Type: "t2.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0272}, "t2.medium": {Region: "us-gov-west-1", Type: "t2.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0544}, "t2.large": {Region: "us-gov-west-1", Type: "t2.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.1088}, "t2.xlarge": {Region: "us-gov-west-1", Type: "t2.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.2176}, "t2.2xlarge": {Region: "us-gov-west-1", Type: "t2.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.4352}, "t3.nano": {Region: "us-gov-west-1", Type: "t3.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0061}, "t3.micro": {Region: "us-gov-west-1", Type: "t3.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0122}, "t3.small": {Region: "us-gov-west-1", Type: "t3.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0244}, "t3.medium": {Region: "us-gov-west-1", Type: "t3.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0488}, "t3.large": {Region: "us-gov-west-1", Type: "t3.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0976}, "t3.xlarge": {Region: "us-gov-west-1", Type: "t3.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1952}, "t3.2xlarge": {Region: "us-gov-west-1", Type: "t3.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3904}, "t3a.nano": {Region: "us-gov-west-1", Type: "t3a.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0055}, "t3a.micro": {Region: "us-gov-west-1", Type: "t3a.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.011}, "t3a.small": {Region: "us-gov-west-1", Type: "t3a.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.022}, "t3a.medium": {Region: "us-gov-west-1", Type: "t3a.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0439}, "t3a.large": {Region: "us-gov-west-1", Type: "t3a.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0878}, "t3a.xlarge": {Region: "us-gov-west-1", Type: "t3a.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1757}, "t3a.2xlarge": {Region: "us-gov-west-1", Type: "t3a.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3514}, "t4g.nano": {Region: "us-gov-west-1", Type: "t4g.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0049}, "t4g.micro": {Region: "us-gov-west-1", Type: "t4g.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0098}, "t4g.small": {Region: "us-gov-west-1", Type: "t4g.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0196}, "t4g.medium": {Region: "us-gov-west-1", Type: "t4g.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0392}, "t4g.large": {Region: "us-gov-west-1", Type: "t4g.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0784}, "t4g.xlarge": {Region: "us-gov-west-1", Type: "t4g.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1568}, "t4g.2xlarge": {Region: "us-gov-west-1", Type: "t4g.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3136}, "u-12tb1.112xlarge": {Region: "us-gov-west-1", Type: "u-12tb1.112xlarge", Memory: kresource.MustParse("12582912Mi"), CPU: kresource.MustParse("448"), GPU: 0, Inf: 0, Price: 130.867}, "u-6tb1.56xlarge": {Region: "us-gov-west-1", Type: "u-6tb1.56xlarge", Memory: kresource.MustParse("6291456Mi"), CPU: kresource.MustParse("224"), GPU: 0, Inf: 0, Price: 55.61075}, "u-6tb1.112xlarge": {Region: "us-gov-west-1", Type: "u-6tb1.112xlarge", Memory: kresource.MustParse("6291456Mi"), CPU: kresource.MustParse("448"), GPU: 0, Inf: 0, Price: 65.433}, "u-9tb1.112xlarge": {Region: "us-gov-west-1", Type: "u-9tb1.112xlarge", Memory: kresource.MustParse("9437184Mi"), CPU: kresource.MustParse("448"), GPU: 0, Inf: 0, Price: 98.15}, "x1.16xlarge": {Region: "us-gov-west-1", Type: "x1.16xlarge", Memory: kresource.MustParse("999424Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 8.003}, "x1.32xlarge": {Region: "us-gov-west-1", Type: "x1.32xlarge", Memory: kresource.MustParse("1998848Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 16.006}, "x1e.xlarge": {Region: "us-gov-west-1", Type: "x1e.xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 1.0}, "x1e.2xlarge": {Region: "us-gov-west-1", Type: "x1e.2xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 2.0}, "x1e.4xlarge": {Region: "us-gov-west-1", Type: "x1e.4xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 4.0}, "x1e.8xlarge": {Region: "us-gov-west-1", Type: "x1e.8xlarge", Memory: kresource.MustParse("999424Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 8.0}, "x1e.16xlarge": {Region: "us-gov-west-1", Type: "x1e.16xlarge", Memory: kresource.MustParse("1998848Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 16.0}, "x1e.32xlarge": {Region: "us-gov-west-1", Type: "x1e.32xlarge", Memory: kresource.MustParse("3997696Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 32.0}, }, "us-west-1": { "c1.medium": {Region: "us-west-1", Type: "c1.medium", Memory: kresource.MustParse("1740Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.148}, "c1.xlarge": {Region: "us-west-1", Type: "c1.xlarge", Memory: kresource.MustParse("7168Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.592}, "c3.large": {Region: "us-west-1", Type: "c3.large", Memory: kresource.MustParse("3840Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.12}, "c3.xlarge": {Region: "us-west-1", Type: "c3.xlarge", Memory: kresource.MustParse("7680Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.239}, "c3.2xlarge": {Region: "us-west-1", Type: "c3.2xlarge", Memory: kresource.MustParse("15360Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.478}, "c3.4xlarge": {Region: "us-west-1", Type: "c3.4xlarge", Memory: kresource.MustParse("30720Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.956}, "c3.8xlarge": {Region: "us-west-1", Type: "c3.8xlarge", Memory: kresource.MustParse("61440Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.912}, "c4.large": {Region: "us-west-1", Type: "c4.large", Memory: kresource.MustParse("3840Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.124}, "c4.xlarge": {Region: "us-west-1", Type: "c4.xlarge", Memory: kresource.MustParse("7680Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.249}, "c4.2xlarge": {Region: "us-west-1", Type: "c4.2xlarge", Memory: kresource.MustParse("15360Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.498}, "c4.4xlarge": {Region: "us-west-1", Type: "c4.4xlarge", Memory: kresource.MustParse("30720Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.997}, "c4.8xlarge": {Region: "us-west-1", Type: "c4.8xlarge", Memory: kresource.MustParse("61440Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 1.993}, "c5.large": {Region: "us-west-1", Type: "c5.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.106}, "c5.xlarge": {Region: "us-west-1", Type: "c5.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.212}, "c5.2xlarge": {Region: "us-west-1", Type: "c5.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.424}, "c5.4xlarge": {Region: "us-west-1", Type: "c5.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.848}, "c5.9xlarge": {Region: "us-west-1", Type: "c5.9xlarge", Memory: kresource.MustParse("73728Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 1.908}, "c5.12xlarge": {Region: "us-west-1", Type: "c5.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.544}, "c5.18xlarge": {Region: "us-west-1", Type: "c5.18xlarge", Memory: kresource.MustParse("147456Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 3.816}, "c5.24xlarge": {Region: "us-west-1", Type: "c5.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.088}, "c5.metal": {Region: "us-west-1", Type: "c5.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.088}, "c5a.large": {Region: "us-west-1", Type: "c5a.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.095}, "c5a.xlarge": {Region: "us-west-1", Type: "c5a.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.19}, "c5a.2xlarge": {Region: "us-west-1", Type: "c5a.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.38}, "c5a.4xlarge": {Region: "us-west-1", Type: "c5a.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.76}, "c5a.8xlarge": {Region: "us-west-1", Type: "c5a.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.52}, "c5a.12xlarge": {Region: "us-west-1", Type: "c5a.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.28}, "c5a.16xlarge": {Region: "us-west-1", Type: "c5a.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.04}, "c5a.24xlarge": {Region: "us-west-1", Type: "c5a.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.56}, "c5d.large": {Region: "us-west-1", Type: "c5d.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.12}, "c5d.xlarge": {Region: "us-west-1", Type: "c5d.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.24}, "c5d.2xlarge": {Region: "us-west-1", Type: "c5d.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.48}, "c5d.4xlarge": {Region: "us-west-1", Type: "c5d.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.96}, "c5d.9xlarge": {Region: "us-west-1", Type: "c5d.9xlarge", Memory: kresource.MustParse("73728Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 2.16}, "c5d.12xlarge": {Region: "us-west-1", Type: "c5d.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.88}, "c5d.18xlarge": {Region: "us-west-1", Type: "c5d.18xlarge", Memory: kresource.MustParse("147456Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 4.32}, "c5d.24xlarge": {Region: "us-west-1", Type: "c5d.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.76}, "c5d.metal": {Region: "us-west-1", Type: "c5d.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.76}, "c5n.large": {Region: "us-west-1", Type: "c5n.large", Memory: kresource.MustParse("5376Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.135}, "c5n.xlarge": {Region: "us-west-1", Type: "c5n.xlarge", Memory: kresource.MustParse("10752Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.27}, "c5n.2xlarge": {Region: "us-west-1", Type: "c5n.2xlarge", Memory: kresource.MustParse("21504Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.54}, "c5n.4xlarge": {Region: "us-west-1", Type: "c5n.4xlarge", Memory: kresource.MustParse("43008Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.08}, "c5n.9xlarge": {Region: "us-west-1", Type: "c5n.9xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 2.43}, "c5n.18xlarge": {Region: "us-west-1", Type: "c5n.18xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 4.86}, "c5n.metal": {Region: "us-west-1", Type: "c5n.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 4.86}, "c6g.medium": {Region: "us-west-1", Type: "c6g.medium", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0424}, "c6g.large": {Region: "us-west-1", Type: "c6g.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0848}, "c6g.xlarge": {Region: "us-west-1", Type: "c6g.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1696}, "c6g.2xlarge": {Region: "us-west-1", Type: "c6g.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3392}, "c6g.4xlarge": {Region: "us-west-1", Type: "c6g.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.6784}, "c6g.8xlarge": {Region: "us-west-1", Type: "c6g.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.3568}, "c6g.12xlarge": {Region: "us-west-1", Type: "c6g.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.0352}, "c6g.16xlarge": {Region: "us-west-1", Type: "c6g.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.7136}, "c6g.metal": {Region: "us-west-1", Type: "c6g.metal", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.7136}, "c6gd.medium": {Region: "us-west-1", Type: "c6gd.medium", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.048}, "c6gd.large": {Region: "us-west-1", Type: "c6gd.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.096}, "c6gd.xlarge": {Region: "us-west-1", Type: "c6gd.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.192}, "c6gd.2xlarge": {Region: "us-west-1", Type: "c6gd.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.384}, "c6gd.4xlarge": {Region: "us-west-1", Type: "c6gd.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.768}, "c6gd.8xlarge": {Region: "us-west-1", Type: "c6gd.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.536}, "c6gd.12xlarge": {Region: "us-west-1", Type: "c6gd.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.304}, "c6gd.16xlarge": {Region: "us-west-1", Type: "c6gd.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.072}, "c6gd.metal": {Region: "us-west-1", Type: "c6gd.metal", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.072}, "c6gn.medium": {Region: "us-west-1", Type: "c6gn.medium", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.054}, "c6gn.large": {Region: "us-west-1", Type: "c6gn.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.108}, "c6gn.xlarge": {Region: "us-west-1", Type: "c6gn.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.216}, "c6gn.2xlarge": {Region: "us-west-1", Type: "c6gn.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.432}, "c6gn.4xlarge": {Region: "us-west-1", Type: "c6gn.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.864}, "c6gn.8xlarge": {Region: "us-west-1", Type: "c6gn.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.728}, "c6gn.12xlarge": {Region: "us-west-1", Type: "c6gn.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.592}, "c6gn.16xlarge": {Region: "us-west-1", Type: "c6gn.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.456}, "c6i.large": {Region: "us-west-1", Type: "c6i.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.106}, "c6i.xlarge": {Region: "us-west-1", Type: "c6i.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.212}, "c6i.2xlarge": {Region: "us-west-1", Type: "c6i.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.424}, "c6i.4xlarge": {Region: "us-west-1", Type: "c6i.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.848}, "c6i.8xlarge": {Region: "us-west-1", Type: "c6i.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.696}, "c6i.12xlarge": {Region: "us-west-1", Type: "c6i.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.544}, "c6i.16xlarge": {Region: "us-west-1", Type: "c6i.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.392}, "c6i.24xlarge": {Region: "us-west-1", Type: "c6i.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.088}, "c6i.32xlarge": {Region: "us-west-1", Type: "c6i.32xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 6.784}, "c6i.metal": {Region: "us-west-1", Type: "c6i.metal", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 6.784}, "d2.xlarge": {Region: "us-west-1", Type: "d2.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.781}, "d2.2xlarge": {Region: "us-west-1", Type: "d2.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.563}, "d2.4xlarge": {Region: "us-west-1", Type: "d2.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 3.125}, "d2.8xlarge": {Region: "us-west-1", Type: "d2.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 6.25}, "g2.2xlarge": {Region: "us-west-1", Type: "g2.2xlarge", Memory: kresource.MustParse("15360Mi"), CPU: kresource.MustParse("8"), GPU: 1, Inf: 0, Price: 0.702}, "g2.8xlarge": {Region: "us-west-1", Type: "g2.8xlarge", Memory: kresource.MustParse("61440Mi"), CPU: kresource.MustParse("32"), GPU: 4, Inf: 0, Price: 2.808}, "g3.4xlarge": {Region: "us-west-1", Type: "g3.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 1, Inf: 0, Price: 1.534}, "g3.8xlarge": {Region: "us-west-1", Type: "g3.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 2, Inf: 0, Price: 3.068}, "g3.16xlarge": {Region: "us-west-1", Type: "g3.16xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("64"), GPU: 4, Inf: 0, Price: 6.136}, "g4dn.xlarge": {Region: "us-west-1", Type: "g4dn.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 1, Inf: 0, Price: 0.631}, "g4dn.2xlarge": {Region: "us-west-1", Type: "g4dn.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 1, Inf: 0, Price: 0.902}, "g4dn.4xlarge": {Region: "us-west-1", Type: "g4dn.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 1, Inf: 0, Price: 1.445}, "g4dn.8xlarge": {Region: "us-west-1", Type: "g4dn.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 1, Inf: 0, Price: 2.611}, "g4dn.12xlarge": {Region: "us-west-1", Type: "g4dn.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 4, Inf: 0, Price: 4.694}, "g4dn.16xlarge": {Region: "us-west-1", Type: "g4dn.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 1, Inf: 0, Price: 5.222}, "g4dn.metal": {Region: "us-west-1", Type: "g4dn.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 8, Inf: 0, Price: 9.389}, "i2.xlarge": {Region: "us-west-1", Type: "i2.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.938}, "i2.2xlarge": {Region: "us-west-1", Type: "i2.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.876}, "i2.4xlarge": {Region: "us-west-1", Type: "i2.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 3.751}, "i2.8xlarge": {Region: "us-west-1", Type: "i2.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 7.502}, "i3.large": {Region: "us-west-1", Type: "i3.large", Memory: kresource.MustParse("15616Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.172}, "i3.xlarge": {Region: "us-west-1", Type: "i3.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.344}, "i3.2xlarge": {Region: "us-west-1", Type: "i3.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.688}, "i3.4xlarge": {Region: "us-west-1", Type: "i3.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.376}, "i3.8xlarge": {Region: "us-west-1", Type: "i3.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.752}, "i3.16xlarge": {Region: "us-west-1", Type: "i3.16xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.504}, "i3.metal": {Region: "us-west-1", Type: "i3.metal", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.504}, "i3en.large": {Region: "us-west-1", Type: "i3en.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.25}, "i3en.xlarge": {Region: "us-west-1", Type: "i3en.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.5}, "i3en.2xlarge": {Region: "us-west-1", Type: "i3en.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.0}, "i3en.3xlarge": {Region: "us-west-1", Type: "i3en.3xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("12"), GPU: 0, Inf: 0, Price: 1.5}, "i3en.6xlarge": {Region: "us-west-1", Type: "i3en.6xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 0, Price: 3.0}, "i3en.12xlarge": {Region: "us-west-1", Type: "i3en.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 6.0}, "i3en.24xlarge": {Region: "us-west-1", Type: "i3en.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 12.0}, "i3en.metal": {Region: "us-west-1", Type: "i3en.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 12.0}, "inf1.xlarge": {Region: "us-west-1", Type: "inf1.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 1, Price: 0.274}, "inf1.2xlarge": {Region: "us-west-1", Type: "inf1.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 1, Price: 0.435}, "inf1.6xlarge": {Region: "us-west-1", Type: "inf1.6xlarge", Memory: kresource.MustParse("49152Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 4, Price: 1.418}, "inf1.24xlarge": {Region: "us-west-1", Type: "inf1.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 16, Price: 5.671}, "m1.small": {Region: "us-west-1", Type: "m1.small", Memory: kresource.MustParse("1740Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.047}, "m1.medium": {Region: "us-west-1", Type: "m1.medium", Memory: kresource.MustParse("3840Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.095}, "m1.large": {Region: "us-west-1", Type: "m1.large", Memory: kresource.MustParse("7680Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.19}, "m1.xlarge": {Region: "us-west-1", Type: "m1.xlarge", Memory: kresource.MustParse("15360Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.379}, "m2.xlarge": {Region: "us-west-1", Type: "m2.xlarge", Memory: kresource.MustParse("17510Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.275}, "m2.2xlarge": {Region: "us-west-1", Type: "m2.2xlarge", Memory: kresource.MustParse("35020Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.55}, "m2.4xlarge": {Region: "us-west-1", Type: "m2.4xlarge", Memory: kresource.MustParse("70041Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.1}, "m3.medium": {Region: "us-west-1", Type: "m3.medium", Memory: kresource.MustParse("3840Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.077}, "m3.large": {Region: "us-west-1", Type: "m3.large", Memory: kresource.MustParse("7680Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.154}, "m3.xlarge": {Region: "us-west-1", Type: "m3.xlarge", Memory: kresource.MustParse("15360Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.308}, "m3.2xlarge": {Region: "us-west-1", Type: "m3.2xlarge", Memory: kresource.MustParse("30720Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.616}, "m4.large": {Region: "us-west-1", Type: "m4.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.117}, "m4.xlarge": {Region: "us-west-1", Type: "m4.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.234}, "m4.2xlarge": {Region: "us-west-1", Type: "m4.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.468}, "m4.4xlarge": {Region: "us-west-1", Type: "m4.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.936}, "m4.10xlarge": {Region: "us-west-1", Type: "m4.10xlarge", Memory: kresource.MustParse("163840Mi"), CPU: kresource.MustParse("40"), GPU: 0, Inf: 0, Price: 2.34}, "m4.16xlarge": {Region: "us-west-1", Type: "m4.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.744}, "m5.large": {Region: "us-west-1", Type: "m5.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.112}, "m5.xlarge": {Region: "us-west-1", Type: "m5.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.224}, "m5.2xlarge": {Region: "us-west-1", Type: "m5.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.448}, "m5.4xlarge": {Region: "us-west-1", Type: "m5.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.896}, "m5.8xlarge": {Region: "us-west-1", Type: "m5.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.792}, "m5.12xlarge": {Region: "us-west-1", Type: "m5.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.688}, "m5.16xlarge": {Region: "us-west-1", Type: "m5.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.584}, "m5.24xlarge": {Region: "us-west-1", Type: "m5.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.376}, "m5.metal": {Region: "us-west-1", Type: "m5.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.376}, "m5a.large": {Region: "us-west-1", Type: "m5a.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.101}, "m5a.xlarge": {Region: "us-west-1", Type: "m5a.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.202}, "m5a.2xlarge": {Region: "us-west-1", Type: "m5a.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.404}, "m5a.4xlarge": {Region: "us-west-1", Type: "m5a.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.808}, "m5a.8xlarge": {Region: "us-west-1", Type: "m5a.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.616}, "m5a.12xlarge": {Region: "us-west-1", Type: "m5a.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.424}, "m5a.16xlarge": {Region: "us-west-1", Type: "m5a.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.232}, "m5a.24xlarge": {Region: "us-west-1", Type: "m5a.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.848}, "m5ad.large": {Region: "us-west-1", Type: "m5ad.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.122}, "m5ad.xlarge": {Region: "us-west-1", Type: "m5ad.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.244}, "m5ad.2xlarge": {Region: "us-west-1", Type: "m5ad.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.488}, "m5ad.4xlarge": {Region: "us-west-1", Type: "m5ad.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.976}, "m5ad.8xlarge": {Region: "us-west-1", Type: "m5ad.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.952}, "m5ad.12xlarge": {Region: "us-west-1", Type: "m5ad.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.928}, "m5ad.16xlarge": {Region: "us-west-1", Type: "m5ad.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.904}, "m5ad.24xlarge": {Region: "us-west-1", Type: "m5ad.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.856}, "m5d.large": {Region: "us-west-1", Type: "m5d.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.133}, "m5d.xlarge": {Region: "us-west-1", Type: "m5d.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.266}, "m5d.2xlarge": {Region: "us-west-1", Type: "m5d.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.532}, "m5d.4xlarge": {Region: "us-west-1", Type: "m5d.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.064}, "m5d.8xlarge": {Region: "us-west-1", Type: "m5d.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.128}, "m5d.12xlarge": {Region: "us-west-1", Type: "m5d.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.192}, "m5d.16xlarge": {Region: "us-west-1", Type: "m5d.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.256}, "m5d.24xlarge": {Region: "us-west-1", Type: "m5d.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.384}, "m5d.metal": {Region: "us-west-1", Type: "m5d.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.384}, "m5zn.large": {Region: "us-west-1", Type: "m5zn.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.1927}, "m5zn.xlarge": {Region: "us-west-1", Type: "m5zn.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.3854}, "m5zn.2xlarge": {Region: "us-west-1", Type: "m5zn.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.7708}, "m5zn.3xlarge": {Region: "us-west-1", Type: "m5zn.3xlarge", Memory: kresource.MustParse("49152Mi"), CPU: kresource.MustParse("12"), GPU: 0, Inf: 0, Price: 1.1562}, "m5zn.6xlarge": {Region: "us-west-1", Type: "m5zn.6xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 0, Price: 2.3124}, "m5zn.12xlarge": {Region: "us-west-1", Type: "m5zn.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.6248}, "m5zn.metal": {Region: "us-west-1", Type: "m5zn.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.6248}, "m6g.medium": {Region: "us-west-1", Type: "m6g.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0448}, "m6g.large": {Region: "us-west-1", Type: "m6g.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0896}, "m6g.xlarge": {Region: "us-west-1", Type: "m6g.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1792}, "m6g.2xlarge": {Region: "us-west-1", Type: "m6g.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3584}, "m6g.4xlarge": {Region: "us-west-1", Type: "m6g.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.7168}, "m6g.8xlarge": {Region: "us-west-1", Type: "m6g.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.4336}, "m6g.12xlarge": {Region: "us-west-1", Type: "m6g.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.1504}, "m6g.16xlarge": {Region: "us-west-1", Type: "m6g.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.8672}, "m6g.metal": {Region: "us-west-1", Type: "m6g.metal", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.8672}, "m6gd.medium": {Region: "us-west-1", Type: "m6gd.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.053}, "m6gd.large": {Region: "us-west-1", Type: "m6gd.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.106}, "m6gd.xlarge": {Region: "us-west-1", Type: "m6gd.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.212}, "m6gd.2xlarge": {Region: "us-west-1", Type: "m6gd.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.424}, "m6gd.4xlarge": {Region: "us-west-1", Type: "m6gd.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.848}, "m6gd.8xlarge": {Region: "us-west-1", Type: "m6gd.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.696}, "m6gd.12xlarge": {Region: "us-west-1", Type: "m6gd.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.544}, "m6gd.16xlarge": {Region: "us-west-1", Type: "m6gd.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.392}, "m6gd.metal": {Region: "us-west-1", Type: "m6gd.metal", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.392}, "m6i.large": {Region: "us-west-1", Type: "m6i.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.112}, "m6i.xlarge": {Region: "us-west-1", Type: "m6i.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.224}, "m6i.2xlarge": {Region: "us-west-1", Type: "m6i.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.448}, "m6i.4xlarge": {Region: "us-west-1", Type: "m6i.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.896}, "m6i.8xlarge": {Region: "us-west-1", Type: "m6i.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.792}, "m6i.12xlarge": {Region: "us-west-1", Type: "m6i.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.688}, "m6i.16xlarge": {Region: "us-west-1", Type: "m6i.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.584}, "m6i.24xlarge": {Region: "us-west-1", Type: "m6i.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.376}, "m6i.32xlarge": {Region: "us-west-1", Type: "m6i.32xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 7.168}, "m6i.metal": {Region: "us-west-1", Type: "m6i.metal", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 7.168}, "r3.large": {Region: "us-west-1", Type: "r3.large", Memory: kresource.MustParse("15616Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.185}, "r3.xlarge": {Region: "us-west-1", Type: "r3.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.371}, "r3.2xlarge": {Region: "us-west-1", Type: "r3.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.741}, "r3.4xlarge": {Region: "us-west-1", Type: "r3.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.482}, "r3.8xlarge": {Region: "us-west-1", Type: "r3.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.964}, "r4.large": {Region: "us-west-1", Type: "r4.large", Memory: kresource.MustParse("15616Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.1482}, "r4.xlarge": {Region: "us-west-1", Type: "r4.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.2964}, "r4.2xlarge": {Region: "us-west-1", Type: "r4.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.5928}, "r4.4xlarge": {Region: "us-west-1", Type: "r4.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.1856}, "r4.8xlarge": {Region: "us-west-1", Type: "r4.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.3712}, "r4.16xlarge": {Region: "us-west-1", Type: "r4.16xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.7424}, "r5.large": {Region: "us-west-1", Type: "r5.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.14}, "r5.xlarge": {Region: "us-west-1", Type: "r5.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.28}, "r5.2xlarge": {Region: "us-west-1", Type: "r5.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.56}, "r5.4xlarge": {Region: "us-west-1", Type: "r5.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.12}, "r5.8xlarge": {Region: "us-west-1", Type: "r5.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.24}, "r5.12xlarge": {Region: "us-west-1", Type: "r5.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.36}, "r5.16xlarge": {Region: "us-west-1", Type: "r5.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.48}, "r5.24xlarge": {Region: "us-west-1", Type: "r5.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.72}, "r5.metal": {Region: "us-west-1", Type: "r5.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.72}, "r5a.large": {Region: "us-west-1", Type: "r5a.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.126}, "r5a.xlarge": {Region: "us-west-1", Type: "r5a.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.252}, "r5a.2xlarge": {Region: "us-west-1", Type: "r5a.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.504}, "r5a.4xlarge": {Region: "us-west-1", Type: "r5a.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.008}, "r5a.8xlarge": {Region: "us-west-1", Type: "r5a.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.016}, "r5a.12xlarge": {Region: "us-west-1", Type: "r5a.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.024}, "r5a.16xlarge": {Region: "us-west-1", Type: "r5a.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.032}, "r5a.24xlarge": {Region: "us-west-1", Type: "r5a.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.048}, "r5ad.large": {Region: "us-west-1", Type: "r5ad.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.148}, "r5ad.xlarge": {Region: "us-west-1", Type: "r5ad.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.296}, "r5ad.2xlarge": {Region: "us-west-1", Type: "r5ad.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.592}, "r5ad.4xlarge": {Region: "us-west-1", Type: "r5ad.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.184}, "r5ad.8xlarge": {Region: "us-west-1", Type: "r5ad.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.368}, "r5ad.12xlarge": {Region: "us-west-1", Type: "r5ad.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.552}, "r5ad.16xlarge": {Region: "us-west-1", Type: "r5ad.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.736}, "r5ad.24xlarge": {Region: "us-west-1", Type: "r5ad.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.104}, "r5d.large": {Region: "us-west-1", Type: "r5d.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.162}, "r5d.xlarge": {Region: "us-west-1", Type: "r5d.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.324}, "r5d.2xlarge": {Region: "us-west-1", Type: "r5d.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.648}, "r5d.4xlarge": {Region: "us-west-1", Type: "r5d.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.296}, "r5d.8xlarge": {Region: "us-west-1", Type: "r5d.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.592}, "r5d.12xlarge": {Region: "us-west-1", Type: "r5d.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.888}, "r5d.16xlarge": {Region: "us-west-1", Type: "r5d.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.184}, "r5d.24xlarge": {Region: "us-west-1", Type: "r5d.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.776}, "r5d.metal": {Region: "us-west-1", Type: "r5d.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.776}, "r5n.large": {Region: "us-west-1", Type: "r5n.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.169}, "r5n.xlarge": {Region: "us-west-1", Type: "r5n.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.338}, "r5n.2xlarge": {Region: "us-west-1", Type: "r5n.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.676}, "r5n.4xlarge": {Region: "us-west-1", Type: "r5n.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.352}, "r5n.8xlarge": {Region: "us-west-1", Type: "r5n.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.704}, "r5n.12xlarge": {Region: "us-west-1", Type: "r5n.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.056}, "r5n.16xlarge": {Region: "us-west-1", Type: "r5n.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.408}, "r5n.24xlarge": {Region: "us-west-1", Type: "r5n.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.112}, "r5n.metal": {Region: "us-west-1", Type: "r5n.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.112}, "r6g.medium": {Region: "us-west-1", Type: "r6g.medium", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.056}, "r6g.large": {Region: "us-west-1", Type: "r6g.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.112}, "r6g.xlarge": {Region: "us-west-1", Type: "r6g.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.224}, "r6g.2xlarge": {Region: "us-west-1", Type: "r6g.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.448}, "r6g.4xlarge": {Region: "us-west-1", Type: "r6g.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.896}, "r6g.8xlarge": {Region: "us-west-1", Type: "r6g.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.792}, "r6g.12xlarge": {Region: "us-west-1", Type: "r6g.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.688}, "r6g.16xlarge": {Region: "us-west-1", Type: "r6g.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.584}, "r6g.metal": {Region: "us-west-1", Type: "r6g.metal", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.584}, "r6gd.medium": {Region: "us-west-1", Type: "r6gd.medium", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.065}, "r6gd.large": {Region: "us-west-1", Type: "r6gd.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.13}, "r6gd.xlarge": {Region: "us-west-1", Type: "r6gd.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.26}, "r6gd.2xlarge": {Region: "us-west-1", Type: "r6gd.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.52}, "r6gd.4xlarge": {Region: "us-west-1", Type: "r6gd.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.04}, "r6gd.8xlarge": {Region: "us-west-1", Type: "r6gd.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.08}, "r6gd.12xlarge": {Region: "us-west-1", Type: "r6gd.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.12}, "r6gd.16xlarge": {Region: "us-west-1", Type: "r6gd.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.16}, "r6gd.metal": {Region: "us-west-1", Type: "r6gd.metal", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.16}, "r6i.large": {Region: "us-west-1", Type: "r6i.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.14}, "r6i.xlarge": {Region: "us-west-1", Type: "r6i.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.28}, "r6i.2xlarge": {Region: "us-west-1", Type: "r6i.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.56}, "r6i.4xlarge": {Region: "us-west-1", Type: "r6i.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.12}, "r6i.8xlarge": {Region: "us-west-1", Type: "r6i.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.24}, "r6i.12xlarge": {Region: "us-west-1", Type: "r6i.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.36}, "r6i.16xlarge": {Region: "us-west-1", Type: "r6i.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.48}, "r6i.24xlarge": {Region: "us-west-1", Type: "r6i.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.72}, "r6i.32xlarge": {Region: "us-west-1", Type: "r6i.32xlarge", Memory: kresource.MustParse("1048576Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 8.96}, "r6i.metal": {Region: "us-west-1", Type: "r6i.metal", Memory: kresource.MustParse("1048576Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 8.96}, "t1.micro": {Region: "us-west-1", Type: "t1.micro", Memory: kresource.MustParse("627Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.025}, "t2.nano": {Region: "us-west-1", Type: "t2.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0069}, "t2.micro": {Region: "us-west-1", Type: "t2.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0138}, "t2.small": {Region: "us-west-1", Type: "t2.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0276}, "t2.medium": {Region: "us-west-1", Type: "t2.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0552}, "t2.large": {Region: "us-west-1", Type: "t2.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.1104}, "t2.xlarge": {Region: "us-west-1", Type: "t2.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.2208}, "t2.2xlarge": {Region: "us-west-1", Type: "t2.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.4416}, "t3.nano": {Region: "us-west-1", Type: "t3.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0062}, "t3.micro": {Region: "us-west-1", Type: "t3.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0124}, "t3.small": {Region: "us-west-1", Type: "t3.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0248}, "t3.medium": {Region: "us-west-1", Type: "t3.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0496}, "t3.large": {Region: "us-west-1", Type: "t3.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0992}, "t3.xlarge": {Region: "us-west-1", Type: "t3.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1984}, "t3.2xlarge": {Region: "us-west-1", Type: "t3.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3968}, "t3a.nano": {Region: "us-west-1", Type: "t3a.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0056}, "t3a.micro": {Region: "us-west-1", Type: "t3a.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0112}, "t3a.small": {Region: "us-west-1", Type: "t3a.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0223}, "t3a.medium": {Region: "us-west-1", Type: "t3a.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0446}, "t3a.large": {Region: "us-west-1", Type: "t3a.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0893}, "t3a.xlarge": {Region: "us-west-1", Type: "t3a.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1786}, "t3a.2xlarge": {Region: "us-west-1", Type: "t3a.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3571}, "t4g.nano": {Region: "us-west-1", Type: "t4g.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.005}, "t4g.micro": {Region: "us-west-1", Type: "t4g.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.01}, "t4g.small": {Region: "us-west-1", Type: "t4g.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.02}, "t4g.medium": {Region: "us-west-1", Type: "t4g.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.04}, "t4g.large": {Region: "us-west-1", Type: "t4g.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.08}, "t4g.xlarge": {Region: "us-west-1", Type: "t4g.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.16}, "t4g.2xlarge": {Region: "us-west-1", Type: "t4g.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.32}, "z1d.large": {Region: "us-west-1", Type: "z1d.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.211}, "z1d.xlarge": {Region: "us-west-1", Type: "z1d.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.422}, "z1d.2xlarge": {Region: "us-west-1", Type: "z1d.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.844}, "z1d.3xlarge": {Region: "us-west-1", Type: "z1d.3xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("12"), GPU: 0, Inf: 0, Price: 1.266}, "z1d.6xlarge": {Region: "us-west-1", Type: "z1d.6xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 0, Price: 2.532}, "z1d.12xlarge": {Region: "us-west-1", Type: "z1d.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 5.064}, "z1d.metal": {Region: "us-west-1", Type: "z1d.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 5.064}, }, "us-west-2": { "a1.medium": {Region: "us-west-2", Type: "a1.medium", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0255}, "a1.large": {Region: "us-west-2", Type: "a1.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.051}, "a1.xlarge": {Region: "us-west-2", Type: "a1.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.102}, "a1.2xlarge": {Region: "us-west-2", Type: "a1.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.204}, "a1.4xlarge": {Region: "us-west-2", Type: "a1.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.408}, "a1.metal": {Region: "us-west-2", Type: "a1.metal", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.408}, "c1.medium": {Region: "us-west-2", Type: "c1.medium", Memory: kresource.MustParse("1740Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.13}, "c1.xlarge": {Region: "us-west-2", Type: "c1.xlarge", Memory: kresource.MustParse("7168Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.52}, "c3.large": {Region: "us-west-2", Type: "c3.large", Memory: kresource.MustParse("3840Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.105}, "c3.xlarge": {Region: "us-west-2", Type: "c3.xlarge", Memory: kresource.MustParse("7680Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.21}, "c3.2xlarge": {Region: "us-west-2", Type: "c3.2xlarge", Memory: kresource.MustParse("15360Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.42}, "c3.4xlarge": {Region: "us-west-2", Type: "c3.4xlarge", Memory: kresource.MustParse("30720Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.84}, "c3.8xlarge": {Region: "us-west-2", Type: "c3.8xlarge", Memory: kresource.MustParse("61440Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.68}, "c4.large": {Region: "us-west-2", Type: "c4.large", Memory: kresource.MustParse("3840Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.1}, "c4.xlarge": {Region: "us-west-2", Type: "c4.xlarge", Memory: kresource.MustParse("7680Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.199}, "c4.2xlarge": {Region: "us-west-2", Type: "c4.2xlarge", Memory: kresource.MustParse("15360Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.398}, "c4.4xlarge": {Region: "us-west-2", Type: "c4.4xlarge", Memory: kresource.MustParse("30720Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.796}, "c4.8xlarge": {Region: "us-west-2", Type: "c4.8xlarge", Memory: kresource.MustParse("61440Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 1.591}, "c5.large": {Region: "us-west-2", Type: "c5.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.085}, "c5.xlarge": {Region: "us-west-2", Type: "c5.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.17}, "c5.2xlarge": {Region: "us-west-2", Type: "c5.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.34}, "c5.4xlarge": {Region: "us-west-2", Type: "c5.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.68}, "c5.9xlarge": {Region: "us-west-2", Type: "c5.9xlarge", Memory: kresource.MustParse("73728Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 1.53}, "c5.12xlarge": {Region: "us-west-2", Type: "c5.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.04}, "c5.18xlarge": {Region: "us-west-2", Type: "c5.18xlarge", Memory: kresource.MustParse("147456Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 3.06}, "c5.24xlarge": {Region: "us-west-2", Type: "c5.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.08}, "c5.metal": {Region: "us-west-2", Type: "c5.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.08}, "c5a.large": {Region: "us-west-2", Type: "c5a.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.077}, "c5a.xlarge": {Region: "us-west-2", Type: "c5a.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.154}, "c5a.2xlarge": {Region: "us-west-2", Type: "c5a.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.308}, "c5a.4xlarge": {Region: "us-west-2", Type: "c5a.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.616}, "c5a.8xlarge": {Region: "us-west-2", Type: "c5a.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.232}, "c5a.12xlarge": {Region: "us-west-2", Type: "c5a.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 1.848}, "c5a.16xlarge": {Region: "us-west-2", Type: "c5a.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.464}, "c5a.24xlarge": {Region: "us-west-2", Type: "c5a.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 3.696}, "c5ad.large": {Region: "us-west-2", Type: "c5ad.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.086}, "c5ad.xlarge": {Region: "us-west-2", Type: "c5ad.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.172}, "c5ad.2xlarge": {Region: "us-west-2", Type: "c5ad.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.344}, "c5ad.4xlarge": {Region: "us-west-2", Type: "c5ad.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.688}, "c5ad.8xlarge": {Region: "us-west-2", Type: "c5ad.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.376}, "c5ad.12xlarge": {Region: "us-west-2", Type: "c5ad.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.064}, "c5ad.16xlarge": {Region: "us-west-2", Type: "c5ad.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.752}, "c5ad.24xlarge": {Region: "us-west-2", Type: "c5ad.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.128}, "c5d.large": {Region: "us-west-2", Type: "c5d.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.096}, "c5d.xlarge": {Region: "us-west-2", Type: "c5d.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.192}, "c5d.2xlarge": {Region: "us-west-2", Type: "c5d.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.384}, "c5d.4xlarge": {Region: "us-west-2", Type: "c5d.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.768}, "c5d.9xlarge": {Region: "us-west-2", Type: "c5d.9xlarge", Memory: kresource.MustParse("73728Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 1.728}, "c5d.12xlarge": {Region: "us-west-2", Type: "c5d.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.304}, "c5d.18xlarge": {Region: "us-west-2", Type: "c5d.18xlarge", Memory: kresource.MustParse("147456Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 3.456}, "c5d.24xlarge": {Region: "us-west-2", Type: "c5d.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.608}, "c5d.metal": {Region: "us-west-2", Type: "c5d.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.608}, "c5n.large": {Region: "us-west-2", Type: "c5n.large", Memory: kresource.MustParse("5376Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.108}, "c5n.xlarge": {Region: "us-west-2", Type: "c5n.xlarge", Memory: kresource.MustParse("10752Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.216}, "c5n.2xlarge": {Region: "us-west-2", Type: "c5n.2xlarge", Memory: kresource.MustParse("21504Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.432}, "c5n.4xlarge": {Region: "us-west-2", Type: "c5n.4xlarge", Memory: kresource.MustParse("43008Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.864}, "c5n.9xlarge": {Region: "us-west-2", Type: "c5n.9xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 1.944}, "c5n.18xlarge": {Region: "us-west-2", Type: "c5n.18xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 3.888}, "c5n.metal": {Region: "us-west-2", Type: "c5n.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("72"), GPU: 0, Inf: 0, Price: 3.888}, "c6a.large": {Region: "us-west-2", Type: "c6a.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0765}, "c6a.xlarge": {Region: "us-west-2", Type: "c6a.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.153}, "c6a.2xlarge": {Region: "us-west-2", Type: "c6a.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.306}, "c6a.4xlarge": {Region: "us-west-2", Type: "c6a.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.612}, "c6a.8xlarge": {Region: "us-west-2", Type: "c6a.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.224}, "c6a.12xlarge": {Region: "us-west-2", Type: "c6a.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 1.836}, "c6a.16xlarge": {Region: "us-west-2", Type: "c6a.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.448}, "c6a.24xlarge": {Region: "us-west-2", Type: "c6a.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 3.672}, "c6a.32xlarge": {Region: "us-west-2", Type: "c6a.32xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 4.896}, "c6a.48xlarge": {Region: "us-west-2", Type: "c6a.48xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("192"), GPU: 0, Inf: 0, Price: 7.344}, "c6a.metal": {Region: "us-west-2", Type: "c6a.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("192"), GPU: 0, Inf: 0, Price: 7.344}, "c6g.medium": {Region: "us-west-2", Type: "c6g.medium", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.034}, "c6g.large": {Region: "us-west-2", Type: "c6g.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.068}, "c6g.xlarge": {Region: "us-west-2", Type: "c6g.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.136}, "c6g.2xlarge": {Region: "us-west-2", Type: "c6g.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.272}, "c6g.4xlarge": {Region: "us-west-2", Type: "c6g.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.544}, "c6g.8xlarge": {Region: "us-west-2", Type: "c6g.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.088}, "c6g.12xlarge": {Region: "us-west-2", Type: "c6g.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 1.632}, "c6g.16xlarge": {Region: "us-west-2", Type: "c6g.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.176}, "c6g.metal": {Region: "us-west-2", Type: "c6g.metal", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.176}, "c6gd.medium": {Region: "us-west-2", Type: "c6gd.medium", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0384}, "c6gd.large": {Region: "us-west-2", Type: "c6gd.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0768}, "c6gd.xlarge": {Region: "us-west-2", Type: "c6gd.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1536}, "c6gd.2xlarge": {Region: "us-west-2", Type: "c6gd.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3072}, "c6gd.4xlarge": {Region: "us-west-2", Type: "c6gd.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.6144}, "c6gd.8xlarge": {Region: "us-west-2", Type: "c6gd.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.2288}, "c6gd.12xlarge": {Region: "us-west-2", Type: "c6gd.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 1.8432}, "c6gd.16xlarge": {Region: "us-west-2", Type: "c6gd.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.4576}, "c6gd.metal": {Region: "us-west-2", Type: "c6gd.metal", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.4576}, "c6gn.medium": {Region: "us-west-2", Type: "c6gn.medium", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0432}, "c6gn.large": {Region: "us-west-2", Type: "c6gn.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0864}, "c6gn.xlarge": {Region: "us-west-2", Type: "c6gn.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1728}, "c6gn.2xlarge": {Region: "us-west-2", Type: "c6gn.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3456}, "c6gn.4xlarge": {Region: "us-west-2", Type: "c6gn.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.6912}, "c6gn.8xlarge": {Region: "us-west-2", Type: "c6gn.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.3824}, "c6gn.12xlarge": {Region: "us-west-2", Type: "c6gn.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.0736}, "c6gn.16xlarge": {Region: "us-west-2", Type: "c6gn.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.7648}, "c6i.large": {Region: "us-west-2", Type: "c6i.large", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.085}, "c6i.xlarge": {Region: "us-west-2", Type: "c6i.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.17}, "c6i.2xlarge": {Region: "us-west-2", Type: "c6i.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.34}, "c6i.4xlarge": {Region: "us-west-2", Type: "c6i.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.68}, "c6i.8xlarge": {Region: "us-west-2", Type: "c6i.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.36}, "c6i.12xlarge": {Region: "us-west-2", Type: "c6i.12xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.04}, "c6i.16xlarge": {Region: "us-west-2", Type: "c6i.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.72}, "c6i.24xlarge": {Region: "us-west-2", Type: "c6i.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.08}, "c6i.32xlarge": {Region: "us-west-2", Type: "c6i.32xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 5.44}, "c6i.metal": {Region: "us-west-2", Type: "c6i.metal", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 5.44}, "cc2.8xlarge": {Region: "us-west-2", Type: "cc2.8xlarge", Memory: kresource.MustParse("61952Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.0}, "cr1.8xlarge": {Region: "us-west-2", Type: "cr1.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 3.5}, "d2.xlarge": {Region: "us-west-2", Type: "d2.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.69}, "d2.2xlarge": {Region: "us-west-2", Type: "d2.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.38}, "d2.4xlarge": {Region: "us-west-2", Type: "d2.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 2.76}, "d2.8xlarge": {Region: "us-west-2", Type: "d2.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("36"), GPU: 0, Inf: 0, Price: 5.52}, "d3.xlarge": {Region: "us-west-2", Type: "d3.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.499}, "d3.2xlarge": {Region: "us-west-2", Type: "d3.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.999}, "d3.4xlarge": {Region: "us-west-2", Type: "d3.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.998}, "d3.8xlarge": {Region: "us-west-2", Type: "d3.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 3.99552}, "d3en.xlarge": {Region: "us-west-2", Type: "d3en.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.526}, "d3en.2xlarge": {Region: "us-west-2", Type: "d3en.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.051}, "d3en.4xlarge": {Region: "us-west-2", Type: "d3en.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 2.103}, "d3en.6xlarge": {Region: "us-west-2", Type: "d3en.6xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 0, Price: 3.154}, "d3en.8xlarge": {Region: "us-west-2", Type: "d3en.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 4.20576}, "d3en.12xlarge": {Region: "us-west-2", Type: "d3en.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 6.30864}, "dl1.24xlarge": {Region: "us-west-2", Type: "dl1.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 13.10904}, "f1.2xlarge": {Region: "us-west-2", Type: "f1.2xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.65}, "f1.4xlarge": {Region: "us-west-2", Type: "f1.4xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 3.3}, "f1.16xlarge": {Region: "us-west-2", Type: "f1.16xlarge", Memory: kresource.MustParse("999424Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 13.2}, "g2.2xlarge": {Region: "us-west-2", Type: "g2.2xlarge", Memory: kresource.MustParse("15360Mi"), CPU: kresource.MustParse("8"), GPU: 1, Inf: 0, Price: 0.65}, "g2.8xlarge": {Region: "us-west-2", Type: "g2.8xlarge", Memory: kresource.MustParse("61440Mi"), CPU: kresource.MustParse("32"), GPU: 4, Inf: 0, Price: 2.6}, "g3.4xlarge": {Region: "us-west-2", Type: "g3.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 1, Inf: 0, Price: 1.14}, "g3.8xlarge": {Region: "us-west-2", Type: "g3.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 2, Inf: 0, Price: 2.28}, "g3.16xlarge": {Region: "us-west-2", Type: "g3.16xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("64"), GPU: 4, Inf: 0, Price: 4.56}, "g3s.xlarge": {Region: "us-west-2", Type: "g3s.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 1, Inf: 0, Price: 0.75}, "g4ad.xlarge": {Region: "us-west-2", Type: "g4ad.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 1, Inf: 0, Price: 0.37853}, "g4ad.2xlarge": {Region: "us-west-2", Type: "g4ad.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 1, Inf: 0, Price: 0.54117}, "g4ad.4xlarge": {Region: "us-west-2", Type: "g4ad.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 1, Inf: 0, Price: 0.867}, "g4ad.8xlarge": {Region: "us-west-2", Type: "g4ad.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 2, Inf: 0, Price: 1.734}, "g4ad.16xlarge": {Region: "us-west-2", Type: "g4ad.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 4, Inf: 0, Price: 3.468}, "g4dn.xlarge": {Region: "us-west-2", Type: "g4dn.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 1, Inf: 0, Price: 0.526}, "g4dn.2xlarge": {Region: "us-west-2", Type: "g4dn.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 1, Inf: 0, Price: 0.752}, "g4dn.4xlarge": {Region: "us-west-2", Type: "g4dn.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 1, Inf: 0, Price: 1.204}, "g4dn.8xlarge": {Region: "us-west-2", Type: "g4dn.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 1, Inf: 0, Price: 2.176}, "g4dn.12xlarge": {Region: "us-west-2", Type: "g4dn.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 4, Inf: 0, Price: 3.912}, "g4dn.16xlarge": {Region: "us-west-2", Type: "g4dn.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 1, Inf: 0, Price: 4.352}, "g4dn.metal": {Region: "us-west-2", Type: "g4dn.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 8, Inf: 0, Price: 7.824}, "g5.xlarge": {Region: "us-west-2", Type: "g5.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 1, Inf: 0, Price: 1.006}, "g5.2xlarge": {Region: "us-west-2", Type: "g5.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 1, Inf: 0, Price: 1.212}, "g5.4xlarge": {Region: "us-west-2", Type: "g5.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 1, Inf: 0, Price: 1.624}, "g5.8xlarge": {Region: "us-west-2", Type: "g5.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 1, Inf: 0, Price: 2.448}, "g5.12xlarge": {Region: "us-west-2", Type: "g5.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 4, Inf: 0, Price: 5.672}, "g5.16xlarge": {Region: "us-west-2", Type: "g5.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 1, Inf: 0, Price: 4.096}, "g5.24xlarge": {Region: "us-west-2", Type: "g5.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 4, Inf: 0, Price: 8.144}, "g5.48xlarge": {Region: "us-west-2", Type: "g5.48xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("192"), GPU: 8, Inf: 0, Price: 16.288}, "g5g.xlarge": {Region: "us-west-2", Type: "g5g.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 1, Inf: 0, Price: 0.42}, "g5g.2xlarge": {Region: "us-west-2", Type: "g5g.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 1, Inf: 0, Price: 0.556}, "g5g.4xlarge": {Region: "us-west-2", Type: "g5g.4xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("16"), GPU: 1, Inf: 0, Price: 0.828}, "g5g.8xlarge": {Region: "us-west-2", Type: "g5g.8xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("32"), GPU: 1, Inf: 0, Price: 1.372}, "g5g.16xlarge": {Region: "us-west-2", Type: "g5g.16xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 2, Inf: 0, Price: 2.744}, "g5g.metal": {Region: "us-west-2", Type: "g5g.metal", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("64"), GPU: 2, Inf: 0, Price: 2.744}, "h1.2xlarge": {Region: "us-west-2", Type: "h1.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.468}, "h1.4xlarge": {Region: "us-west-2", Type: "h1.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.936}, "h1.8xlarge": {Region: "us-west-2", Type: "h1.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.872}, "h1.16xlarge": {Region: "us-west-2", Type: "h1.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.744}, "hs1.8xlarge": {Region: "us-west-2", Type: "hs1.8xlarge", Memory: kresource.MustParse("119808Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 4.6}, "i2.xlarge": {Region: "us-west-2", Type: "i2.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.853}, "i2.2xlarge": {Region: "us-west-2", Type: "i2.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.705}, "i2.4xlarge": {Region: "us-west-2", Type: "i2.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 3.41}, "i2.8xlarge": {Region: "us-west-2", Type: "i2.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 6.82}, "i3.large": {Region: "us-west-2", Type: "i3.large", Memory: kresource.MustParse("15616Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.156}, "i3.xlarge": {Region: "us-west-2", Type: "i3.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.312}, "i3.2xlarge": {Region: "us-west-2", Type: "i3.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.624}, "i3.4xlarge": {Region: "us-west-2", Type: "i3.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.248}, "i3.8xlarge": {Region: "us-west-2", Type: "i3.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.496}, "i3.16xlarge": {Region: "us-west-2", Type: "i3.16xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.992}, "i3.metal": {Region: "us-west-2", Type: "i3.metal", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.992}, "i3en.large": {Region: "us-west-2", Type: "i3en.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.226}, "i3en.xlarge": {Region: "us-west-2", Type: "i3en.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.452}, "i3en.2xlarge": {Region: "us-west-2", Type: "i3en.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.904}, "i3en.3xlarge": {Region: "us-west-2", Type: "i3en.3xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("12"), GPU: 0, Inf: 0, Price: 1.356}, "i3en.6xlarge": {Region: "us-west-2", Type: "i3en.6xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 0, Price: 2.712}, "i3en.12xlarge": {Region: "us-west-2", Type: "i3en.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 5.424}, "i3en.24xlarge": {Region: "us-west-2", Type: "i3en.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 10.848}, "i3en.metal": {Region: "us-west-2", Type: "i3en.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 10.848}, "im4gn.large": {Region: "us-west-2", Type: "im4gn.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.1819}, "im4gn.xlarge": {Region: "us-west-2", Type: "im4gn.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.36379}, "im4gn.2xlarge": {Region: "us-west-2", Type: "im4gn.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.72758}, "im4gn.4xlarge": {Region: "us-west-2", Type: "im4gn.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.45517}, "im4gn.8xlarge": {Region: "us-west-2", Type: "im4gn.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.91034}, "im4gn.16xlarge": {Region: "us-west-2", Type: "im4gn.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.82067}, "inf1.xlarge": {Region: "us-west-2", Type: "inf1.xlarge", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 1, Price: 0.228}, "inf1.2xlarge": {Region: "us-west-2", Type: "inf1.2xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 1, Price: 0.362}, "inf1.6xlarge": {Region: "us-west-2", Type: "inf1.6xlarge", Memory: kresource.MustParse("49152Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 4, Price: 1.18}, "inf1.24xlarge": {Region: "us-west-2", Type: "inf1.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 16, Price: 4.721}, "is4gen.medium": {Region: "us-west-2", Type: "is4gen.medium", Memory: kresource.MustParse("6144Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.14408}, "is4gen.large": {Region: "us-west-2", Type: "is4gen.large", Memory: kresource.MustParse("12288Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.28815}, "is4gen.xlarge": {Region: "us-west-2", Type: "is4gen.xlarge", Memory: kresource.MustParse("24576Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.5763}, "is4gen.2xlarge": {Region: "us-west-2", Type: "is4gen.2xlarge", Memory: kresource.MustParse("49152Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.1526}, "is4gen.4xlarge": {Region: "us-west-2", Type: "is4gen.4xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 2.3052}, "is4gen.8xlarge": {Region: "us-west-2", Type: "is4gen.8xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 4.6104}, "m1.small": {Region: "us-west-2", Type: "m1.small", Memory: kresource.MustParse("1740Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.044}, "m1.medium": {Region: "us-west-2", Type: "m1.medium", Memory: kresource.MustParse("3840Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.087}, "m1.large": {Region: "us-west-2", Type: "m1.large", Memory: kresource.MustParse("7680Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.175}, "m1.xlarge": {Region: "us-west-2", Type: "m1.xlarge", Memory: kresource.MustParse("15360Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.35}, "m2.xlarge": {Region: "us-west-2", Type: "m2.xlarge", Memory: kresource.MustParse("17510Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.245}, "m2.2xlarge": {Region: "us-west-2", Type: "m2.2xlarge", Memory: kresource.MustParse("35020Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.49}, "m2.4xlarge": {Region: "us-west-2", Type: "m2.4xlarge", Memory: kresource.MustParse("70041Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.98}, "m3.medium": {Region: "us-west-2", Type: "m3.medium", Memory: kresource.MustParse("3840Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.067}, "m3.large": {Region: "us-west-2", Type: "m3.large", Memory: kresource.MustParse("7680Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.133}, "m3.xlarge": {Region: "us-west-2", Type: "m3.xlarge", Memory: kresource.MustParse("15360Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.266}, "m3.2xlarge": {Region: "us-west-2", Type: "m3.2xlarge", Memory: kresource.MustParse("30720Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.532}, "m4.large": {Region: "us-west-2", Type: "m4.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.1}, "m4.xlarge": {Region: "us-west-2", Type: "m4.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.2}, "m4.2xlarge": {Region: "us-west-2", Type: "m4.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.4}, "m4.4xlarge": {Region: "us-west-2", Type: "m4.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.8}, "m4.10xlarge": {Region: "us-west-2", Type: "m4.10xlarge", Memory: kresource.MustParse("163840Mi"), CPU: kresource.MustParse("40"), GPU: 0, Inf: 0, Price: 2.0}, "m4.16xlarge": {Region: "us-west-2", Type: "m4.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.2}, "m5.large": {Region: "us-west-2", Type: "m5.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.096}, "m5.xlarge": {Region: "us-west-2", Type: "m5.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.192}, "m5.2xlarge": {Region: "us-west-2", Type: "m5.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.384}, "m5.4xlarge": {Region: "us-west-2", Type: "m5.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.768}, "m5.8xlarge": {Region: "us-west-2", Type: "m5.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.536}, "m5.12xlarge": {Region: "us-west-2", Type: "m5.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.304}, "m5.16xlarge": {Region: "us-west-2", Type: "m5.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.072}, "m5.24xlarge": {Region: "us-west-2", Type: "m5.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.608}, "m5.metal": {Region: "us-west-2", Type: "m5.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.608}, "m5a.large": {Region: "us-west-2", Type: "m5a.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.086}, "m5a.xlarge": {Region: "us-west-2", Type: "m5a.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.172}, "m5a.2xlarge": {Region: "us-west-2", Type: "m5a.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.344}, "m5a.4xlarge": {Region: "us-west-2", Type: "m5a.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.688}, "m5a.8xlarge": {Region: "us-west-2", Type: "m5a.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.376}, "m5a.12xlarge": {Region: "us-west-2", Type: "m5a.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.064}, "m5a.16xlarge": {Region: "us-west-2", Type: "m5a.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.752}, "m5a.24xlarge": {Region: "us-west-2", Type: "m5a.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.128}, "m5ad.large": {Region: "us-west-2", Type: "m5ad.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.103}, "m5ad.xlarge": {Region: "us-west-2", Type: "m5ad.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.206}, "m5ad.2xlarge": {Region: "us-west-2", Type: "m5ad.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.412}, "m5ad.4xlarge": {Region: "us-west-2", Type: "m5ad.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.824}, "m5ad.8xlarge": {Region: "us-west-2", Type: "m5ad.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.648}, "m5ad.12xlarge": {Region: "us-west-2", Type: "m5ad.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.472}, "m5ad.16xlarge": {Region: "us-west-2", Type: "m5ad.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.296}, "m5ad.24xlarge": {Region: "us-west-2", Type: "m5ad.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.944}, "m5d.large": {Region: "us-west-2", Type: "m5d.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.113}, "m5d.xlarge": {Region: "us-west-2", Type: "m5d.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.226}, "m5d.2xlarge": {Region: "us-west-2", Type: "m5d.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.452}, "m5d.4xlarge": {Region: "us-west-2", Type: "m5d.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.904}, "m5d.8xlarge": {Region: "us-west-2", Type: "m5d.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.808}, "m5d.12xlarge": {Region: "us-west-2", Type: "m5d.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.712}, "m5d.16xlarge": {Region: "us-west-2", Type: "m5d.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.616}, "m5d.24xlarge": {Region: "us-west-2", Type: "m5d.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.424}, "m5d.metal": {Region: "us-west-2", Type: "m5d.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.424}, "m5dn.large": {Region: "us-west-2", Type: "m5dn.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.136}, "m5dn.xlarge": {Region: "us-west-2", Type: "m5dn.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.272}, "m5dn.2xlarge": {Region: "us-west-2", Type: "m5dn.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.544}, "m5dn.4xlarge": {Region: "us-west-2", Type: "m5dn.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.088}, "m5dn.8xlarge": {Region: "us-west-2", Type: "m5dn.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.176}, "m5dn.12xlarge": {Region: "us-west-2", Type: "m5dn.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.264}, "m5dn.16xlarge": {Region: "us-west-2", Type: "m5dn.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.352}, "m5dn.24xlarge": {Region: "us-west-2", Type: "m5dn.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.528}, "m5dn.metal": {Region: "us-west-2", Type: "m5dn.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.528}, "m5n.large": {Region: "us-west-2", Type: "m5n.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.119}, "m5n.xlarge": {Region: "us-west-2", Type: "m5n.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.238}, "m5n.2xlarge": {Region: "us-west-2", Type: "m5n.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.476}, "m5n.4xlarge": {Region: "us-west-2", Type: "m5n.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.952}, "m5n.8xlarge": {Region: "us-west-2", Type: "m5n.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.904}, "m5n.12xlarge": {Region: "us-west-2", Type: "m5n.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.856}, "m5n.16xlarge": {Region: "us-west-2", Type: "m5n.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.808}, "m5n.24xlarge": {Region: "us-west-2", Type: "m5n.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.712}, "m5n.metal": {Region: "us-west-2", Type: "m5n.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.712}, "m5zn.large": {Region: "us-west-2", Type: "m5zn.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.1652}, "m5zn.xlarge": {Region: "us-west-2", Type: "m5zn.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.3303}, "m5zn.2xlarge": {Region: "us-west-2", Type: "m5zn.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.6607}, "m5zn.3xlarge": {Region: "us-west-2", Type: "m5zn.3xlarge", Memory: kresource.MustParse("49152Mi"), CPU: kresource.MustParse("12"), GPU: 0, Inf: 0, Price: 0.991}, "m5zn.6xlarge": {Region: "us-west-2", Type: "m5zn.6xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 0, Price: 1.982}, "m5zn.12xlarge": {Region: "us-west-2", Type: "m5zn.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.9641}, "m5zn.metal": {Region: "us-west-2", Type: "m5zn.metal", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.9641}, "m6a.large": {Region: "us-west-2", Type: "m6a.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0864}, "m6a.xlarge": {Region: "us-west-2", Type: "m6a.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1728}, "m6a.2xlarge": {Region: "us-west-2", Type: "m6a.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3456}, "m6a.4xlarge": {Region: "us-west-2", Type: "m6a.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.6912}, "m6a.8xlarge": {Region: "us-west-2", Type: "m6a.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.3824}, "m6a.12xlarge": {Region: "us-west-2", Type: "m6a.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.0736}, "m6a.16xlarge": {Region: "us-west-2", Type: "m6a.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.7648}, "m6a.24xlarge": {Region: "us-west-2", Type: "m6a.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.1472}, "m6a.32xlarge": {Region: "us-west-2", Type: "m6a.32xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 5.5296}, "m6a.48xlarge": {Region: "us-west-2", Type: "m6a.48xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("192"), GPU: 0, Inf: 0, Price: 8.2944}, "m6a.metal": {Region: "us-west-2", Type: "m6a.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("192"), GPU: 0, Inf: 0, Price: 8.2944}, "m6g.medium": {Region: "us-west-2", Type: "m6g.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0385}, "m6g.large": {Region: "us-west-2", Type: "m6g.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.077}, "m6g.xlarge": {Region: "us-west-2", Type: "m6g.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.154}, "m6g.2xlarge": {Region: "us-west-2", Type: "m6g.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.308}, "m6g.4xlarge": {Region: "us-west-2", Type: "m6g.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.616}, "m6g.8xlarge": {Region: "us-west-2", Type: "m6g.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.232}, "m6g.12xlarge": {Region: "us-west-2", Type: "m6g.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 1.848}, "m6g.16xlarge": {Region: "us-west-2", Type: "m6g.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.464}, "m6g.metal": {Region: "us-west-2", Type: "m6g.metal", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.464}, "m6gd.medium": {Region: "us-west-2", Type: "m6gd.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0452}, "m6gd.large": {Region: "us-west-2", Type: "m6gd.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0904}, "m6gd.xlarge": {Region: "us-west-2", Type: "m6gd.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1808}, "m6gd.2xlarge": {Region: "us-west-2", Type: "m6gd.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3616}, "m6gd.4xlarge": {Region: "us-west-2", Type: "m6gd.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.7232}, "m6gd.8xlarge": {Region: "us-west-2", Type: "m6gd.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.4464}, "m6gd.12xlarge": {Region: "us-west-2", Type: "m6gd.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.1696}, "m6gd.16xlarge": {Region: "us-west-2", Type: "m6gd.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.8928}, "m6gd.metal": {Region: "us-west-2", Type: "m6gd.metal", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 2.8928}, "m6i.large": {Region: "us-west-2", Type: "m6i.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.096}, "m6i.xlarge": {Region: "us-west-2", Type: "m6i.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.192}, "m6i.2xlarge": {Region: "us-west-2", Type: "m6i.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.384}, "m6i.4xlarge": {Region: "us-west-2", Type: "m6i.4xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.768}, "m6i.8xlarge": {Region: "us-west-2", Type: "m6i.8xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.536}, "m6i.12xlarge": {Region: "us-west-2", Type: "m6i.12xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.304}, "m6i.16xlarge": {Region: "us-west-2", Type: "m6i.16xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.072}, "m6i.24xlarge": {Region: "us-west-2", Type: "m6i.24xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 4.608}, "m6i.32xlarge": {Region: "us-west-2", Type: "m6i.32xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 6.144}, "m6i.metal": {Region: "us-west-2", Type: "m6i.metal", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 6.144}, "p2.xlarge": {Region: "us-west-2", Type: "p2.xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("4"), GPU: 1, Inf: 0, Price: 0.9}, "p2.8xlarge": {Region: "us-west-2", Type: "p2.8xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("32"), GPU: 8, Inf: 0, Price: 7.2}, "p2.16xlarge": {Region: "us-west-2", Type: "p2.16xlarge", Memory: kresource.MustParse("749568Mi"), CPU: kresource.MustParse("64"), GPU: 16, Inf: 0, Price: 14.4}, "p3.2xlarge": {Region: "us-west-2", Type: "p3.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 1, Inf: 0, Price: 3.06}, "p3.8xlarge": {Region: "us-west-2", Type: "p3.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 4, Inf: 0, Price: 12.24}, "p3.16xlarge": {Region: "us-west-2", Type: "p3.16xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("64"), GPU: 8, Inf: 0, Price: 24.48}, "p3dn.24xlarge": {Region: "us-west-2", Type: "p3dn.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 8, Inf: 0, Price: 31.212}, "p4d.24xlarge": {Region: "us-west-2", Type: "p4d.24xlarge", Memory: kresource.MustParse("1179648Mi"), CPU: kresource.MustParse("96"), GPU: 8, Inf: 0, Price: 32.7726}, "r3.large": {Region: "us-west-2", Type: "r3.large", Memory: kresource.MustParse("15616Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.166}, "r3.xlarge": {Region: "us-west-2", Type: "r3.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.333}, "r3.2xlarge": {Region: "us-west-2", Type: "r3.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.665}, "r3.4xlarge": {Region: "us-west-2", Type: "r3.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.33}, "r3.8xlarge": {Region: "us-west-2", Type: "r3.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.66}, "r4.large": {Region: "us-west-2", Type: "r4.large", Memory: kresource.MustParse("15616Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.133}, "r4.xlarge": {Region: "us-west-2", Type: "r4.xlarge", Memory: kresource.MustParse("31232Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.266}, "r4.2xlarge": {Region: "us-west-2", Type: "r4.2xlarge", Memory: kresource.MustParse("62464Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.532}, "r4.4xlarge": {Region: "us-west-2", Type: "r4.4xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.064}, "r4.8xlarge": {Region: "us-west-2", Type: "r4.8xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.128}, "r4.16xlarge": {Region: "us-west-2", Type: "r4.16xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.256}, "r5.large": {Region: "us-west-2", Type: "r5.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.126}, "r5.xlarge": {Region: "us-west-2", Type: "r5.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.252}, "r5.2xlarge": {Region: "us-west-2", Type: "r5.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.504}, "r5.4xlarge": {Region: "us-west-2", Type: "r5.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.008}, "r5.8xlarge": {Region: "us-west-2", Type: "r5.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.016}, "r5.12xlarge": {Region: "us-west-2", Type: "r5.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.024}, "r5.16xlarge": {Region: "us-west-2", Type: "r5.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.032}, "r5.24xlarge": {Region: "us-west-2", Type: "r5.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.048}, "r5.metal": {Region: "us-west-2", Type: "r5.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.048}, "r5a.large": {Region: "us-west-2", Type: "r5a.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.113}, "r5a.xlarge": {Region: "us-west-2", Type: "r5a.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.226}, "r5a.2xlarge": {Region: "us-west-2", Type: "r5a.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.452}, "r5a.4xlarge": {Region: "us-west-2", Type: "r5a.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.904}, "r5a.8xlarge": {Region: "us-west-2", Type: "r5a.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.808}, "r5a.12xlarge": {Region: "us-west-2", Type: "r5a.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.712}, "r5a.16xlarge": {Region: "us-west-2", Type: "r5a.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.616}, "r5a.24xlarge": {Region: "us-west-2", Type: "r5a.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.424}, "r5ad.large": {Region: "us-west-2", Type: "r5ad.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.131}, "r5ad.xlarge": {Region: "us-west-2", Type: "r5ad.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.262}, "r5ad.2xlarge": {Region: "us-west-2", Type: "r5ad.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.524}, "r5ad.4xlarge": {Region: "us-west-2", Type: "r5ad.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.048}, "r5ad.8xlarge": {Region: "us-west-2", Type: "r5ad.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.096}, "r5ad.12xlarge": {Region: "us-west-2", Type: "r5ad.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.144}, "r5ad.16xlarge": {Region: "us-west-2", Type: "r5ad.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.192}, "r5ad.24xlarge": {Region: "us-west-2", Type: "r5ad.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.288}, "r5b.large": {Region: "us-west-2", Type: "r5b.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.149}, "r5b.xlarge": {Region: "us-west-2", Type: "r5b.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.298}, "r5b.2xlarge": {Region: "us-west-2", Type: "r5b.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.596}, "r5b.4xlarge": {Region: "us-west-2", Type: "r5b.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.192}, "r5b.8xlarge": {Region: "us-west-2", Type: "r5b.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.384}, "r5b.12xlarge": {Region: "us-west-2", Type: "r5b.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.576}, "r5b.16xlarge": {Region: "us-west-2", Type: "r5b.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.768}, "r5b.24xlarge": {Region: "us-west-2", Type: "r5b.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.152}, "r5b.metal": {Region: "us-west-2", Type: "r5b.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.152}, "r5d.large": {Region: "us-west-2", Type: "r5d.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.144}, "r5d.xlarge": {Region: "us-west-2", Type: "r5d.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.288}, "r5d.2xlarge": {Region: "us-west-2", Type: "r5d.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.576}, "r5d.4xlarge": {Region: "us-west-2", Type: "r5d.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.152}, "r5d.8xlarge": {Region: "us-west-2", Type: "r5d.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.304}, "r5d.12xlarge": {Region: "us-west-2", Type: "r5d.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.456}, "r5d.16xlarge": {Region: "us-west-2", Type: "r5d.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.608}, "r5d.24xlarge": {Region: "us-west-2", Type: "r5d.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.912}, "r5d.metal": {Region: "us-west-2", Type: "r5d.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.912}, "r5dn.large": {Region: "us-west-2", Type: "r5dn.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.167}, "r5dn.xlarge": {Region: "us-west-2", Type: "r5dn.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.334}, "r5dn.2xlarge": {Region: "us-west-2", Type: "r5dn.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.668}, "r5dn.4xlarge": {Region: "us-west-2", Type: "r5dn.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.336}, "r5dn.8xlarge": {Region: "us-west-2", Type: "r5dn.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.672}, "r5dn.12xlarge": {Region: "us-west-2", Type: "r5dn.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.008}, "r5dn.16xlarge": {Region: "us-west-2", Type: "r5dn.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.344}, "r5dn.24xlarge": {Region: "us-west-2", Type: "r5dn.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.016}, "r5dn.metal": {Region: "us-west-2", Type: "r5dn.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 8.016}, "r5n.large": {Region: "us-west-2", Type: "r5n.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.149}, "r5n.xlarge": {Region: "us-west-2", Type: "r5n.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.298}, "r5n.2xlarge": {Region: "us-west-2", Type: "r5n.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.596}, "r5n.4xlarge": {Region: "us-west-2", Type: "r5n.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.192}, "r5n.8xlarge": {Region: "us-west-2", Type: "r5n.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.384}, "r5n.12xlarge": {Region: "us-west-2", Type: "r5n.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.576}, "r5n.16xlarge": {Region: "us-west-2", Type: "r5n.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.768}, "r5n.24xlarge": {Region: "us-west-2", Type: "r5n.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.152}, "r5n.metal": {Region: "us-west-2", Type: "r5n.metal", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 7.152}, "r6g.medium": {Region: "us-west-2", Type: "r6g.medium", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0504}, "r6g.large": {Region: "us-west-2", Type: "r6g.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.1008}, "r6g.xlarge": {Region: "us-west-2", Type: "r6g.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.2016}, "r6g.2xlarge": {Region: "us-west-2", Type: "r6g.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.4032}, "r6g.4xlarge": {Region: "us-west-2", Type: "r6g.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.8064}, "r6g.8xlarge": {Region: "us-west-2", Type: "r6g.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.6128}, "r6g.12xlarge": {Region: "us-west-2", Type: "r6g.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.4192}, "r6g.16xlarge": {Region: "us-west-2", Type: "r6g.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.2256}, "r6g.metal": {Region: "us-west-2", Type: "r6g.metal", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.2256}, "r6gd.medium": {Region: "us-west-2", Type: "r6gd.medium", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0576}, "r6gd.large": {Region: "us-west-2", Type: "r6gd.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.1152}, "r6gd.xlarge": {Region: "us-west-2", Type: "r6gd.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.2304}, "r6gd.2xlarge": {Region: "us-west-2", Type: "r6gd.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.4608}, "r6gd.4xlarge": {Region: "us-west-2", Type: "r6gd.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 0.9216}, "r6gd.8xlarge": {Region: "us-west-2", Type: "r6gd.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 1.8432}, "r6gd.12xlarge": {Region: "us-west-2", Type: "r6gd.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 2.7648}, "r6gd.16xlarge": {Region: "us-west-2", Type: "r6gd.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.6864}, "r6gd.metal": {Region: "us-west-2", Type: "r6gd.metal", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 3.6864}, "r6i.large": {Region: "us-west-2", Type: "r6i.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.126}, "r6i.xlarge": {Region: "us-west-2", Type: "r6i.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.252}, "r6i.2xlarge": {Region: "us-west-2", Type: "r6i.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.504}, "r6i.4xlarge": {Region: "us-west-2", Type: "r6i.4xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.008}, "r6i.8xlarge": {Region: "us-west-2", Type: "r6i.8xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.016}, "r6i.12xlarge": {Region: "us-west-2", Type: "r6i.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 3.024}, "r6i.16xlarge": {Region: "us-west-2", Type: "r6i.16xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 4.032}, "r6i.24xlarge": {Region: "us-west-2", Type: "r6i.24xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 6.048}, "r6i.32xlarge": {Region: "us-west-2", Type: "r6i.32xlarge", Memory: kresource.MustParse("1048576Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 8.064}, "r6i.metal": {Region: "us-west-2", Type: "r6i.metal", Memory: kresource.MustParse("1048576Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 8.064}, "t1.micro": {Region: "us-west-2", Type: "t1.micro", Memory: kresource.MustParse("627Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.02}, "t2.nano": {Region: "us-west-2", Type: "t2.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0058}, "t2.micro": {Region: "us-west-2", Type: "t2.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0116}, "t2.small": {Region: "us-west-2", Type: "t2.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.023}, "t2.medium": {Region: "us-west-2", Type: "t2.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0464}, "t2.large": {Region: "us-west-2", Type: "t2.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0928}, "t2.xlarge": {Region: "us-west-2", Type: "t2.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1856}, "t2.2xlarge": {Region: "us-west-2", Type: "t2.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3712}, "t3.nano": {Region: "us-west-2", Type: "t3.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0052}, "t3.micro": {Region: "us-west-2", Type: "t3.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0104}, "t3.small": {Region: "us-west-2", Type: "t3.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0208}, "t3.medium": {Region: "us-west-2", Type: "t3.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0416}, "t3.large": {Region: "us-west-2", Type: "t3.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0832}, "t3.xlarge": {Region: "us-west-2", Type: "t3.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1664}, "t3.2xlarge": {Region: "us-west-2", Type: "t3.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3328}, "t3a.nano": {Region: "us-west-2", Type: "t3a.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0047}, "t3a.micro": {Region: "us-west-2", Type: "t3a.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0094}, "t3a.small": {Region: "us-west-2", Type: "t3a.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0188}, "t3a.medium": {Region: "us-west-2", Type: "t3a.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0376}, "t3a.large": {Region: "us-west-2", Type: "t3a.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0752}, "t3a.xlarge": {Region: "us-west-2", Type: "t3a.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1504}, "t3a.2xlarge": {Region: "us-west-2", Type: "t3a.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.3008}, "t4g.nano": {Region: "us-west-2", Type: "t4g.nano", Memory: kresource.MustParse("512Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0042}, "t4g.micro": {Region: "us-west-2", Type: "t4g.micro", Memory: kresource.MustParse("1024Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0084}, "t4g.small": {Region: "us-west-2", Type: "t4g.small", Memory: kresource.MustParse("2048Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0168}, "t4g.medium": {Region: "us-west-2", Type: "t4g.medium", Memory: kresource.MustParse("4096Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0336}, "t4g.large": {Region: "us-west-2", Type: "t4g.large", Memory: kresource.MustParse("8192Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.0672}, "t4g.xlarge": {Region: "us-west-2", Type: "t4g.xlarge", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.1344}, "t4g.2xlarge": {Region: "us-west-2", Type: "t4g.2xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.2688}, "u-12tb1.112xlarge": {Region: "us-west-2", Type: "u-12tb1.112xlarge", Memory: kresource.MustParse("12582912Mi"), CPU: kresource.MustParse("448"), GPU: 0, Inf: 0, Price: 109.2}, "u-3tb1.56xlarge": {Region: "us-west-2", Type: "u-3tb1.56xlarge", Memory: kresource.MustParse("3145728Mi"), CPU: kresource.MustParse("224"), GPU: 0, Inf: 0, Price: 27.3}, "u-6tb1.56xlarge": {Region: "us-west-2", Type: "u-6tb1.56xlarge", Memory: kresource.MustParse("6291456Mi"), CPU: kresource.MustParse("224"), GPU: 0, Inf: 0, Price: 46.40391}, "u-6tb1.112xlarge": {Region: "us-west-2", Type: "u-6tb1.112xlarge", Memory: kresource.MustParse("6291456Mi"), CPU: kresource.MustParse("448"), GPU: 0, Inf: 0, Price: 54.6}, "u-9tb1.112xlarge": {Region: "us-west-2", Type: "u-9tb1.112xlarge", Memory: kresource.MustParse("9437184Mi"), CPU: kresource.MustParse("448"), GPU: 0, Inf: 0, Price: 81.9}, "vt1.3xlarge": {Region: "us-west-2", Type: "vt1.3xlarge", Memory: kresource.MustParse("24576Mi"), CPU: kresource.MustParse("12"), GPU: 0, Inf: 0, Price: 0.65}, "vt1.6xlarge": {Region: "us-west-2", Type: "vt1.6xlarge", Memory: kresource.MustParse("49152Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 0, Price: 1.3}, "vt1.24xlarge": {Region: "us-west-2", Type: "vt1.24xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 5.2}, "x1.16xlarge": {Region: "us-west-2", Type: "x1.16xlarge", Memory: kresource.MustParse("999424Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 6.669}, "x1.32xlarge": {Region: "us-west-2", Type: "x1.32xlarge", Memory: kresource.MustParse("1998848Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 13.338}, "x1e.xlarge": {Region: "us-west-2", Type: "x1e.xlarge", Memory: kresource.MustParse("124928Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.834}, "x1e.2xlarge": {Region: "us-west-2", Type: "x1e.2xlarge", Memory: kresource.MustParse("249856Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.668}, "x1e.4xlarge": {Region: "us-west-2", Type: "x1e.4xlarge", Memory: kresource.MustParse("499712Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 3.336}, "x1e.8xlarge": {Region: "us-west-2", Type: "x1e.8xlarge", Memory: kresource.MustParse("999424Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 6.672}, "x1e.16xlarge": {Region: "us-west-2", Type: "x1e.16xlarge", Memory: kresource.MustParse("1998848Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 13.344}, "x1e.32xlarge": {Region: "us-west-2", Type: "x1e.32xlarge", Memory: kresource.MustParse("3997696Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 26.688}, "x2gd.medium": {Region: "us-west-2", Type: "x2gd.medium", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("1"), GPU: 0, Inf: 0, Price: 0.0835}, "x2gd.large": {Region: "us-west-2", Type: "x2gd.large", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.167}, "x2gd.xlarge": {Region: "us-west-2", Type: "x2gd.xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.334}, "x2gd.2xlarge": {Region: "us-west-2", Type: "x2gd.2xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.668}, "x2gd.4xlarge": {Region: "us-west-2", Type: "x2gd.4xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 1.336}, "x2gd.8xlarge": {Region: "us-west-2", Type: "x2gd.8xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 2.672}, "x2gd.12xlarge": {Region: "us-west-2", Type: "x2gd.12xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.008}, "x2gd.16xlarge": {Region: "us-west-2", Type: "x2gd.16xlarge", Memory: kresource.MustParse("1048576Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.344}, "x2gd.metal": {Region: "us-west-2", Type: "x2gd.metal", Memory: kresource.MustParse("1048576Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 5.344}, "x2iedn.xlarge": {Region: "us-west-2", Type: "x2iedn.xlarge", Memory: kresource.MustParse("131072Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.83363}, "x2iedn.2xlarge": {Region: "us-west-2", Type: "x2iedn.2xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.66725}, "x2iedn.4xlarge": {Region: "us-west-2", Type: "x2iedn.4xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 3.3345}, "x2iedn.8xlarge": {Region: "us-west-2", Type: "x2iedn.8xlarge", Memory: kresource.MustParse("1048576Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 6.669}, "x2iedn.16xlarge": {Region: "us-west-2", Type: "x2iedn.16xlarge", Memory: kresource.MustParse("2097152Mi"), CPU: kresource.MustParse("64"), GPU: 0, Inf: 0, Price: 13.338}, "x2iedn.24xlarge": {Region: "us-west-2", Type: "x2iedn.24xlarge", Memory: kresource.MustParse("3145728Mi"), CPU: kresource.MustParse("96"), GPU: 0, Inf: 0, Price: 20.007}, "x2iedn.32xlarge": {Region: "us-west-2", Type: "x2iedn.32xlarge", Memory: kresource.MustParse("4194304Mi"), CPU: kresource.MustParse("128"), GPU: 0, Inf: 0, Price: 26.676}, "x2iezn.2xlarge": {Region: "us-west-2", Type: "x2iezn.2xlarge", Memory: kresource.MustParse("262144Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 1.668}, "x2iezn.4xlarge": {Region: "us-west-2", Type: "x2iezn.4xlarge", Memory: kresource.MustParse("524288Mi"), CPU: kresource.MustParse("16"), GPU: 0, Inf: 0, Price: 3.336}, "x2iezn.6xlarge": {Region: "us-west-2", Type: "x2iezn.6xlarge", Memory: kresource.MustParse("786432Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 0, Price: 5.004}, "x2iezn.8xlarge": {Region: "us-west-2", Type: "x2iezn.8xlarge", Memory: kresource.MustParse("1048576Mi"), CPU: kresource.MustParse("32"), GPU: 0, Inf: 0, Price: 6.672}, "x2iezn.12xlarge": {Region: "us-west-2", Type: "x2iezn.12xlarge", Memory: kresource.MustParse("1572864Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 10.008}, "x2iezn.metal": {Region: "us-west-2", Type: "x2iezn.metal", Memory: kresource.MustParse("1572864Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 10.008}, "z1d.large": {Region: "us-west-2", Type: "z1d.large", Memory: kresource.MustParse("16384Mi"), CPU: kresource.MustParse("2"), GPU: 0, Inf: 0, Price: 0.186}, "z1d.xlarge": {Region: "us-west-2", Type: "z1d.xlarge", Memory: kresource.MustParse("32768Mi"), CPU: kresource.MustParse("4"), GPU: 0, Inf: 0, Price: 0.372}, "z1d.2xlarge": {Region: "us-west-2", Type: "z1d.2xlarge", Memory: kresource.MustParse("65536Mi"), CPU: kresource.MustParse("8"), GPU: 0, Inf: 0, Price: 0.744}, "z1d.3xlarge": {Region: "us-west-2", Type: "z1d.3xlarge", Memory: kresource.MustParse("98304Mi"), CPU: kresource.MustParse("12"), GPU: 0, Inf: 0, Price: 1.116}, "z1d.6xlarge": {Region: "us-west-2", Type: "z1d.6xlarge", Memory: kresource.MustParse("196608Mi"), CPU: kresource.MustParse("24"), GPU: 0, Inf: 0, Price: 2.232}, "z1d.12xlarge": {Region: "us-west-2", Type: "z1d.12xlarge", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.464}, "z1d.metal": {Region: "us-west-2", Type: "z1d.metal", Memory: kresource.MustParse("393216Mi"), CPU: kresource.MustParse("48"), GPU: 0, Inf: 0, Price: 4.464}, }, } // region -> NLB metadata var NLBMetadatas = map[string]NLBMetadata{ "af-south-1": {Region: "af-south-1", Price: 0.029988}, "ap-east-1": {Region: "ap-east-1", Price: 0.0277}, "ap-northeast-1": {Region: "ap-northeast-1", Price: 0.0243}, "ap-northeast-2": {Region: "ap-northeast-2", Price: 0.0225}, "ap-northeast-3": {Region: "ap-northeast-3", Price: 0.0243}, "ap-south-1": {Region: "ap-south-1", Price: 0.0239}, "ap-southeast-1": {Region: "ap-southeast-1", Price: 0.0252}, "ap-southeast-2": {Region: "ap-southeast-2", Price: 0.0252}, "ca-central-1": {Region: "ca-central-1", Price: 0.02475}, "eu-central-1": {Region: "eu-central-1", Price: 0.027}, "eu-north-1": {Region: "eu-north-1", Price: 0.02394}, "eu-south-1": {Region: "eu-south-1", Price: 0.02646}, "eu-west-1": {Region: "eu-west-1", Price: 0.0252}, "eu-west-2": {Region: "eu-west-2", Price: 0.02646}, "eu-west-3": {Region: "eu-west-3", Price: 0.02646}, "me-south-1": {Region: "me-south-1", Price: 0.02772}, "sa-east-1": {Region: "sa-east-1", Price: 0.034}, "us-east-1": {Region: "us-east-1", Price: 0.0225}, "us-east-2": {Region: "us-east-2", Price: 0.0225}, "us-gov-east-1": {Region: "us-gov-east-1", Price: 0.032}, "us-gov-west-1": {Region: "us-gov-west-1", Price: 0.032}, "us-west-1": {Region: "us-west-1", Price: 0.0252}, "us-west-2": {Region: "us-west-2", Price: 0.0225}, } // region -> ELB metadata var ELBMetadatas = map[string]ELBMetadata{ "af-south-1": {Region: "af-south-1", Price: 0.03332}, "ap-east-1": {Region: "ap-east-1", Price: 0.0308}, "ap-northeast-1": {Region: "ap-northeast-1", Price: 0.027}, "ap-northeast-2": {Region: "ap-northeast-2", Price: 0.025}, "ap-northeast-3": {Region: "ap-northeast-3", Price: 0.027}, "ap-south-1": {Region: "ap-south-1", Price: 0.0266}, "ap-southeast-1": {Region: "ap-southeast-1", Price: 0.028}, "ap-southeast-2": {Region: "ap-southeast-2", Price: 0.028}, "ca-central-1": {Region: "ca-central-1", Price: 0.0275}, "eu-central-1": {Region: "eu-central-1", Price: 0.03}, "eu-north-1": {Region: "eu-north-1", Price: 0.0266}, "eu-south-1": {Region: "eu-south-1", Price: 0.0294}, "eu-west-1": {Region: "eu-west-1", Price: 0.028}, "eu-west-2": {Region: "eu-west-2", Price: 0.0294}, "eu-west-3": {Region: "eu-west-3", Price: 0.0294}, "me-south-1": {Region: "me-south-1", Price: 0.0308}, "sa-east-1": {Region: "sa-east-1", Price: 0.034}, "us-east-1": {Region: "us-east-1", Price: 0.025}, "us-east-2": {Region: "us-east-2", Price: 0.025}, "us-gov-east-1": {Region: "us-gov-east-1", Price: 0.032}, "us-gov-west-1": {Region: "us-gov-west-1", Price: 0.032}, "us-west-1": {Region: "us-west-1", Price: 0.028}, "us-west-2": {Region: "us-west-2", Price: 0.025}, } // region -> NAT metadata var NATMetadatas = map[string]NATMetadata{ "af-south-1": {Region: "af-south-1", Price: 0.057}, "ap-east-1": {Region: "ap-east-1", Price: 0.065}, "ap-northeast-1": {Region: "ap-northeast-1", Price: 0.062}, "ap-northeast-2": {Region: "ap-northeast-2", Price: 0.059}, "ap-northeast-3": {Region: "ap-northeast-3", Price: 0.062}, "ap-south-1": {Region: "ap-south-1", Price: 0.056}, "ap-southeast-1": {Region: "ap-southeast-1", Price: 0.059}, "ap-southeast-2": {Region: "ap-southeast-2", Price: 0.059}, "ca-central-1": {Region: "ca-central-1", Price: 0.05}, "eu-central-1": {Region: "eu-central-1", Price: 0.052}, "eu-north-1": {Region: "eu-north-1", Price: 0.046}, "eu-south-1": {Region: "eu-south-1", Price: 0.05}, "eu-west-1": {Region: "eu-west-1", Price: 0.048}, "eu-west-2": {Region: "eu-west-2", Price: 0.05}, "eu-west-3": {Region: "eu-west-3", Price: 0.05}, "me-south-1": {Region: "me-south-1", Price: 0.0528}, "sa-east-1": {Region: "sa-east-1", Price: 0.093}, "us-east-1": {Region: "us-east-1", Price: 0.045}, "us-east-2": {Region: "us-east-2", Price: 0.045}, "us-gov-east-1": {Region: "us-gov-east-1", Price: 0.054}, "us-gov-west-1": {Region: "us-gov-west-1", Price: 0.054}, "us-west-1": {Region: "us-west-1", Price: 0.048}, "us-west-2": {Region: "us-west-2", Price: 0.045}, } // region -> EBS metadata var EBSMetadatas = map[string]map[string]EBSMetadata{ "af-south-1": { "gp2": {Region: "af-south-1", Type: "gp2", PriceGB: 0.1309, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, "gp3": {Region: "af-south-1", Type: "gp3", PriceGB: 0.1047, PriceIOPS: 0.0065000000, PriceThroughput: 0.0536576, IOPSConfigurable: true, ThroughputConfigurable: true}, "io1": {Region: "af-south-1", Type: "io1", PriceGB: 0.16422, PriceIOPS: 0.0856800000, PriceThroughput: 0, IOPSConfigurable: true, ThroughputConfigurable: false}, "sc1": {Region: "af-south-1", Type: "sc1", PriceGB: 0.019992, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, "st1": {Region: "af-south-1", Type: "st1", PriceGB: 0.0595, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, }, "ap-east-1": { "gp2": {Region: "ap-east-1", Type: "gp2", PriceGB: 0.132, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, "gp3": {Region: "ap-east-1", Type: "gp3", PriceGB: 0.1056, PriceIOPS: 0.0066000000, PriceThroughput: 0.0540672, IOPSConfigurable: true, ThroughputConfigurable: true}, "io1": {Region: "ap-east-1", Type: "io1", PriceGB: 0.1518, PriceIOPS: 0.0792000000, PriceThroughput: 0, IOPSConfigurable: true, ThroughputConfigurable: false}, "io2": {Region: "ap-east-1", Type: "io2", PriceGB: 0.1518, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, "sc1": {Region: "ap-east-1", Type: "sc1", PriceGB: 0.0198, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, "st1": {Region: "ap-east-1", Type: "st1", PriceGB: 0.0594, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, }, "ap-northeast-1": { "gp2": {Region: "ap-northeast-1", Type: "gp2", PriceGB: 0.12, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, "gp3": {Region: "ap-northeast-1", Type: "gp3", PriceGB: 0.096, PriceIOPS: 0.0060000000, PriceThroughput: 0.049152, IOPSConfigurable: true, ThroughputConfigurable: true}, "io1": {Region: "ap-northeast-1", Type: "io1", PriceGB: 0.142, PriceIOPS: 0.0740000000, PriceThroughput: 0, IOPSConfigurable: true, ThroughputConfigurable: false}, "io2": {Region: "ap-northeast-1", Type: "io2", PriceGB: 0.142, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, "sc1": {Region: "ap-northeast-1", Type: "sc1", PriceGB: 0.018, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, "st1": {Region: "ap-northeast-1", Type: "st1", PriceGB: 0.054, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, }, "ap-northeast-2": { "gp2": {Region: "ap-northeast-2", Type: "gp2", PriceGB: 0.114, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, "gp3": {Region: "ap-northeast-2", Type: "gp3", PriceGB: 0.0912, PriceIOPS: 0.0057000000, PriceThroughput: 0.046694400000000004, IOPSConfigurable: true, ThroughputConfigurable: true}, "io1": {Region: "ap-northeast-2", Type: "io1", PriceGB: 0.1278, PriceIOPS: 0.0666000000, PriceThroughput: 0, IOPSConfigurable: true, ThroughputConfigurable: false}, "io2": {Region: "ap-northeast-2", Type: "io2", PriceGB: 0.1278, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, "sc1": {Region: "ap-northeast-2", Type: "sc1", PriceGB: 0.0174, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, "st1": {Region: "ap-northeast-2", Type: "st1", PriceGB: 0.051, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, }, "ap-northeast-3": { "gp2": {Region: "ap-northeast-3", Type: "gp2", PriceGB: 0.12, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, "gp3": {Region: "ap-northeast-3", Type: "gp3", PriceGB: 0.096, PriceIOPS: 0.0060000000, PriceThroughput: 0.049152, IOPSConfigurable: true, ThroughputConfigurable: true}, "io1": {Region: "ap-northeast-3", Type: "io1", PriceGB: 0.142, PriceIOPS: 0.0740000000, PriceThroughput: 0, IOPSConfigurable: true, ThroughputConfigurable: false}, "sc1": {Region: "ap-northeast-3", Type: "sc1", PriceGB: 0.018, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, "st1": {Region: "ap-northeast-3", Type: "st1", PriceGB: 0.054, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, }, "ap-south-1": { "gp2": {Region: "ap-south-1", Type: "gp2", PriceGB: 0.114, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, "gp3": {Region: "ap-south-1", Type: "gp3", PriceGB: 0.0912, PriceIOPS: 0.0057000000, PriceThroughput: 0.046694400000000004, IOPSConfigurable: true, ThroughputConfigurable: true}, "io1": {Region: "ap-south-1", Type: "io1", PriceGB: 0.131, PriceIOPS: 0.0680000000, PriceThroughput: 0, IOPSConfigurable: true, ThroughputConfigurable: false}, "io2": {Region: "ap-south-1", Type: "io2", PriceGB: 0.131, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, "sc1": {Region: "ap-south-1", Type: "sc1", PriceGB: 0.0174, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, "st1": {Region: "ap-south-1", Type: "st1", PriceGB: 0.051, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, }, "ap-southeast-1": { "gp2": {Region: "ap-southeast-1", Type: "gp2", PriceGB: 0.12, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, "gp3": {Region: "ap-southeast-1", Type: "gp3", PriceGB: 0.096, PriceIOPS: 0.0060000000, PriceThroughput: 0.049152, IOPSConfigurable: true, ThroughputConfigurable: true}, "io1": {Region: "ap-southeast-1", Type: "io1", PriceGB: 0.138, PriceIOPS: 0.0720000000, PriceThroughput: 0, IOPSConfigurable: true, ThroughputConfigurable: false}, "io2": {Region: "ap-southeast-1", Type: "io2", PriceGB: 0.138, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, "sc1": {Region: "ap-southeast-1", Type: "sc1", PriceGB: 0.018, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, "st1": {Region: "ap-southeast-1", Type: "st1", PriceGB: 0.054, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, }, "ap-southeast-2": { "gp2": {Region: "ap-southeast-2", Type: "gp2", PriceGB: 0.12, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, "gp3": {Region: "ap-southeast-2", Type: "gp3", PriceGB: 0.096, PriceIOPS: 0.0060000000, PriceThroughput: 0.049152, IOPSConfigurable: true, ThroughputConfigurable: true}, "io1": {Region: "ap-southeast-2", Type: "io1", PriceGB: 0.138, PriceIOPS: 0.0720000000, PriceThroughput: 0, IOPSConfigurable: true, ThroughputConfigurable: false}, "io2": {Region: "ap-southeast-2", Type: "io2", PriceGB: 0.138, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, "sc1": {Region: "ap-southeast-2", Type: "sc1", PriceGB: 0.018, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, "st1": {Region: "ap-southeast-2", Type: "st1", PriceGB: 0.054, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, }, "ca-central-1": { "gp2": {Region: "ca-central-1", Type: "gp2", PriceGB: 0.11, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, "gp3": {Region: "ca-central-1", Type: "gp3", PriceGB: 0.088, PriceIOPS: 0.0055000000, PriceThroughput: 0.045056, IOPSConfigurable: true, ThroughputConfigurable: true}, "io1": {Region: "ca-central-1", Type: "io1", PriceGB: 0.138, PriceIOPS: 0.0720000000, PriceThroughput: 0, IOPSConfigurable: true, ThroughputConfigurable: false}, "io2": {Region: "ca-central-1", Type: "io2", PriceGB: 0.138, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, "sc1": {Region: "ca-central-1", Type: "sc1", PriceGB: 0.0168, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, "st1": {Region: "ca-central-1", Type: "st1", PriceGB: 0.05, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, }, "eu-central-1": { "gp2": {Region: "eu-central-1", Type: "gp2", PriceGB: 0.119, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, "gp3": {Region: "eu-central-1", Type: "gp3", PriceGB: 0.0952, PriceIOPS: 0.0060000000, PriceThroughput: 0.048742400000000005, IOPSConfigurable: true, ThroughputConfigurable: true}, "io1": {Region: "eu-central-1", Type: "io1", PriceGB: 0.149, PriceIOPS: 0.0780000000, PriceThroughput: 0, IOPSConfigurable: true, ThroughputConfigurable: false}, "io2": {Region: "eu-central-1", Type: "io2", PriceGB: 0.149, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, "sc1": {Region: "eu-central-1", Type: "sc1", PriceGB: 0.018, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, "st1": {Region: "eu-central-1", Type: "st1", PriceGB: 0.054, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, }, "eu-north-1": { "gp2": {Region: "eu-north-1", Type: "gp2", PriceGB: 0.1045, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, "gp3": {Region: "eu-north-1", Type: "gp3", PriceGB: 0.0836, PriceIOPS: 0.0052000000, PriceThroughput: 0.0428032, IOPSConfigurable: true, ThroughputConfigurable: true}, "io1": {Region: "eu-north-1", Type: "io1", PriceGB: 0.1311, PriceIOPS: 0.0684000000, PriceThroughput: 0, IOPSConfigurable: true, ThroughputConfigurable: false}, "io2": {Region: "eu-north-1", Type: "io2", PriceGB: 0.1311, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, "sc1": {Region: "eu-north-1", Type: "sc1", PriceGB: 0.01596, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, "st1": {Region: "eu-north-1", Type: "st1", PriceGB: 0.0475, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, }, "eu-south-1": { "gp2": {Region: "eu-south-1", Type: "gp2", PriceGB: 0.1155, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, "gp3": {Region: "eu-south-1", Type: "gp3", PriceGB: 0.0924, PriceIOPS: 0.0058000000, PriceThroughput: 0.0473088, IOPSConfigurable: true, ThroughputConfigurable: true}, "io1": {Region: "eu-south-1", Type: "io1", PriceGB: 0.1449, PriceIOPS: 0.0756000000, PriceThroughput: 0, IOPSConfigurable: true, ThroughputConfigurable: false}, "sc1": {Region: "eu-south-1", Type: "sc1", PriceGB: 0.01764, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, "st1": {Region: "eu-south-1", Type: "st1", PriceGB: 0.0525, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, }, "eu-west-1": { "gp2": {Region: "eu-west-1", Type: "gp2", PriceGB: 0.11, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, "gp3": {Region: "eu-west-1", Type: "gp3", PriceGB: 0.088, PriceIOPS: 0.0055000000, PriceThroughput: 0.045056, IOPSConfigurable: true, ThroughputConfigurable: true}, "io1": {Region: "eu-west-1", Type: "io1", PriceGB: 0.138, PriceIOPS: 0.0720000000, PriceThroughput: 0, IOPSConfigurable: true, ThroughputConfigurable: false}, "io2": {Region: "eu-west-1", Type: "io2", PriceGB: 0.138, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, "sc1": {Region: "eu-west-1", Type: "sc1", PriceGB: 0.0168, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, "st1": {Region: "eu-west-1", Type: "st1", PriceGB: 0.05, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, }, "eu-west-2": { "gp2": {Region: "eu-west-2", Type: "gp2", PriceGB: 0.116, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, "gp3": {Region: "eu-west-2", Type: "gp3", PriceGB: 0.0928, PriceIOPS: 0.0058000000, PriceThroughput: 0.047513599999999996, IOPSConfigurable: true, ThroughputConfigurable: true}, "io1": {Region: "eu-west-2", Type: "io1", PriceGB: 0.145, PriceIOPS: 0.0760000000, PriceThroughput: 0, IOPSConfigurable: true, ThroughputConfigurable: false}, "io2": {Region: "eu-west-2", Type: "io2", PriceGB: 0.145, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, "sc1": {Region: "eu-west-2", Type: "sc1", PriceGB: 0.0174, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, "st1": {Region: "eu-west-2", Type: "st1", PriceGB: 0.053, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, }, "eu-west-3": { "gp2": {Region: "eu-west-3", Type: "gp2", PriceGB: 0.116, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, "gp3": {Region: "eu-west-3", Type: "gp3", PriceGB: 0.0928, PriceIOPS: 0.0058000000, PriceThroughput: 0.047513599999999996, IOPSConfigurable: true, ThroughputConfigurable: true}, "io1": {Region: "eu-west-3", Type: "io1", PriceGB: 0.145, PriceIOPS: 0.0760000000, PriceThroughput: 0, IOPSConfigurable: true, ThroughputConfigurable: false}, "sc1": {Region: "eu-west-3", Type: "sc1", PriceGB: 0.0174, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, "st1": {Region: "eu-west-3", Type: "st1", PriceGB: 0.053, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, }, "me-south-1": { "gp2": {Region: "me-south-1", Type: "gp2", PriceGB: 0.121, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, "gp3": {Region: "me-south-1", Type: "gp3", PriceGB: 0.0968, PriceIOPS: 0.0061000000, PriceThroughput: 0.0495616, IOPSConfigurable: true, ThroughputConfigurable: true}, "io1": {Region: "me-south-1", Type: "io1", PriceGB: 0.1518, PriceIOPS: 0.0792000000, PriceThroughput: 0, IOPSConfigurable: true, ThroughputConfigurable: false}, "io2": {Region: "me-south-1", Type: "io2", PriceGB: 0.1518, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, "sc1": {Region: "me-south-1", Type: "sc1", PriceGB: 0.01848, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, "st1": {Region: "me-south-1", Type: "st1", PriceGB: 0.055, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, }, "sa-east-1": { "gp2": {Region: "sa-east-1", Type: "gp2", PriceGB: 0.19, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, "gp3": {Region: "sa-east-1", Type: "gp3", PriceGB: 0.152, PriceIOPS: 0.0095000000, PriceThroughput: 0.077824, IOPSConfigurable: true, ThroughputConfigurable: true}, "io1": {Region: "sa-east-1", Type: "io1", PriceGB: 0.238, PriceIOPS: 0.0910000000, PriceThroughput: 0, IOPSConfigurable: true, ThroughputConfigurable: false}, "sc1": {Region: "sa-east-1", Type: "sc1", PriceGB: 0.0288, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, "st1": {Region: "sa-east-1", Type: "st1", PriceGB: 0.086, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, }, "us-east-1": { "gp2": {Region: "us-east-1", Type: "gp2", PriceGB: 0.1, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, "gp3": {Region: "us-east-1", Type: "gp3", PriceGB: 0.08, PriceIOPS: 0.0050000000, PriceThroughput: 0.04096, IOPSConfigurable: true, ThroughputConfigurable: true}, "io1": {Region: "us-east-1", Type: "io1", PriceGB: 0.125, PriceIOPS: 0.0650000000, PriceThroughput: 0, IOPSConfigurable: true, ThroughputConfigurable: false}, "io2": {Region: "us-east-1", Type: "io2", PriceGB: 0.125, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, "sc1": {Region: "us-east-1", Type: "sc1", PriceGB: 0.015, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, "st1": {Region: "us-east-1", Type: "st1", PriceGB: 0.045, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, }, "us-east-2": { "gp2": {Region: "us-east-2", Type: "gp2", PriceGB: 0.1, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, "gp3": {Region: "us-east-2", Type: "gp3", PriceGB: 0.08, PriceIOPS: 0.0050000000, PriceThroughput: 0.04096, IOPSConfigurable: true, ThroughputConfigurable: true}, "io1": {Region: "us-east-2", Type: "io1", PriceGB: 0.125, PriceIOPS: 0.0650000000, PriceThroughput: 0, IOPSConfigurable: true, ThroughputConfigurable: false}, "io2": {Region: "us-east-2", Type: "io2", PriceGB: 0.125, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, "sc1": {Region: "us-east-2", Type: "sc1", PriceGB: 0.015, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, "st1": {Region: "us-east-2", Type: "st1", PriceGB: 0.045, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, }, "us-gov-east-1": { "gp2": {Region: "us-gov-east-1", Type: "gp2", PriceGB: 0.12, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, "gp3": {Region: "us-gov-east-1", Type: "gp3", PriceGB: 0.096, PriceIOPS: 0.0060000000, PriceThroughput: 0.049152, IOPSConfigurable: true, ThroughputConfigurable: true}, "io1": {Region: "us-gov-east-1", Type: "io1", PriceGB: 0.15, PriceIOPS: 0.0780000000, PriceThroughput: 0, IOPSConfigurable: true, ThroughputConfigurable: false}, "sc1": {Region: "us-gov-east-1", Type: "sc1", PriceGB: 0.018, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, "st1": {Region: "us-gov-east-1", Type: "st1", PriceGB: 0.054, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, }, "us-gov-west-1": { "gp2": {Region: "us-gov-west-1", Type: "gp2", PriceGB: 0.12, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, "gp3": {Region: "us-gov-west-1", Type: "gp3", PriceGB: 0.096, PriceIOPS: 0.0060000000, PriceThroughput: 0.049152, IOPSConfigurable: true, ThroughputConfigurable: true}, "io1": {Region: "us-gov-west-1", Type: "io1", PriceGB: 0.15, PriceIOPS: 0.0780000000, PriceThroughput: 0, IOPSConfigurable: true, ThroughputConfigurable: false}, "sc1": {Region: "us-gov-west-1", Type: "sc1", PriceGB: 0.018, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, "st1": {Region: "us-gov-west-1", Type: "st1", PriceGB: 0.054, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, }, "us-west-1": { "gp2": {Region: "us-west-1", Type: "gp2", PriceGB: 0.12, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, "gp3": {Region: "us-west-1", Type: "gp3", PriceGB: 0.096, PriceIOPS: 0.0060000000, PriceThroughput: 0.049152, IOPSConfigurable: true, ThroughputConfigurable: true}, "io1": {Region: "us-west-1", Type: "io1", PriceGB: 0.138, PriceIOPS: 0.0720000000, PriceThroughput: 0, IOPSConfigurable: true, ThroughputConfigurable: false}, "io2": {Region: "us-west-1", Type: "io2", PriceGB: 0.138, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, "sc1": {Region: "us-west-1", Type: "sc1", PriceGB: 0.018, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, "st1": {Region: "us-west-1", Type: "st1", PriceGB: 0.054, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, }, "us-west-2": { "gp2": {Region: "us-west-2", Type: "gp2", PriceGB: 0.1, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, "gp3": {Region: "us-west-2", Type: "gp3", PriceGB: 0.08, PriceIOPS: 0.0050000000, PriceThroughput: 0.04096, IOPSConfigurable: true, ThroughputConfigurable: true}, "io1": {Region: "us-west-2", Type: "io1", PriceGB: 0.125, PriceIOPS: 0.0650000000, PriceThroughput: 0, IOPSConfigurable: true, ThroughputConfigurable: false}, "io2": {Region: "us-west-2", Type: "io2", PriceGB: 0.125, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, "sc1": {Region: "us-west-2", Type: "sc1", PriceGB: 0.015, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, "st1": {Region: "us-west-2", Type: "st1", PriceGB: 0.045, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false}, }, } // region -> EKS price var EKSPrices = map[string]float64{ "af-south-1": 0.1, "ap-east-1": 0.1, "ap-northeast-1": 0.1, "ap-northeast-2": 0.1, "ap-northeast-3": 0.1, "ap-south-1": 0.1, "ap-southeast-1": 0.1, "ap-southeast-2": 0.1, "ca-central-1": 0.1, "eu-central-1": 0.1, "eu-north-1": 0.1, "eu-south-1": 0.1, "eu-west-1": 0.1, "eu-west-2": 0.1, "eu-west-3": 0.1, "me-south-1": 0.1, "sa-east-1": 0.1, "us-east-1": 0.1, "us-east-2": 0.1, "us-gov-east-1": 0.1, "us-gov-west-1": 0.1, "us-west-1": 0.1, "us-west-2": 0.1, } ================================================ FILE: pkg/lib/aws/s3.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package aws import ( "bytes" "crypto/md5" "encoding/hex" "fmt" "io" "path/filepath" "strings" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/aws/endpoints" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3/s3manager" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/files" "github.com/cortexlabs/cortex/pkg/lib/json" "github.com/cortexlabs/cortex/pkg/lib/msgpack" "github.com/cortexlabs/cortex/pkg/lib/pointer" "github.com/cortexlabs/cortex/pkg/lib/sets/strset" "github.com/cortexlabs/cortex/pkg/lib/slices" s "github.com/cortexlabs/cortex/pkg/lib/strings" ) const DefaultS3Region string = endpoints.UsWest2RegionID var S3Regions strset.Set func init() { resolver := endpoints.DefaultResolver() partitions := resolver.(endpoints.EnumPartitions).Partitions() S3Regions = strset.New() for _, p := range partitions { if p.ID() == endpoints.AwsPartitionID || p.ID() == endpoints.AwsCnPartitionID { for id := range p.Regions() { S3Regions.Add(id) } } } } func S3Path(bucket string, key string) string { return "s3://" + filepath.Join(bucket, key) } func JoinS3Path(paths ...string) string { if len(paths) == 0 { return "" } paths[0] = paths[0][5:] return "s3://" + filepath.Join(paths...) } func SplitS3Path(s3Path string) (string, string, error) { if !IsValidS3Path(s3Path) { return "", "", ErrorInvalidS3Path(s3Path) } fullPath := s3Path[len("s3://"):] slashIndex := strings.Index(fullPath, "/") if slashIndex == -1 { return fullPath, "", nil } bucket := fullPath[0:slashIndex] key := fullPath[slashIndex+1:] return bucket, key, nil } func SplitS3aPath(s3aPath string) (string, string, error) { if !IsValidS3aPath(s3aPath) { return "", "", ErrorInvalidS3aPath(s3aPath) } fullPath := s3aPath[len("s3a://"):] slashIndex := strings.Index(fullPath, "/") bucket := fullPath[0:slashIndex] key := fullPath[slashIndex+1:] return bucket, key, nil } func IsValidS3Path(s3Path string) bool { if !strings.HasPrefix(s3Path, "s3://") { return false } parts := strings.Split(s3Path[5:], "/") if len(parts) == 0 { return false } if parts[0] == "" { return false } return true } func IsValidS3aPath(s3aPath string) bool { if !strings.HasPrefix(s3aPath, "s3a://") { return false } parts := strings.Split(s3aPath[6:], "/") if len(parts) < 2 { return false } if parts[0] == "" || parts[1] == "" { return false } return true } // List all S3 objects that are "depth" levels or deeper than the given "s3Path". // Setting depth to 1 effectively translates to listing the objects one level or deeper than the given prefix (aka listing the directory contents). // // 1st returned value is the list of paths found at level or deeper. // 2nd returned value is the list of paths found at all levels. func (c *Client) GetNLevelsDeepFromS3Path(s3Path string, depth int, includeDirObjects bool, maxResults *int64) ([]string, []string, error) { paths := strset.New() _, key, err := SplitS3Path(s3Path) if err != nil { return nil, nil, err } allS3Objects, err := c.ListS3PathDir(s3Path, includeDirObjects, maxResults, nil) if err != nil { return nil, nil, err } allPaths := ConvertS3ObjectsToKeys(allS3Objects...) keySplit := slices.RemoveEmpties(strings.Split(key, "/")) for _, path := range allPaths { pathSplit := slices.RemoveEmpties(strings.Split(path, "/")) if len(pathSplit)-len(keySplit) >= depth { computedPath := strings.Join(pathSplit[:len(keySplit)+depth], "/") paths.Add(computedPath) } } return paths.Slice(), allPaths, nil } // Efficient way to list all top-level directory prefixes on a bucket. // // Warning: when using this function, make sure the "~" character isn't used in any of the S3 objects. func (c *Client) ListS3TopLevelDirs(bucket string) ([]string, error) { // find first top-level directory s3Objects, err := c.ListS3Prefix(bucket, "", false, pointer.Int64(1), nil) if err != nil { return nil, err } dirs := []string{} for _, prefix := range ConvertS3ObjectsToKeys(s3Objects...) { dirs = append(dirs, files.GetTopLevelDirectory(prefix)) } // detect all remaining top-level dirs for { if len(dirs) == 0 { break } previousDir := dirs[len(dirs)-1] s3Objects, err := c.ListS3Prefix( bucket, "", true, pointer.Int64(1), pointer.String(filepath.Join(previousDir, "~~~")), ) if err != nil { return nil, err } if len(ConvertS3ObjectsToKeys(s3Objects...)) == 0 { break } for _, prefix := range ConvertS3ObjectsToKeys(s3Objects...) { dirs = append(dirs, files.GetTopLevelDirectory(prefix)) } } return dirs, nil } func ConvertS3ObjectsToKeys(s3Objects ...*s3.Object) []string { paths := make([]string, 0, len(s3Objects)) for _, object := range s3Objects { if object != nil { paths = append(paths, *object.Key) } } return paths } func GetBucketRegionFromS3Path(s3Path string) (string, error) { bucket, _, err := SplitS3Path(s3Path) if err != nil { return "", err } return GetBucketRegion(bucket) } func GetBucketRegion(bucket string) (string, error) { sess := session.Must(session.NewSession()) // credentials are not necessary for this request, and will not be used region, err := s3manager.GetBucketRegion(aws.BackgroundContext(), sess, bucket, endpoints.UsWest2RegionID) if err != nil { return "", ErrorBucketNotFound(bucket) } return region, nil } func (c *Client) IsS3PathFile(s3Path string, s3Paths ...string) (bool, error) { allS3Paths := append(s3Paths, s3Path) for _, s3Path := range allS3Paths { bucket, prefix, err := SplitS3Path(s3Path) if err != nil { return false, err } exists, err := c.IsS3File(bucket, prefix) if err != nil { return false, err } if !exists { return false, nil } } return true, nil } func (c *Client) IsS3File(bucket string, fileKey string, fileKeys ...string) (bool, error) { allFileKeys := append(fileKeys, fileKey) for _, key := range allFileKeys { _, err := c.S3().HeadObject(&s3.HeadObjectInput{ Bucket: aws.String(bucket), Key: aws.String(key), }) if IsNotFoundErr(err) { return false, nil } if err != nil { return false, errors.Wrap(err, S3Path(bucket, key)) } } return true, nil } func (c *Client) IsS3PathPrefix(s3Path string, s3Paths ...string) (bool, error) { allS3Paths := append(s3Paths, s3Path) for _, s3Path := range allS3Paths { bucket, prefix, err := SplitS3Path(s3Path) if err != nil { return false, err } exists, err := c.IsS3Prefix(bucket, prefix) if err != nil { return false, err } if !exists { return false, nil } } return true, nil } func (c *Client) IsS3Prefix(bucket string, prefix string, prefixes ...string) (bool, error) { allPrefixes := append(prefixes, prefix) for _, prefix := range allPrefixes { out, err := c.S3().ListObjectsV2(&s3.ListObjectsV2Input{ Bucket: aws.String(bucket), Prefix: aws.String(prefix), MaxKeys: aws.Int64(1), }) if err != nil { return false, errors.Wrap(err, S3Path(bucket, prefix)) } if *out.KeyCount == 0 { return false, nil } } return true, nil } func (c *Client) IsS3PathDir(s3Path string, s3Paths ...string) (bool, error) { allS3Paths := append(s3Paths, s3Path) for _, s3Path := range allS3Paths { bucket, prefix, err := SplitS3Path(s3Path) if err != nil { return false, err } exists, err := c.IsS3Dir(bucket, prefix) if err != nil { return false, err } if !exists { return false, nil } } return true, nil } func (c *Client) IsS3Dir(bucket string, dirPath string, dirPaths ...string) (bool, error) { fullDirPaths := make([]string, len(dirPaths)+1) allDirPaths := append(dirPaths, dirPath) for i, dirPath := range allDirPaths { fullDirPaths[i] = s.EnsureSuffix(dirPath, "/") } return c.IsS3Prefix(bucket, fullDirPaths[0], fullDirPaths[1:]...) } // Checks bucket existence and accessibility with credentials func (c *Client) DoesBucketExist(bucket string) (bool, error) { _, err := c.S3().HeadBucket(&s3.HeadBucketInput{ Bucket: aws.String(bucket), }) if err != nil { if aerr, ok := err.(awserr.Error); ok { switch aerr.Code() { case "NotFound": return false, nil case "Forbidden": return false, ErrorBucketInaccessible(bucket) } } return false, errors.Wrap(err, "bucket "+bucket) } return true, nil } func (c *Client) CreateBucket(bucket string) error { var bucketConfiguration *s3.CreateBucketConfiguration if c.Region != "us-east-1" { bucketConfiguration = &s3.CreateBucketConfiguration{ LocationConstraint: aws.String(c.Region), } } _, err := c.S3().CreateBucket(&s3.CreateBucketInput{ Bucket: aws.String(bucket), CreateBucketConfiguration: bucketConfiguration, }) if err != nil { return errors.Wrap(err, "creating bucket "+bucket) } return nil } func (c *Client) EnableBucketEncryption(bucket string) error { _, err := c.S3().PutBucketEncryption(&s3.PutBucketEncryptionInput{ Bucket: aws.String(bucket), ServerSideEncryptionConfiguration: &s3.ServerSideEncryptionConfiguration{ Rules: []*s3.ServerSideEncryptionRule{ { ApplyServerSideEncryptionByDefault: &s3.ServerSideEncryptionByDefault{ SSEAlgorithm: pointer.String("AES256"), }, }, }, }, }) if err != nil { return errors.Wrap(err, "enabling encryption for bucket "+bucket) } return nil } func (c *Client) UploadReaderToS3(data io.Reader, bucket string, key string) error { _, err := c.S3Uploader().Upload(&s3manager.UploadInput{ Bucket: aws.String(bucket), Key: aws.String(key), Body: data, ACL: aws.String("private"), ContentDisposition: aws.String("attachment"), }) if err != nil { return errors.Wrap(err, S3Path(bucket, key)) } return nil } func (c *Client) UploadFileToS3(path string, bucket string, key string) error { file, err := files.Open(path) if err != nil { return err } defer file.Close() return c.UploadReaderToS3(file, bucket, key) } func (c *Client) UploadBytesToS3(data []byte, bucket string, key string) error { return c.UploadReaderToS3(bytes.NewReader(data), bucket, key) } func (c *Client) UploadStringToS3(str string, bucket string, key string) error { return c.UploadReaderToS3(strings.NewReader(str), bucket, key) } func (c *Client) UploadJSONToS3(obj interface{}, bucket string, key string) error { jsonBytes, err := json.Marshal(obj) if err != nil { return err } return c.UploadBytesToS3(jsonBytes, bucket, key) } func (c *Client) UploadMsgpackToS3(obj interface{}, bucket string, key string) error { msgpackBytes, err := msgpack.Marshal(obj) if err != nil { return err } return c.UploadBytesToS3(msgpackBytes, bucket, key) } func (c *Client) CreateEmptyS3File(bucket string, key string) error { return c.UploadReaderToS3(bytes.NewReader(nil), bucket, key) } func (c *Client) UploadDirToS3(localDirPath string, bucket string, s3Dir string, ignoreFns ...files.IgnoreFn) error { localRelPaths, err := files.ListDirRecursive(localDirPath, true, ignoreFns...) if err != nil { return err } for _, localRelPath := range localRelPaths { localPath := filepath.Join(localDirPath, localRelPath) key := filepath.Join(s3Dir, localRelPath) if err := c.UploadFileToS3(localPath, bucket, key); err != nil { return err } } return nil } // returned io.ReadCloser should be closed by the caller func (c *Client) ReadReaderFromS3(bucket string, key string) (io.ReadCloser, error) { // for reading into memory, s3.S3.GetObject() seems faster than s3manager.Downloader.Download() with aws.NewWriteAtBuffer([]byte{}) response, err := c.S3().GetObject(&s3.GetObjectInput{ Key: aws.String(key), Bucket: aws.String(bucket), }) if err != nil { return nil, errors.Wrap(err, S3Path(bucket, key)) } return response.Body, nil } func (c *Client) ReadBufferFromS3(bucket string, key string) (*bytes.Buffer, error) { reader, err := c.ReadReaderFromS3(bucket, key) if err != nil { return nil, err } defer reader.Close() buf := new(bytes.Buffer) buf.ReadFrom(reader) return buf, nil } func (c *Client) ReadBytesFromS3(bucket string, key string) ([]byte, error) { buf, err := c.ReadBufferFromS3(bucket, key) if err != nil { return nil, err } return buf.Bytes(), nil } func (c *Client) ReadStringFromS3(bucket string, key string) (string, error) { buf, err := c.ReadBufferFromS3(bucket, key) if err != nil { return "", err } return buf.String(), nil } func (c *Client) ReadJSONFromS3(objPtr interface{}, bucket string, key string) error { jsonBytes, err := c.ReadBytesFromS3(bucket, key) if err != nil { return err } return errors.Wrap(json.Unmarshal(jsonBytes, objPtr), S3Path(bucket, key)) } func (c *Client) ReadMsgpackFromS3(objPtr interface{}, bucket string, key string) error { msgpackBytes, err := c.ReadBytesFromS3(bucket, key) if err != nil { return err } return errors.Wrap(msgpack.Unmarshal(msgpackBytes, objPtr), S3Path(bucket, key)) } func (c *Client) ReadStringFromS3Path(s3Path string) (string, error) { bucket, key, err := SplitS3Path(s3Path) if err != nil { return "", err } return c.ReadStringFromS3(bucket, key) } func (c *Client) ReadBytesFromS3Path(s3Path string) ([]byte, error) { bucket, key, err := SplitS3Path(s3Path) if err != nil { return nil, err } return c.ReadBytesFromS3(bucket, key) } func (c *Client) ReadMsgpackFromS3Path(objPtr interface{}, s3Path string) error { bucket, key, err := SplitS3Path(s3Path) if err != nil { return err } return c.ReadMsgpackFromS3(objPtr, bucket, key) } // overwrites existing file func (c *Client) DownloadFileFromS3(bucket string, key string, localPath string) error { file, err := files.Create(localPath) if err != nil { return err } defer file.Close() // for downloading files, s3manager.Downloader.Download() is faster than s3.S3.GetObject() _, err = c.S3Downloader().Download(file, &s3.GetObjectInput{ Bucket: aws.String(bucket), Key: aws.String(key), }) if err != nil { return errors.Wrap(err, S3Path(bucket, key)) } return nil } func (c *Client) DownloadDirFromS3(bucket string, s3Dir string, localDirPath string, shouldTrimDirPrefix bool, maxFiles *int64) error { prefix := s.EnsureSuffix(s3Dir, "/") return c.DownloadPrefixFromS3(bucket, prefix, localDirPath, shouldTrimDirPrefix, maxFiles) } // if shouldTrimDirPrefix is true, the directory path of prefix will be trimmed when downloading files // // example: // s3: [test/dir/1.txt, test/dir2/2.txt, test/directions.txt] // localDirPath: ~/downloads // // shouldTrimDirPrefix = true // prefix: "test/dir" // result: [~/downloads/dir/1.txt, ~/downloads/dir2/1.txt, ~/downloads/directions.txt] // // prefix: "test/dir/" // result: [~/downloads/1.txt] // // shouldTrimDirPrefix = false // prefix: "test/dir" // result: [~/downloads/test/dir/1.txt, ~/downloads/test/dir2/1.txt, ~/downloads/test/directions.txt] // // prefix: "test/dir/" // result: [~/downloads/test/dir/1.txt] func (c *Client) DownloadPrefixFromS3(bucket string, prefix string, localDirPath string, shouldTrimDirPrefix bool, maxFiles *int64) error { if _, err := files.CreateDirIfMissing(localDirPath); err != nil { return err } createdDirs := strset.New(localDirPath) var trimPrefix string if shouldTrimDirPrefix { lastIndex := strings.LastIndex(prefix, "/") if lastIndex == -1 { trimPrefix = "" } else { trimPrefix = prefix[:lastIndex+1] } } err := c.S3Iterator(bucket, prefix, true, maxFiles, nil, func(object *s3.Object) (bool, error) { localRelPath := *object.Key if shouldTrimDirPrefix { localRelPath = strings.TrimPrefix(localRelPath, trimPrefix) } localPath := filepath.Join(localDirPath, localRelPath) // check for directory objects if strings.HasSuffix(*object.Key, "/") { if !createdDirs.Has(localPath) { if _, err := files.CreateDirIfMissing(localPath); err != nil { return false, err } createdDirs.Add(localPath) } return true, nil } localDir := filepath.Dir(localPath) if !createdDirs.Has(localDir) { if _, err := files.CreateDirIfMissing(localDir); err != nil { return false, err } createdDirs.Add(localDir) } if err := c.DownloadFileFromS3(bucket, *object.Key, localPath); err != nil { return false, err } return true, nil }) if err != nil { return err } return nil } func (c *Client) S3FileIterator(bucket string, s3Obj *s3.Object, partSize int, fn func(buffer io.ReadCloser, isLastPart bool) (bool, error)) error { size := int(*s3Obj.Size) iters := size / partSize if size%partSize != 0 { iters++ } for i := 0; i < iters; i++ { min := i * (partSize) max := (i + 1) * (partSize) if max > size { max = size } max-- byteRange := fmt.Sprintf("bytes=%d-%d", min, max) obj, err := c.S3().GetObject(&s3.GetObjectInput{ Bucket: aws.String(bucket), Key: s3Obj.Key, Range: aws.String(byteRange), // use range instead of part numbers because only files uploaded using multipart have parts }) if err != nil { return errors.Wrap(err, S3Path(bucket, *s3Obj.Key), "range "+byteRange) } isLastChunk := i+1 == iters shouldContinue, err := fn(obj.Body, isLastChunk) if err != nil { return errors.Wrap(err, S3Path(bucket, *s3Obj.Key)) } if !shouldContinue { break } } return nil } func (c *Client) ListS3Dir(bucket string, s3Dir string, includeDirObjects bool, maxResults *int64, startAfter *string) ([]*s3.Object, error) { prefix := s.EnsureSuffix(s3Dir, "/") return c.ListS3Prefix(bucket, prefix, includeDirObjects, maxResults, startAfter) } func (c *Client) ListS3PathDir(s3DirPath string, includeDirObjects bool, maxResults *int64, startAfter *string) ([]*s3.Object, error) { s3Path := s.EnsureSuffix(s3DirPath, "/") return c.ListS3PathPrefix(s3Path, includeDirObjects, maxResults, startAfter) } // This behaves like you'd expect `ls` to behave on a local filesystem // "directory" names will be returned even if S3 directory objects don't exist func (c *Client) ListS3DirOneLevel(bucket string, s3Dir string, maxResults *int64, startAfter *string) ([]string, error) { s3Dir = s.EnsureSuffix(s3Dir, "/") allNames := strset.New() err := c.S3Iterator(bucket, s3Dir, true, nil, startAfter, func(object *s3.Object) (bool, error) { relativePath := strings.TrimPrefix(*object.Key, s3Dir) oneLevelPath := strings.Split(relativePath, "/")[0] allNames.Add(oneLevelPath) if maxResults != nil && int64(len(allNames)) >= *maxResults { return false, nil } return true, nil }) if err != nil { return nil, errors.Wrap(err, S3Path(bucket, s3Dir)) } return allNames.SliceSorted(), nil } func (c *Client) ListS3Prefix(bucket string, prefix string, includeDirObjects bool, maxResults *int64, startAfter *string) ([]*s3.Object, error) { var allObjects []*s3.Object err := c.S3BatchIterator(bucket, prefix, includeDirObjects, maxResults, startAfter, func(objects []*s3.Object) (bool, error) { allObjects = append(allObjects, objects...) return true, nil }) if err != nil { return nil, errors.Wrap(err, S3Path(bucket, prefix)) } return allObjects, nil } func (c *Client) ListS3PathPrefix(s3Path string, includeDirObjects bool, maxResults *int64, startAfter *string) ([]*s3.Object, error) { bucket, prefix, err := SplitS3Path(s3Path) if err != nil { return nil, err } return c.ListS3Prefix(bucket, prefix, includeDirObjects, maxResults, startAfter) } func (c *Client) DeleteS3File(bucket string, key string) error { _, err := c.S3().DeleteObject( &s3.DeleteObjectInput{ Bucket: aws.String(bucket), Key: aws.String(key), }, ) if err != nil { return errors.WithStack(err) } return nil } func (c *Client) DeleteS3Dir(bucket string, s3Dir string, continueIfFailure bool) error { prefix := s.EnsureSuffix(s3Dir, "/") return c.DeleteS3Prefix(bucket, prefix, continueIfFailure) } func (c *Client) DeleteS3Prefix(bucket string, prefix string, continueIfFailure bool) error { // This implementation is confirmed to be considerably faster than using s3manager.NewDeleteListIterator() + s3manager.NewBatchDeleteWithClient() err := c.S3BatchIterator(bucket, prefix, true, nil, nil, func(objects []*s3.Object) (bool, error) { deleteObjects := make([]*s3.ObjectIdentifier, len(objects)) for i, object := range objects { deleteObjects[i] = &s3.ObjectIdentifier{Key: object.Key} } _, err := c.S3().DeleteObjects(&s3.DeleteObjectsInput{ Bucket: aws.String(bucket), Delete: &s3.Delete{ Objects: deleteObjects, Quiet: aws.Bool(true), }, }) if err != nil { err := errors.Wrap(err, S3Path(bucket, prefix)) if !continueIfFailure { return false, err } return true, err } return true, nil }) if err != nil { return err } return nil } func (c *Client) HashS3Dir(bucket string, prefix string, maxResults *int64, startAfter *string) (string, error) { md5Hash := md5.New() err := c.S3BatchIterator(bucket, prefix, true, maxResults, startAfter, func(objects []*s3.Object) (bool, error) { var subErr error for _, object := range objects { io.WriteString(md5Hash, *object.ETag) } return true, subErr }) if err != nil { return "", err } return hex.EncodeToString(md5Hash.Sum(nil)), nil } // Directory objects are empty objects ending in "/". They are not guaranteed to exists, and there may or may not be files "in" the directory func (c *Client) S3Iterator(bucket string, prefix string, includeDirObjects bool, maxResults *int64, startAfter *string, fn func(*s3.Object) (bool, error)) error { err := c.S3BatchIterator(bucket, prefix, includeDirObjects, maxResults, startAfter, func(objects []*s3.Object) (bool, error) { var subErr error for _, object := range objects { shouldContinue, newSubErr := fn(object) if newSubErr != nil { subErr = newSubErr } if !shouldContinue { return false, subErr } } return true, subErr }) if err != nil { return err } return nil } // The return value of fn([]*s3.Object) (bool, error) should be whether to continue iterating, and an error (if any occurred) // Directory objects are empty objects ending in "/". They are not guaranteed to exists, and there may or may not be files "in" the directory func (c *Client) S3BatchIterator(bucket string, prefix string, includeDirObjects bool, maxResults *int64, startAfter *string, fn func([]*s3.Object) (bool, error)) error { var maxResultsRemaining *int64 if maxResults != nil { maxResultsRemaining = pointer.Int64(*maxResults) } listObjectsInput := &s3.ListObjectsV2Input{ Bucket: aws.String(bucket), Prefix: aws.String(prefix), MaxKeys: maxResultsRemaining, StartAfter: startAfter, } var numSeen int64 var subErr error err := c.S3().ListObjectsV2Pages(listObjectsInput, func(listObjectsOutput *s3.ListObjectsV2Output, lastPage bool) bool { objects := listObjectsOutput.Contents // filter directory objects (https://github.com/golang/go/wiki/SliceTricks#filtering-without-allocating) if !includeDirObjects { filtered := objects[:0] for _, object := range objects { if !strings.HasSuffix(*object.Key, "/") { filtered = append(filtered, object) } } objects = filtered } if len(objects) == 0 { return true } shouldContinue, newSubErr := fn(objects) if newSubErr != nil { subErr = newSubErr } if !shouldContinue { return false } numSeen += int64(len(objects)) if maxResults != nil { if numSeen >= *maxResults { return false } *maxResultsRemaining = *maxResults - numSeen } return true }) if subErr != nil { return subErr } if err != nil { return err } return nil } func (c *Client) TagBucket(bucket string, tagMap map[string]string) error { var tagSet []*s3.Tag for key, value := range tagMap { tagSet = append(tagSet, &s3.Tag{ Key: aws.String(key), Value: aws.String(value), }) } _, err := c.S3().PutBucketTagging( &s3.PutBucketTaggingInput{ Bucket: aws.String(bucket), Tagging: &s3.Tagging{ TagSet: tagSet, }, }, ) if err != nil { return errors.Wrap(err, "failed to add tags to bucket", bucket) } return nil } func (c *Client) SetLifecycleRules(bucket string, rules []s3.LifecycleRule) error { pointerRules := []*s3.LifecycleRule{} for i := range rules { pointerRules = append(pointerRules, &rules[i]) } _, err := c.S3().PutBucketLifecycleConfiguration(&s3.PutBucketLifecycleConfigurationInput{ Bucket: pointer.String(bucket), LifecycleConfiguration: &s3.BucketLifecycleConfiguration{ Rules: pointerRules, }, }) return errors.WithStack(err) } func (c *Client) GetLifecycleRules(bucket string) ([]s3.LifecycleRule, error) { lifecycleOutput, err := c.S3().GetBucketLifecycleConfiguration(&s3.GetBucketLifecycleConfigurationInput{ Bucket: pointer.String(bucket), }) if err != nil { return nil, errors.WithStack(err) } if lifecycleOutput == nil { return nil, nil } s3Rules := []s3.LifecycleRule{} for _, rule := range lifecycleOutput.Rules { if rule != nil { s3Rules = append(s3Rules, *rule) } } return s3Rules, nil } func (c *Client) DeleteLifecycleRules(bucket string) error { _, err := c.S3().DeleteBucketLifecycle(&s3.DeleteBucketLifecycleInput{ Bucket: pointer.String(bucket), }) return errors.WithStack(err) } ================================================ FILE: pkg/lib/aws/servicequotas.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package aws import ( "strings" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/servicequotas" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/pointer" "github.com/cortexlabs/cortex/pkg/lib/sets/strset" ) var _standardInstanceFamilies = strset.New("a", "c", "d", "h", "i", "m", "r", "t", "z") var _knownInstanceFamilies = strset.Union(_standardInstanceFamilies, strset.New("p", "g", "inf", "x", "f", "mac", "vt", "dl", "trn", "im", "is", "hpc")) const ( // 11 inbound rules _baseInboundRulesForNodeGroup = 11 _inboundRulesPerAZ = 8 // ClusterSharedNodeSecurityGroup, ControlPlaneSecurityGroup, eks-cluster-sg-, and operator security group _baseNumberOfSecurityGroups = 4 ) type InstanceTypeRequests struct { InstanceType string RequiredOnDemandInstances int64 RequiredSpotInstances int64 } type instanceClassRequest struct { InstanceClass string InstanceTypes strset.Set // the instance class (e.g. "standard") can have multiple instance types RequiredOnDemandCPUs int64 RequiredSpotCPUs int64 OnDemandCPUQuota *int64 OnDemandQuotaCode string SpotCPUQuota *int64 SpotQuotaCode string } func (c *Client) VerifyInstanceQuota(instances []InstanceTypeRequests) error { instanceClassRequests := []instanceClassRequest{} for _, instance := range instances { if instance.RequiredOnDemandInstances == 0 && instance.RequiredSpotInstances == 0 { continue } parsedType, err := ParseInstanceType(instance.InstanceType) if err != nil { continue } // Allow the instance if we don't recognize the type if !_knownInstanceFamilies.Has(parsedType.Family) { continue } instanceClass := parsedType.Family if _standardInstanceFamilies.Has(parsedType.Family) { instanceClass = "standard" } cpusPerInstance := InstanceMetadatas[c.Region][instance.InstanceType].CPU instanceClassFound := false for idx, r := range instanceClassRequests { if r.InstanceClass == instanceClass { instanceClassRequests[idx].InstanceTypes.Add(instance.InstanceType) instanceClassRequests[idx].RequiredOnDemandCPUs += instance.RequiredOnDemandInstances * cpusPerInstance.Value() instanceClassRequests[idx].RequiredSpotCPUs += instance.RequiredSpotInstances * cpusPerInstance.Value() instanceClassFound = true break } } if !instanceClassFound { instanceClassRequests = append(instanceClassRequests, instanceClassRequest{ InstanceClass: instanceClass, InstanceTypes: strset.New(instance.InstanceType), RequiredOnDemandCPUs: instance.RequiredOnDemandInstances * cpusPerInstance.Value(), RequiredSpotCPUs: instance.RequiredSpotInstances * cpusPerInstance.Value(), }) } } err := c.ServiceQuotas().ListServiceQuotasPages( &servicequotas.ListServiceQuotasInput{ ServiceCode: aws.String("ec2"), }, func(page *servicequotas.ListServiceQuotasOutput, lastPage bool) bool { if page == nil { return false } for _, quota := range page.Quotas { if quota == nil || quota.UsageMetric == nil || len(quota.UsageMetric.MetricDimensions) == 0 { continue } metricClass, ok := quota.UsageMetric.MetricDimensions["Class"] if !ok || metricClass == nil || !(strings.HasSuffix(*metricClass, "/OnDemand") || strings.HasSuffix(*metricClass, "/Spot")) { continue } for idx, r := range instanceClassRequests { // quota is specified in number of vCPU permitted per family if strings.ToLower(*metricClass) == r.InstanceClass+"/ondemand" { instanceClassRequests[idx].OnDemandCPUQuota = pointer.Int64(int64(*quota.Value)) instanceClassRequests[idx].OnDemandQuotaCode = *quota.QuotaCode } else if strings.ToLower(*metricClass) == r.InstanceClass+"/spot" { instanceClassRequests[idx].SpotCPUQuota = pointer.Int64(int64(*quota.Value)) instanceClassRequests[idx].SpotQuotaCode = *quota.QuotaCode } } } return true }, ) if err != nil { return errors.WithStack(err) } for _, r := range instanceClassRequests { if r.OnDemandCPUQuota != nil && *r.OnDemandCPUQuota < r.RequiredOnDemandCPUs { return ErrorInsufficientInstanceQuota(r.InstanceTypes.Slice(), "on-demand", c.Region, r.RequiredOnDemandCPUs, *r.OnDemandCPUQuota, r.OnDemandQuotaCode) } if r.SpotCPUQuota != nil && *r.SpotCPUQuota < r.RequiredSpotCPUs { return ErrorInsufficientInstanceQuota(r.InstanceTypes.Slice(), "spot", c.Region, r.RequiredSpotCPUs, *r.SpotCPUQuota, r.SpotQuotaCode) } } return nil } func (c *Client) ListServiceQuotas(quotaCodes []string, serviceCodes []string) (map[string]int, error) { desiredQuotaCodes := strset.New(quotaCodes...) quotaCodeToValueMap := map[string]int{} for _, serviceCode := range serviceCodes { err := c.ServiceQuotas().ListServiceQuotasPages( &servicequotas.ListServiceQuotasInput{ ServiceCode: aws.String(serviceCode), }, func(page *servicequotas.ListServiceQuotasOutput, lastPage bool) bool { if page == nil { return false } for _, quota := range page.Quotas { if quota == nil || quota.QuotaCode == nil || quota.Value == nil { continue } if desiredQuotaCodes.Has(*quota.QuotaCode) { quotaCodeToValueMap[*quota.QuotaCode] = int(*quota.Value) } } return true }, ) if err != nil { return nil, errors.Wrap(err, serviceCode) } } return quotaCodeToValueMap, nil } func (c *Client) VerifyInternetGatewayQuota(internetGatewayQuota int, requiredInternetGateways int) error { internetGatewaysInUse, err := c.ListInternetGateways() if err != nil { return err } additionalQuotaRequired := len(internetGatewaysInUse) + requiredInternetGateways - internetGatewayQuota if additionalQuotaRequired > 0 { return ErrorInternetGatewayLimitExceeded(internetGatewayQuota, additionalQuotaRequired, c.Region) } return nil } func (c *Client) VerifyNATGatewayQuota(natGatewayQuota int, availabilityZones strset.Set, highlyAvailableNATGateway bool) error { // get NAT GW in use per selected AZ natGateways, err := c.DescribeNATGateways() if err != nil { return err } subnets, err := c.DescribeSubnets() if err != nil { return err } azToGatewaysInUse := map[string]int{} for _, natGateway := range natGateways { if natGateway.SubnetId == nil { continue } for _, subnet := range subnets { if subnet.SubnetId == nil || subnet.AvailabilityZone == nil { continue } if !availabilityZones.Has(*subnet.AvailabilityZone) { continue } if *subnet.SubnetId == *natGateway.SubnetId { azToGatewaysInUse[*subnet.AvailabilityZone]++ } } } // check NAT GW quota numOfExhaustedNATGatewayAZs := 0 azsWithQuotaDeficit := []string{} for az, numActiveGatewaysOnAZ := range azToGatewaysInUse { // -1 comes from the NAT gateway we require per AZ azDeficit := natGatewayQuota - numActiveGatewaysOnAZ - 1 if azDeficit < 0 { numOfExhaustedNATGatewayAZs++ azsWithQuotaDeficit = append(azsWithQuotaDeficit, az) } } if (highlyAvailableNATGateway && numOfExhaustedNATGatewayAZs > 0) || (!highlyAvailableNATGateway && numOfExhaustedNATGatewayAZs == len(availabilityZones)) { return ErrorNATGatewayLimitExceeded(natGatewayQuota, 1, azsWithQuotaDeficit, c.Region) } return nil } func (c *Client) VerifyEIPQuota(eipQuota int, availabilityZones strset.Set, highlyAvailableNATGateway bool) error { elasticIPsInUse, err := c.ListElasticIPs() if err != nil { return err } var requiredElasticIPs int if highlyAvailableNATGateway { requiredElasticIPs = len(availabilityZones) } else { requiredElasticIPs = 1 } additionalQuotaRequired := len(elasticIPsInUse) + requiredElasticIPs - eipQuota if additionalQuotaRequired > 0 { return ErrorEIPLimitExceeded(eipQuota, additionalQuotaRequired, c.Region) } return nil } func (c *Client) VerifyVPCQuota(vpcQuota int, requiredVPCs int) error { vpcs, err := c.DescribeVpcs() if err != nil { return err } additionalQuotaRequired := len(vpcs) + requiredVPCs - vpcQuota if additionalQuotaRequired > 0 { return ErrorVPCLimitExceeded(vpcQuota, additionalQuotaRequired, c.Region) } return nil } func (c *Client) VerifySecurityGroupQuota(securifyGroupsQuota int, numNodeGroups int, clusterAlreadyExists bool) error { requiredSecurityGroups := requiredSecurityGroups(numNodeGroups, clusterAlreadyExists) sgs, err := c.DescribeSecurityGroups() if err != nil { return err } additionalQuotaRequired := len(sgs) + requiredSecurityGroups - securifyGroupsQuota if additionalQuotaRequired > 0 { return ErrorSecurityGroupLimitExceeded(securifyGroupsQuota, additionalQuotaRequired, c.Region) } return nil } func (c *Client) VerifySecurityGroupRulesQuota( securifyGroupRulesQuota int, availabilityZones strset.Set, numNodeGroups int, longestCIDRWhiteList int) error { // check rules quota for nodegroup SGs requiredRulesForSG := requiredRulesForNodeGroupSecurityGroup(len(availabilityZones), longestCIDRWhiteList) if requiredRulesForSG > securifyGroupRulesQuota { additionalQuotaRequired := requiredRulesForSG - securifyGroupRulesQuota return ErrorSecurityGroupRulesExceeded(securifyGroupRulesQuota, additionalQuotaRequired, c.Region) } // check rules quota for control plane SG requiredRulesForCPSG := requiredRulesForControlPlaneSecurityGroup(numNodeGroups) if requiredRulesForCPSG > securifyGroupRulesQuota { additionalQuotaRequired := requiredRulesForCPSG - securifyGroupRulesQuota return ErrorSecurityGroupRulesExceeded(securifyGroupRulesQuota, additionalQuotaRequired, c.Region) } return nil } func requiredRulesForNodeGroupSecurityGroup(numAZs, whitelistLength int) int { whitelistRuleCount := 0 if whitelistLength == 1 { whitelistRuleCount = 1 } else if whitelistLength > 1 { whitelistRuleCount = 1 + 5*(whitelistLength-1) } return _baseInboundRulesForNodeGroup + numAZs*_inboundRulesPerAZ + whitelistRuleCount } func requiredRulesForControlPlaneSecurityGroup(numNodeGroups int) int { // +2 for the operator and prometheus node groups // this is the number of outbound rules (there are half as many inbound rules, so that is not the limiting factor) return 2 * (numNodeGroups + 2) } func requiredSecurityGroups(numNodeGroups int, clusterAlreadyExists bool) int { if clusterAlreadyExists { return numNodeGroups } // each node group requires a security group return _baseNumberOfSecurityGroups + numNodeGroups } ================================================ FILE: pkg/lib/aws/sqs.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package aws import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/sqs" "github.com/cortexlabs/cortex/pkg/lib/errors" ) func (c *Client) GetAllQueueAttributes(queueURL string) (map[string]string, error) { output, err := c.SQS().GetQueueAttributes(&sqs.GetQueueAttributesInput{ QueueUrl: aws.String(queueURL), AttributeNames: aws.StringSlice([]string{"All"}), }) if err != nil { return nil, errors.Wrap(err, "unable to get queue attributes", queueURL) } return aws.StringValueMap(output.Attributes), nil } func (c *Client) ListQueuesByQueueNamePrefix(queueNamePrefix string) ([]string, error) { var queueURLs []string err := c.SQS().ListQueuesPages(&sqs.ListQueuesInput{ QueueNamePrefix: aws.String(queueNamePrefix), }, func(output *sqs.ListQueuesOutput, lastPage bool) bool { queueURLs = append(queueURLs, aws.StringValueSlice(output.QueueUrls)...) return true }) if err != nil { return nil, errors.WithStack(err) } return queueURLs, nil } func (c *Client) DoesQueueExist(queueName string) (bool, error) { _, err := c.SQS().GetQueueUrl(&sqs.GetQueueUrlInput{ QueueName: aws.String(queueName), }) if err != nil { if IsErrCode(err, sqs.ErrCodeQueueDoesNotExist) { return false, nil } return false, errors.Wrap(err, "failed to check if queue exists", queueName) } return true, nil } func (c *Client) DeleteQueuesWithPrefix(queueNamePrefix string) (int, error) { var numDeleted int var deleteError error err := c.SQS().ListQueuesPages(&sqs.ListQueuesInput{ QueueNamePrefix: aws.String(queueNamePrefix), }, func(output *sqs.ListQueuesOutput, lastPage bool) bool { for _, queueURL := range output.QueueUrls { _, err := c.SQS().DeleteQueue(&sqs.DeleteQueueInput{ // best effort delete QueueUrl: queueURL, }) numDeleted++ if deleteError != nil { deleteError = err } } return true }) if err != nil { return 0, errors.WithStack(err) } if deleteError != nil { return 0, errors.WithStack(deleteError) } return numDeleted, nil } ================================================ FILE: pkg/lib/aws/sts.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package aws import ( "encoding/base64" "encoding/xml" "io/ioutil" "net/http" "net/url" "strings" "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/aws/request" "github.com/aws/aws-sdk-go/private/protocol/query" "github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil" "github.com/aws/aws-sdk-go/service/sts" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/hash" libjson "github.com/cortexlabs/cortex/pkg/lib/json" "github.com/cortexlabs/cortex/pkg/lib/pointer" ) // Returns account ID, whether the credentials were valid, any other error that occurred // Ignores cache, so will re-run on every call to this method func (c *Client) CheckCredentials() (string, string, error) { response, err := c.STS().GetCallerIdentity(nil) if err != nil { return "", "", ErrorInvalidAWSCredentials(err) } c.accountID = response.Account c.hashedAccountID = pointer.String(hash.String(*c.accountID)) return *c.accountID, *c.hashedAccountID, nil } // Only re-checks the credentials if they have never been checked (so will not catch e.g. credentials expiring or getting revoked) func (c *Client) GetCachedAccountID() (string, string, error) { if c.accountID == nil || c.hashedAccountID == nil { if _, _, err := c.CheckCredentials(); err != nil { return "", "", err } } return *c.accountID, *c.hashedAccountID, nil } type awsRequest struct { Header http.Header URL string Method string Host string Body string ContentLength int64 } func (c *Client) IdentityRequestAsHeader() (string, error) { req, _ := c.STS().GetCallerIdentityRequest(nil) err := req.Sign() if err != nil { return "", errors.WithStack(err) } reqBody, err := ioutil.ReadAll(req.HTTPRequest.Body) if err != nil { return "", errors.WithStack(err) } signedRequestArtifacts := awsRequest{ Header: req.HTTPRequest.Header, URL: req.HTTPRequest.URL.String(), Method: req.HTTPRequest.Method, Host: req.HTTPRequest.Host, Body: string(reqBody), ContentLength: req.HTTPRequest.ContentLength, } jsonSignedRequestArtifacts, err := libjson.Marshal(signedRequestArtifacts) if err != nil { return "", err } return base64.RawURLEncoding.EncodeToString(jsonSignedRequestArtifacts), nil } // ExecuteIdentityRequestFromHeader executes identity request marshalled from header and returns account id if successful func ExecuteIdentityRequestFromHeader(indentityRequestheader string) (string, error) { jsonObj, err := base64.RawURLEncoding.DecodeString(indentityRequestheader) if err != nil { return "", errors.WithStack(err) } signedRequestArtifacts := awsRequest{} err = libjson.Unmarshal(jsonObj, &signedRequestArtifacts) if err != nil { return "", err } httpClient := http.Client{} url, err := url.Parse(signedRequestArtifacts.URL) if err != nil { return "", errors.WithStack(err) } req := http.Request{ Header: signedRequestArtifacts.Header, Method: signedRequestArtifacts.Method, URL: url, Body: ioutil.NopCloser(strings.NewReader(signedRequestArtifacts.Body)), ContentLength: signedRequestArtifacts.ContentLength, Host: signedRequestArtifacts.Host, } resp, err := httpClient.Do(&req) if err != nil { return "", errors.WithStack(err) } defer resp.Body.Close() if resp.StatusCode >= 400 { awsReq := request.Request{HTTPResponse: resp} query.UnmarshalError(&awsReq) return "", errors.WithStack(awsReq.Error) } decoder := xml.NewDecoder(resp.Body) result := sts.GetCallerIdentityOutput{} err = xmlutil.UnmarshalXML(&result, decoder, "GetCallerIdentityResult") if err != nil { return "", awserr.NewRequestFailure( awserr.New(request.ErrCodeSerialization, "failed decoding Query response", err), resp.StatusCode, resp.Header.Get("X-Amzn-Requestid"), ) } if result.Account == nil { return "", errors.ErrorUnexpected("GetCallerIdentityResult xml parsing failed") } return *result.Account, nil } ================================================ FILE: pkg/lib/cast/interface.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package cast import ( "encoding/json" "reflect" ) func InterfaceToInt8(in interface{}) (int8, bool) { var ok bool if in, ok = JSONNumberToInt(in); !ok { return 0, false } switch casted := in.(type) { case int8: return casted, true case int16: if val := int8(casted); int16(val) == casted { return val, true } case int32: if val := int8(casted); int32(val) == casted { return val, true } case int: if val := int8(casted); int(val) == casted { return val, true } case int64: if val := int8(casted); int64(val) == casted { return val, true } } return 0, false } func InterfaceToInt8Downcast(in interface{}) (int8, bool) { var ok bool if in, ok = JSONNumberToIntOrFloat(in); !ok { return 0, false } switch casted := in.(type) { case int8: return casted, true case int16: if val := int8(casted); int16(val) == casted { return val, true } case int32: if val := int8(casted); int32(val) == casted { return val, true } case int: if val := int8(casted); int(val) == casted { return val, true } case int64: if val := int8(casted); int64(val) == casted { return val, true } case float32: if val := int8(casted); float32(val) == casted { return val, true } case float64: if val := int8(casted); float64(val) == casted { return val, true } } return 0, false } func InterfaceToInt16(in interface{}) (int16, bool) { var ok bool if in, ok = JSONNumberToInt(in); !ok { return 0, false } switch casted := in.(type) { case int8: return int16(casted), true case int16: return casted, true case int32: if val := int16(casted); int32(val) == casted { return val, true } case int: if val := int16(casted); int(val) == casted { return val, true } case int64: if val := int16(casted); int64(val) == casted { return val, true } } return 0, false } func InterfaceToInt16Downcast(in interface{}) (int16, bool) { var ok bool if in, ok = JSONNumberToIntOrFloat(in); !ok { return 0, false } switch casted := in.(type) { case int8: return int16(casted), true case int16: return casted, true case int32: if val := int16(casted); int32(val) == casted { return val, true } case int: if val := int16(casted); int(val) == casted { return val, true } case int64: if val := int16(casted); int64(val) == casted { return val, true } case float32: if val := int16(casted); float32(val) == casted { return val, true } case float64: if val := int16(casted); float64(val) == casted { return val, true } } return 0, false } func InterfaceToInt32(in interface{}) (int32, bool) { var ok bool if in, ok = JSONNumberToInt(in); !ok { return 0, false } switch casted := in.(type) { case int8: return int32(casted), true case int16: return int32(casted), true case int32: return casted, true case int: if val := int32(casted); int(val) == casted { return val, true } case int64: if val := int32(casted); int64(val) == casted { return val, true } } return 0, false } func InterfaceToInt32Downcast(in interface{}) (int32, bool) { var ok bool if in, ok = JSONNumberToIntOrFloat(in); !ok { return 0, false } switch casted := in.(type) { case int8: return int32(casted), true case int16: return int32(casted), true case int32: return casted, true case int: if val := int32(casted); int(val) == casted { return val, true } case int64: if val := int32(casted); int64(val) == casted { return val, true } case float32: if val := int32(casted); float32(val) == casted { return val, true } case float64: if val := int32(casted); float64(val) == casted { return val, true } } return 0, false } func InterfaceToInt(in interface{}) (int, bool) { var ok bool if in, ok = JSONNumberToInt(in); !ok { return 0, false } switch casted := in.(type) { case int8: return int(casted), true case int16: return int(casted), true case int32: return int(casted), true case int: return casted, true case int64: if val := int(casted); int64(val) == casted { return val, true } } return 0, false } func InterfaceToIntDowncast(in interface{}) (int, bool) { var ok bool if in, ok = JSONNumberToIntOrFloat(in); !ok { return 0, false } switch casted := in.(type) { case int8: return int(casted), true case int16: return int(casted), true case int32: return int(casted), true case int: return casted, true case int64: if val := int(casted); int64(val) == casted { return val, true } case float32: if val := int(casted); float32(val) == casted { return val, true } case float64: if val := int(casted); float64(val) == casted { return val, true } } return 0, false } func InterfaceToInt64(in interface{}) (int64, bool) { var ok bool if in, ok = JSONNumberToInt(in); !ok { return 0, false } switch casted := in.(type) { case int8: return int64(casted), true case int16: return int64(casted), true case int32: return int64(casted), true case int: return int64(casted), true case int64: return casted, true } return 0, false } func InterfaceToInt64Downcast(in interface{}) (int64, bool) { var ok bool if in, ok = JSONNumberToIntOrFloat(in); !ok { return 0, false } switch casted := in.(type) { case int8: return int64(casted), true case int16: return int64(casted), true case int32: return int64(casted), true case int: return int64(casted), true case int64: return casted, true case float32: if val := int64(casted); float32(val) == casted { return val, true } case float64: if val := int64(casted); float64(val) == casted { return val, true } } return 0, false } // InterfaceToFloat32 will convert any int or float type func InterfaceToFloat32(in interface{}) (float32, bool) { var ok bool if in, ok = JSONNumberToIntOrFloat(in); !ok { return 0, false } switch casted := in.(type) { case int8: return float32(casted), true case int16: return float32(casted), true case int32: return float32(casted), true case int: return float32(casted), true case int64: return float32(casted), true case float32: return casted, true case float64: return float32(casted), true } return 0, false } // InterfaceToFloat64 will convert any int or float type func InterfaceToFloat64(in interface{}) (float64, bool) { var ok bool if in, ok = JSONNumberToIntOrFloat(in); !ok { return 0, false } switch casted := in.(type) { case int8: return float64(casted), true case int16: return float64(casted), true case int32: return float64(casted), true case int: return float64(casted), true case int64: return float64(casted), true case float32: return float64(casted), true case float64: return casted, true } return 0, false } func JSONNumberToInt(in interface{}) (interface{}, bool) { number, ok := in.(json.Number) if !ok { return in, true } inInt, err := number.Int64() if err == nil { return inInt, true } return nil, false } func JSONNumberToIntOrFloat(in interface{}) (interface{}, bool) { number, ok := in.(json.Number) if !ok { return in, true } inInt, err := number.Int64() if err == nil { return inInt, true } inFloat, err := number.Float64() if err == nil { return inFloat, true } return nil, false } func JSONNumber(in interface{}) interface{} { number, ok := in.(json.Number) if !ok { return in } inInt, err := number.Int64() if err == nil { return inInt } inFloat, err := number.Float64() if err == nil { return inFloat } return in // unexpected } func JSONNumbers(in []interface{}) []interface{} { casted := make([]interface{}, len(in)) for i, element := range in { casted[i] = JSONNumber(element) } return casted } func InterfaceToInterfaceSlice(in interface{}) ([]interface{}, bool) { if in == nil { return nil, true } if inSlice, ok := in.([]interface{}); ok { return inSlice, true } if reflect.TypeOf(in).Kind() != reflect.Slice { return nil, false } inVal := reflect.ValueOf(in) if inVal.IsNil() { return nil, true } out := make([]interface{}, inVal.Len()) for i := 0; i < inVal.Len(); i++ { out[i] = inVal.Index(i).Interface() } return out, true } func InterfaceToIntSlice(in interface{}) ([]int, bool) { if in == nil { return nil, true } if intSlice, ok := in.([]int); ok { return intSlice, true } inSlice, ok := InterfaceToInterfaceSlice(in) if !ok { return nil, false } out := make([]int, len(inSlice)) for i, elem := range inSlice { casted, ok := InterfaceToInt(elem) if !ok { return nil, false } out[i] = casted } return out, true } func InterfaceToInt32Slice(in interface{}) ([]int32, bool) { if in == nil { return nil, true } if intSlice, ok := in.([]int32); ok { return intSlice, true } inSlice, ok := InterfaceToInterfaceSlice(in) if !ok { return nil, false } out := make([]int32, len(inSlice)) for i, elem := range inSlice { casted, ok := InterfaceToInt32(elem) if !ok { return nil, false } out[i] = casted } return out, true } func InterfaceToInt64Slice(in interface{}) ([]int64, bool) { if in == nil { return nil, true } if intSlice, ok := in.([]int64); ok { return intSlice, true } inSlice, ok := InterfaceToInterfaceSlice(in) if !ok { return nil, false } out := make([]int64, len(inSlice)) for i, elem := range inSlice { casted, ok := InterfaceToInt64(elem) if !ok { return nil, false } out[i] = casted } return out, true } func InterfaceToFloat32Slice(in interface{}) ([]float32, bool) { if in == nil { return nil, true } if floatSlice, ok := in.([]float32); ok { return floatSlice, true } inSlice, ok := InterfaceToInterfaceSlice(in) if !ok { return nil, false } out := make([]float32, len(inSlice)) for i, elem := range inSlice { casted, ok := InterfaceToFloat32(elem) if !ok { return nil, false } out[i] = casted } return out, true } func InterfaceToFloat64Slice(in interface{}) ([]float64, bool) { if in == nil { return nil, true } if floatSlice, ok := in.([]float64); ok { return floatSlice, true } inSlice, ok := InterfaceToInterfaceSlice(in) if !ok { return nil, false } out := make([]float64, len(inSlice)) for i, elem := range inSlice { casted, ok := InterfaceToFloat64(elem) if !ok { return nil, false } out[i] = casted } return out, true } func InterfaceToStrSlice(in interface{}) ([]string, bool) { if in == nil { return nil, true } if strSlice, ok := in.([]string); ok { return strSlice, true } inSlice, ok := InterfaceToInterfaceSlice(in) if !ok { return nil, false } out := make([]string, len(inSlice)) for i, elem := range inSlice { casted, ok := elem.(string) if !ok { return nil, false } out[i] = casted } return out, true } func InterfaceToBoolSlice(in interface{}) ([]bool, bool) { if in == nil { return nil, true } if boolSlice, ok := in.([]bool); ok { return boolSlice, true } inSlice, ok := InterfaceToInterfaceSlice(in) if !ok { return nil, false } out := make([]bool, len(inSlice)) for i, elem := range inSlice { casted, ok := elem.(bool) if !ok { return nil, false } out[i] = casted } return out, true } func InterfaceToStrInterfaceMapSlice(in interface{}) ([]map[string]interface{}, bool) { if in == nil { return nil, true } if strMapSlice, ok := in.([]map[string]interface{}); ok { return strMapSlice, true } inSlice, ok := InterfaceToInterfaceSlice(in) if !ok { return nil, false } out := make([]map[string]interface{}, len(inSlice)) for i, elem := range inSlice { casted, ok := InterfaceToStrInterfaceMap(elem) if !ok { return nil, false } out[i] = casted } return out, true } func InterfaceToInterfaceInterfaceMap(in interface{}) (map[interface{}]interface{}, bool) { if in == nil { return nil, true } if inMap, ok := in.(map[interface{}]interface{}); ok { return inMap, true } if reflect.TypeOf(in).Kind() != reflect.Map { return nil, false } inVal := reflect.ValueOf(in) if inVal.IsNil() { return nil, true } out := make(map[interface{}]interface{}, inVal.Len()) for _, key := range inVal.MapKeys() { out[key.Interface()] = inVal.MapIndex(key).Interface() } return out, true } func InterfaceToStrInterfaceMap(in interface{}) (map[string]interface{}, bool) { if in == nil { return nil, true } if strMap, ok := in.(map[string]interface{}); ok { return strMap, true } inMap, ok := InterfaceToInterfaceInterfaceMap(in) if !ok { return nil, false } out := map[string]interface{}{} for key, value := range inMap { casted, ok := key.(string) if !ok { return nil, false } out[casted] = value } return out, true } // Recursively casts interface->interface maps to string->interface maps func JSONMarshallable(in interface{}) (interface{}, bool) { if in == nil { return nil, true } if inMap, ok := InterfaceToInterfaceInterfaceMap(in); ok { out := map[string]interface{}{} for key, value := range inMap { castedKey, ok := key.(string) if !ok { return nil, false } castedValue, ok := JSONMarshallable(value) if !ok { return nil, false } out[castedKey] = castedValue } return out, true } else if inSlice, ok := InterfaceToInterfaceSlice(in); ok { out := make([]interface{}, 0, len(inSlice)) for _, inValue := range inSlice { castedInValue, ok := JSONMarshallable(inValue) if !ok { return nil, false } out = append(out, castedInValue) } return out, true } return in, true } func InterfaceToStrStrMap(in interface{}) (map[string]string, bool) { if in == nil { return nil, true } if strMap, ok := in.(map[string]string); ok { return strMap, true } inMap, ok := InterfaceToInterfaceInterfaceMap(in) if !ok { return nil, false } out := map[string]string{} for key, value := range inMap { castedKey, ok := key.(string) if !ok { return nil, false } castedVal, ok := value.(string) if !ok { return nil, false } out[castedKey] = castedVal } return out, true } func StrMapToStrInterfaceMap(in map[string]string) map[string]interface{} { if in == nil { return nil } out := map[string]interface{}{} for k, v := range in { out[k] = v } return out } func IsIntType(in interface{}) bool { switch in := in.(type) { case json.Number: _, err := in.Int64() return err == nil case int8: return true case int16: return true case int32: return true case int64: return true case int: return true } return false } func IsFloatType(in interface{}) bool { switch in := in.(type) { case json.Number: _, intErr := in.Int64() _, floatErr := in.Float64() return floatErr == nil && intErr != nil case float32: return true case float64: return true } return false } func IsNumericType(in interface{}) bool { return IsIntType(in) || IsFloatType(in) } func IsScalarType(in interface{}) bool { if IsNumericType(in) { return true } switch in.(type) { case string: return true case bool: return true } return false } func FlattenInterfaceSlices(in ...interface{}) []interface{} { var result []interface{} for _, item := range in { if item == nil { result = append(result, nil) continue } if subItems, ok := InterfaceToInterfaceSlice(item); ok { if len(subItems) != 0 { result = append(result, FlattenInterfaceSlices(subItems...)...) } continue } result = append(result, item) } return result } ================================================ FILE: pkg/lib/cast/interface_test.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package cast import ( "encoding/json" "testing" "github.com/stretchr/testify/require" ) func TestInterfaceToInterfaceSlice(t *testing.T) { slice1 := []string{"test1", "test2", "test3"} slice2 := []interface{}{"test1", "test2", "test3"} var ok bool _, ok = InterfaceToInterfaceSlice(slice1) require.True(t, ok) _, ok = InterfaceToInterfaceSlice(slice2) require.True(t, ok) _, ok = InterfaceToStrSlice(slice1) require.True(t, ok) _, ok = InterfaceToStrSlice(slice2) require.True(t, ok) } func TestInterfaceToFloat64(t *testing.T) { var out float64 var ok bool out, ok = InterfaceToFloat64(float64(1.1)) require.True(t, ok) require.Equal(t, float64(1.1), out) out, ok = InterfaceToFloat64(float32(2.2)) require.True(t, ok) require.Equal(t, float64(float32(2.2)), out) out, ok = InterfaceToFloat64(int(3)) require.True(t, ok) require.Equal(t, float64(3), out) _, ok = InterfaceToFloat64("test") require.False(t, ok) } func TestInterfaceToIntDowncast(t *testing.T) { var out int var ok bool out, ok = InterfaceToIntDowncast(int(1)) require.True(t, ok) require.Equal(t, int(1), out) out, ok = InterfaceToIntDowncast(float32(2)) require.True(t, ok) require.Equal(t, int(2), out) out, ok = InterfaceToIntDowncast(float32(2.0)) require.True(t, ok) require.Equal(t, int(2), out) _, ok = InterfaceToIntDowncast(float32(2.2)) require.False(t, ok) out, ok = InterfaceToIntDowncast(float64(3)) require.True(t, ok) require.Equal(t, int(3), out) out, ok = InterfaceToIntDowncast(float64(3.0)) require.True(t, ok) require.Equal(t, int(3), out) _, ok = InterfaceToIntDowncast(float64(3.3)) require.False(t, ok) _, ok = InterfaceToIntDowncast("test") require.False(t, ok) } func TestInterfaceToInt(t *testing.T) { var out int var ok bool out, ok = InterfaceToInt(int(1)) require.True(t, ok) require.Equal(t, int(1), out) _, ok = InterfaceToInt(float32(2)) require.False(t, ok) _, ok = InterfaceToInt("test") require.False(t, ok) } func TestInterfaceToInt8Downcast(t *testing.T) { var out int8 var ok bool out, ok = InterfaceToInt8Downcast(int(1)) require.True(t, ok) require.Equal(t, int8(1), out) out, ok = InterfaceToInt8Downcast(float32(2)) require.True(t, ok) require.Equal(t, int8(2), out) out, ok = InterfaceToInt8Downcast(float32(2.0)) require.True(t, ok) require.Equal(t, int8(2), out) _, ok = InterfaceToInt8Downcast(float32(2.2)) require.False(t, ok) out, ok = InterfaceToInt8Downcast(float64(3)) require.True(t, ok) require.Equal(t, int8(3), out) out, ok = InterfaceToInt8Downcast(float64(3.0)) require.True(t, ok) require.Equal(t, int8(3), out) _, ok = InterfaceToInt8Downcast(float64(3.3)) require.False(t, ok) _, ok = InterfaceToInt8Downcast("test") require.False(t, ok) _, ok = InterfaceToInt8Downcast(int(999999)) require.False(t, ok) } func TestInterfaceToInt8(t *testing.T) { var out int8 var ok bool out, ok = InterfaceToInt8(int(1)) require.True(t, ok) require.Equal(t, int8(1), out) _, ok = InterfaceToInt8(float32(2)) require.False(t, ok) _, ok = InterfaceToInt8("test") require.False(t, ok) } func TestInterfaceToInterfaceInterfaceMap(t *testing.T) { var ok bool var in interface{} var casted map[interface{}]interface{} var expected map[interface{}]interface{} in = map[string]string{"test": "str"} expected = map[interface{}]interface{}{"test": "str"} casted, ok = InterfaceToInterfaceInterfaceMap(in) require.True(t, ok) require.Equal(t, expected, casted) in = map[int]bool{2: true} expected = map[interface{}]interface{}{int(2): true} casted, ok = InterfaceToInterfaceInterfaceMap(in) require.True(t, ok) require.Equal(t, expected, casted) in = map[interface{}]float32{"test": float32(2.2)} expected = map[interface{}]interface{}{"test": float32(2.2)} casted, ok = InterfaceToInterfaceInterfaceMap(in) require.True(t, ok) require.Equal(t, expected, casted) } func TestJSONMarshallable(t *testing.T) { var ok bool var in interface{} var casted interface{} var expected interface{} var err error in = map[string]interface{}{"test": map[interface{}]interface{}{"testing": []string{}}} expected = map[string]interface{}{"test": map[string]interface{}{"testing": []interface{}{}}} casted, ok = JSONMarshallable(in) require.True(t, ok) require.Equal(t, expected, casted) _, err = json.Marshal(casted) require.Equal(t, err, nil) in = map[string]interface{}{"test": map[interface{}]interface{}{1: []string{}}, "slice": []int{1}} casted, ok = JSONMarshallable(in) require.False(t, ok) in = map[string]interface{}{"test": map[interface{}]interface{}{"1": []string{}}, "slice": []int{1}} expected = map[string]interface{}{"test": map[string]interface{}{"1": []interface{}{}}, "slice": []interface{}{1}} casted, ok = JSONMarshallable(in) require.True(t, ok) require.Equal(t, expected, casted) _, err = json.Marshal(casted) require.Equal(t, err, nil) in = map[string]interface{}{"test": nil} expected = map[string]interface{}{"test": nil} casted, ok = JSONMarshallable(in) require.True(t, ok) require.Equal(t, expected, casted) _, err = json.Marshal(casted) require.Equal(t, err, nil) in = map[string]interface{}{"slice": []interface{}{1, "1", map[interface{}]interface{}{"key": false}}} expected = map[string]interface{}{"slice": []interface{}{1, "1", map[string]interface{}{"key": false}}} casted, ok = JSONMarshallable(in) require.True(t, ok) require.Equal(t, expected, casted) _, err = json.Marshal(casted) require.Equal(t, err, nil) in = map[string]interface{}{} expected = map[string]interface{}{} casted, ok = JSONMarshallable(in) require.True(t, ok) require.Equal(t, expected, casted) _, err = json.Marshal(casted) require.Equal(t, err, nil) } func TestFlattenInterfaceSlices(t *testing.T) { expected := []interface{}{"a", "b", "c"} in := []interface{}{"a", "b", "c"} require.Equal(t, expected, FlattenInterfaceSlices(in)) in2 := [][]interface{}{in} require.Equal(t, expected, FlattenInterfaceSlices(in2)) in3 := [][]interface{}{{"a"}, {"b", "c"}} require.Equal(t, expected, FlattenInterfaceSlices(in3)) in4 := [][]interface{}{{"a"}, {[]interface{}{"b"}, "c"}} require.Equal(t, expected, FlattenInterfaceSlices(in4)) } ================================================ FILE: pkg/lib/configreader/bool.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package configreader import ( "strings" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/exit" "github.com/cortexlabs/cortex/pkg/lib/files" "github.com/cortexlabs/cortex/pkg/lib/prompt" s "github.com/cortexlabs/cortex/pkg/lib/strings" ) type BoolValidation struct { Required bool Default bool TreatNullAsFalse bool // `: ` and `: null` will be read as `: false` CantBeSpecifiedErrStr *string StrToBool map[string]bool // lowercase } func Bool(inter interface{}, v *BoolValidation) (bool, error) { if inter == nil { if v.TreatNullAsFalse { return ValidateBoolProvided(false, v) } return false, ErrorCannotBeNull(v.Required) } casted, castOk := inter.(bool) if !castOk { return false, ErrorInvalidPrimitiveType(inter, PrimTypeBool) } return ValidateBoolProvided(casted, v) } func BoolFromInterfaceMap(key string, iMap map[string]interface{}, v *BoolValidation) (bool, error) { inter, ok := ReadInterfaceMapValue(key, iMap) if !ok { val, err := ValidateBoolMissing(v) if err != nil { return false, errors.Wrap(err, key) } return val, nil } val, err := Bool(inter, v) if err != nil { return false, errors.Wrap(err, key) } return val, nil } func BoolFromStrMap(key string, sMap map[string]string, v *BoolValidation) (bool, error) { valStr, ok := sMap[key] if !ok || valStr == "" { val, err := ValidateBoolMissing(v) if err != nil { return false, errors.Wrap(err, key) } return val, nil } val, err := BoolFromStr(valStr, v) if err != nil { return false, errors.Wrap(err, key) } return val, nil } func BoolFromStr(valStr string, v *BoolValidation) (bool, error) { if valStr == "" { return ValidateBoolMissing(v) } if len(v.StrToBool) > 0 { casted, ok := v.StrToBool[strings.ToLower(valStr)] if !ok { keys := make([]string, 0, len(v.StrToBool)) for key := range v.StrToBool { keys = append(keys, key) } return false, ErrorInvalidStr(valStr, keys[0], keys[1:]...) } return ValidateBoolProvided(casted, v) } casted, castOk := s.ParseBool(valStr) if !castOk { return false, ErrorInvalidPrimitiveType(valStr, PrimTypeBool) } return ValidateBoolProvided(casted, v) } func BoolFromEnv(envVarName string, v *BoolValidation) (bool, error) { valStr := ReadEnvVar(envVarName) if valStr == nil || *valStr == "" { val, err := ValidateBoolMissing(v) if err != nil { return false, errors.Wrap(err, EnvVar(envVarName)) } return val, nil } val, err := BoolFromStr(*valStr, v) if err != nil { return false, errors.Wrap(err, EnvVar(envVarName)) } return val, nil } func BoolFromFile(filePath string, v *BoolValidation) (bool, error) { if !files.IsFile(filePath) { val, err := ValidateBoolMissing(v) if err != nil { return false, errors.Wrap(err, filePath) } return val, nil } valStr, err := files.ReadFile(filePath) if err != nil { return false, err } if len(valStr) == 0 { val, err := ValidateBoolMissing(v) if err != nil { return false, errors.Wrap(err, filePath) } return val, nil } val, err := BoolFromStr(valStr, v) if err != nil { return false, errors.Wrap(err, filePath) } return val, nil } func BoolFromEnvOrFile(envVarName string, filePath string, v *BoolValidation) (bool, error) { valStr := ReadEnvVar(envVarName) if valStr != nil && *valStr != "" { return BoolFromEnv(envVarName, v) } return BoolFromFile(filePath, v) } func BoolFromPrompt(promptOpts *prompt.Options, v *BoolValidation) (bool, error) { promptOpts.DefaultStr = s.Bool(v.Default) valStr := prompt.Prompt(promptOpts) if valStr == "" { return ValidateBoolMissing(v) } return BoolFromStr(valStr, v) } func ValidateBoolMissing(v *BoolValidation) (bool, error) { if v.Required { return false, ErrorMustBeDefined() } return validateBool(v.Default, v) } func ValidateBoolProvided(val bool, v *BoolValidation) (bool, error) { if v.CantBeSpecifiedErrStr != nil { return false, ErrorFieldCantBeSpecified(*v.CantBeSpecifiedErrStr) } return validateBool(val, v) } func validateBool(val bool, v *BoolValidation) (bool, error) { return val, nil } // // Musts // func MustBoolFromEnv(envVarName string, v *BoolValidation) bool { val, err := BoolFromEnv(envVarName, v) if err != nil { exit.Panic(err) } return val } func MustBoolFromFile(filePath string, v *BoolValidation) bool { val, err := BoolFromFile(filePath, v) if err != nil { exit.Panic(err) } return val } func MustBoolFromEnvOrFile(envVarName string, filePath string, v *BoolValidation) bool { val, err := BoolFromEnvOrFile(envVarName, filePath, v) if err != nil { exit.Panic(err) } return val } ================================================ FILE: pkg/lib/configreader/bool_list.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package configreader import ( "github.com/cortexlabs/cortex/pkg/lib/cast" "github.com/cortexlabs/cortex/pkg/lib/errors" ) type BoolListValidation struct { Required bool Default []bool AllowExplicitNull bool AllowEmpty bool CantBeSpecifiedErrStr *string CastSingleItem bool MinLength int MaxLength int InvalidLengths []int Validator func([]bool) ([]bool, error) } func BoolList(inter interface{}, v *BoolListValidation) ([]bool, error) { casted, castOk := cast.InterfaceToBoolSlice(inter) if !castOk { if v.CastSingleItem { castedItem, castOk := inter.(bool) if !castOk { return nil, ErrorInvalidPrimitiveType(inter, PrimTypeBool, PrimTypeBoolList) } casted = []bool{castedItem} } else { return nil, ErrorInvalidPrimitiveType(inter, PrimTypeBoolList) } } return ValidateBoolListProvided(casted, v) } func BoolListFromInterfaceMap(key string, iMap map[string]interface{}, v *BoolListValidation) ([]bool, error) { inter, ok := ReadInterfaceMapValue(key, iMap) if !ok { val, err := ValidateBoolListMissing(v) if err != nil { return nil, errors.Wrap(err, key) } return val, nil } val, err := BoolList(inter, v) if err != nil { return nil, errors.Wrap(err, key) } return val, nil } func ValidateBoolListMissing(v *BoolListValidation) ([]bool, error) { if v.Required { return nil, ErrorMustBeDefined() } return validateBoolList(v.Default, v) } func ValidateBoolListProvided(val []bool, v *BoolListValidation) ([]bool, error) { if v.CantBeSpecifiedErrStr != nil { return nil, ErrorFieldCantBeSpecified(*v.CantBeSpecifiedErrStr) } if !v.AllowExplicitNull && val == nil { return nil, ErrorCannotBeNull(v.Required) } return validateBoolList(val, v) } func validateBoolList(val []bool, v *BoolListValidation) ([]bool, error) { if !v.AllowEmpty { if val != nil && len(val) == 0 { return nil, ErrorCannotBeEmpty() } } if v.MinLength != 0 { if len(val) < v.MinLength { return nil, ErrorTooFewElements(v.MinLength) } } if v.MaxLength != 0 { if len(val) > v.MaxLength { return nil, ErrorTooManyElements(v.MaxLength) } } for _, invalidLength := range v.InvalidLengths { if len(val) == invalidLength { return nil, ErrorWrongNumberOfElements(v.InvalidLengths) } } if v.Validator != nil { return v.Validator(val) } return val, nil } ================================================ FILE: pkg/lib/configreader/bool_ptr.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package configreader import ( "strings" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/files" "github.com/cortexlabs/cortex/pkg/lib/prompt" s "github.com/cortexlabs/cortex/pkg/lib/strings" ) type BoolPtrValidation struct { Required bool Default *bool AllowExplicitNull bool CantBeSpecifiedErrStr *string StrToBool map[string]bool // lowercase } func BoolPtr(inter interface{}, v *BoolPtrValidation) (*bool, error) { if inter == nil { return ValidateBoolPtrProvided(nil, v) } casted, castOk := inter.(bool) if !castOk { return nil, ErrorInvalidPrimitiveType(inter, PrimTypeBool) } return ValidateBoolPtrProvided(&casted, v) } func BoolPtrFromInterfaceMap(key string, iMap map[string]interface{}, v *BoolPtrValidation) (*bool, error) { inter, ok := ReadInterfaceMapValue(key, iMap) if !ok { val, err := ValidateBoolPtrMissing(v) if err != nil { return nil, errors.Wrap(err, key) } return val, nil } val, err := BoolPtr(inter, v) if err != nil { return nil, errors.Wrap(err, key) } return val, nil } func BoolPtrFromStrMap(key string, sMap map[string]string, v *BoolPtrValidation) (*bool, error) { valStr, ok := sMap[key] if !ok || valStr == "" { val, err := ValidateBoolPtrMissing(v) if err != nil { return nil, errors.Wrap(err, key) } return val, nil } val, err := BoolPtrFromStr(valStr, v) if err != nil { return nil, errors.Wrap(err, key) } return val, nil } func BoolPtrFromStr(valStr string, v *BoolPtrValidation) (*bool, error) { if valStr == "" { return ValidateBoolPtrMissing(v) } if len(v.StrToBool) > 0 { casted, ok := v.StrToBool[strings.ToLower(valStr)] if !ok { keys := make([]string, 0, len(v.StrToBool)) for key := range v.StrToBool { keys = append(keys, key) } return nil, ErrorInvalidStr(valStr, keys[0], keys[1:]...) } return ValidateBoolPtrProvided(&casted, v) } casted, castOk := s.ParseBool(valStr) if !castOk { return nil, ErrorInvalidPrimitiveType(valStr, PrimTypeBool) } return ValidateBoolPtrProvided(&casted, v) } func BoolPtrFromEnv(envVarName string, v *BoolPtrValidation) (*bool, error) { valStr := ReadEnvVar(envVarName) if valStr == nil || *valStr == "" { val, err := ValidateBoolPtrMissing(v) if err != nil { return nil, errors.Wrap(err, EnvVar(envVarName)) } return val, nil } val, err := BoolPtrFromStr(*valStr, v) if err != nil { return nil, errors.Wrap(err, EnvVar(envVarName)) } return val, nil } func BoolPtrFromFile(filePath string, v *BoolPtrValidation) (*bool, error) { if !files.IsFile(filePath) { val, err := ValidateBoolPtrMissing(v) if err != nil { return nil, errors.Wrap(err, filePath) } return val, nil } valStr, err := files.ReadFile(filePath) if err != nil { return nil, err } if len(valStr) == 0 { val, err := ValidateBoolPtrMissing(v) if err != nil { return nil, errors.Wrap(err, filePath) } return val, nil } val, err := BoolPtrFromStr(valStr, v) if err != nil { return nil, errors.Wrap(err, filePath) } return val, nil } func BoolPtrFromEnvOrFile(envVarName string, filePath string, v *BoolPtrValidation) (*bool, error) { valStr := ReadEnvVar(envVarName) if valStr != nil && *valStr != "" { return BoolPtrFromEnv(envVarName, v) } return BoolPtrFromFile(filePath, v) } func BoolPtrFromPrompt(promptOpts *prompt.Options, v *BoolPtrValidation) (*bool, error) { if v.Default != nil && promptOpts.DefaultStr == "" { promptOpts.DefaultStr = s.Bool(*v.Default) } valStr := prompt.Prompt(promptOpts) if valStr == "" { return ValidateBoolPtrMissing(v) } return BoolPtrFromStr(valStr, v) } func ValidateBoolPtrMissing(v *BoolPtrValidation) (*bool, error) { if v.Required { return nil, ErrorMustBeDefined() } return v.Default, nil } func ValidateBoolPtrProvided(val *bool, v *BoolPtrValidation) (*bool, error) { if v.CantBeSpecifiedErrStr != nil { return nil, ErrorFieldCantBeSpecified(*v.CantBeSpecifiedErrStr) } if !v.AllowExplicitNull && val == nil { return nil, ErrorCannotBeNull(v.Required) } return val, nil } ================================================ FILE: pkg/lib/configreader/errors.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package configreader import ( "fmt" "reflect" "strings" "github.com/cortexlabs/cortex/pkg/lib/errors" s "github.com/cortexlabs/cortex/pkg/lib/strings" ) const ( ErrParseConfig = "configreader.parse_config" ErrUnsupportedFieldValidation = "configreader.unsupported_field_validation" ErrUnsupportedKey = "configreader.unsupported_key" ErrInvalidYAML = "configreader.invalid_yaml" ErrTooLong = "configreader.too_long" ErrTooShort = "configreader.too_short" ErrLeadingWhitespace = "configreader.leading_whitespace" ErrTrailingWhitespace = "configreader.trailing_whitespace" ErrAlphaNumericDashUnderscore = "configreader.alpha_numeric_dash_underscore" ErrAlphaNumericDash = "configreader.alpha_numeric_dash" ErrAlphaNumericDotUnderscore = "configreader.alpha_numeric_dot_underscore" ErrAlphaNumericDashDotUnderscore = "configreader.alpha_numeric_dash_dot_underscore" ErrInvalidAWSTag = "configreader.invalid_aws_tag" ErrInvalidDockerImage = "configreader.invalid_docker_image" ErrMustHavePrefix = "configreader.must_have_prefix" ErrMustHaveSuffix = "configreader.must_have_suffix" ErrCantHavePrefix = "configreader.cant_have_prefix" ErrCantHaveSuffix = "configreader.cant_have_suffix" ErrInvalidInterface = "configreader.invalid_interface" ErrInvalidFloat64 = "configreader.invalid_float64" ErrInvalidFloat32 = "configreader.invalid_float32" ErrInvalidInt64 = "configreader.invalid_int64" ErrInvalidInt32 = "configreader.invalid_int32" ErrInvalidInt = "configreader.invalid_int" ErrInvalidStr = "configreader.invalid_str" ErrDisallowedValue = "configreader.disallowed_value" ErrMustBeLessThanOrEqualTo = "configreader.must_be_less_than_or_equal_to" ErrMustBeLessThan = "configreader.must_be_less_than" ErrMustBeGreaterThanOrEqualTo = "configreader.must_be_greater_than_or_equal_to" ErrMustBeGreaterThan = "configreader.must_be_greater_than" ErrIsNotMultiple = "configreader.is_not_multiple" ErrNonStringKeyFound = "configreader.non_string_key_found" ErrInvalidPrimitiveType = "configreader.invalid_primitive_type" ErrDuplicatedValue = "configreader.duplicated_value" ErrTooFewElements = "configreader.too_few_elements" ErrTooManyElements = "configreader.too_many_elements" ErrWrongNumberOfElements = "configreader.wrong_number_of_elements" ErrCannotSetStructField = "configreader.cannot_set_struct_field" ErrCannotBeNull = "configreader.cannot_be_null" ErrCannotBeEmptyOrNull = "configreader.cannot_be_empty_or_null" ErrCannotBeEmpty = "configreader.cannot_be_empty" ErrMustBeDefined = "configreader.must_be_defined" ErrMapMustBeDefined = "configreader.map_must_be_defined" ErrMustBeEmpty = "configreader.must_be_empty" ErrEmailTooLong = "configreader.email_too_long" ErrEmailInvalid = "configreader.email_invalid" ErrCortexResourceOnlyAllowed = "configreader.cortex_resource_only_allowed" ErrCortexResourceNotAllowed = "configreader.cortex_resource_not_allowed" ErrImageVersionMismatch = "configreader.image_version_mismatch" ErrFieldCantBeSpecified = "configreader.field_cant_be_specified" ) func ErrorParseConfig() error { return errors.WithStack(&errors.Error{ Kind: ErrParseConfig, Message: "failed to parse config file", }) } func ErrorUnsupportedFieldValidation() error { return errors.WithStack(&errors.Error{ Kind: ErrUnsupportedFieldValidation, Message: "undefined or unsupported field validation", }) } func ErrorUnsupportedKey(key interface{}) error { return errors.WithStack(&errors.Error{ Kind: ErrUnsupportedKey, Message: fmt.Sprintf("key %s is not supported", s.UserStr(key)), }) } func ErrorInvalidYAML(err error) error { str := strings.TrimPrefix(errors.Message(err), "yaml: ") return errors.WithStack(&errors.Error{ Kind: ErrInvalidYAML, Message: fmt.Sprintf("invalid yaml: %s", str), }) } func ErrorTooLong(provided string, maxLen int) error { return errors.WithStack(&errors.Error{ Kind: ErrTooLong, Message: fmt.Sprintf("%s must be no more than %d characters", s.UserStr(provided), maxLen), }) } func ErrorTooShort(provided string, minLen int) error { return errors.WithStack(&errors.Error{ Kind: ErrTooShort, Message: fmt.Sprintf("%s must be at least %d characters", s.UserStr(provided), minLen), }) } func ErrorLeadingWhitespace(provided string) error { return errors.WithStack(&errors.Error{ Kind: ErrLeadingWhitespace, Message: fmt.Sprintf("%s cannot start with whitespace", s.UserStr(provided)), }) } func ErrorTrailingWhitespace(provided string) error { return errors.WithStack(&errors.Error{ Kind: ErrTrailingWhitespace, Message: fmt.Sprintf("%s cannot end with whitespace", s.UserStr(provided)), }) } func ErrorAlphaNumericDashUnderscore(provided string) error { return errors.WithStack(&errors.Error{ Kind: ErrAlphaNumericDashUnderscore, Message: fmt.Sprintf("%s must contain only letters, numbers, underscores, and dashes", s.UserStr(provided)), }) } func ErrorAlphaNumericDash(provided string) error { return errors.WithStack(&errors.Error{ Kind: ErrAlphaNumericDash, Message: fmt.Sprintf("%s must contain only letters, numbers, and dashes", s.UserStr(provided)), }) } func ErrorAlphaNumericDotUnderscore(provided string) error { return errors.WithStack(&errors.Error{ Kind: ErrAlphaNumericDotUnderscore, Message: fmt.Sprintf("%s must contain only letters, numbers, underscores and periods", s.UserStr(provided)), }) } func ErrorAlphaNumericDashDotUnderscore(provided string) error { return errors.WithStack(&errors.Error{ Kind: ErrAlphaNumericDashDotUnderscore, Message: fmt.Sprintf("%s must contain only letters, numbers, underscores, dashes, and periods", s.UserStr(provided)), }) } func ErrorInvalidAWSTag(provided string) error { return errors.WithStack(&errors.Error{ Kind: ErrInvalidAWSTag, Message: fmt.Sprintf("%s must contain only letters, numbers, spaces, and the following characters: _ . : / + - @", s.UserStr(provided)), }) } func ErrorInvalidDockerImage(provided string) error { return errors.WithStack(&errors.Error{ Kind: ErrInvalidDockerImage, Message: fmt.Sprintf("%s is not a valid docker image path", s.UserStr(provided)), }) } func ErrorMustHavePrefix(provided string, prefix string, prefixes ...string) error { allAllowedPrefixes := append([]string{prefix}, prefixes...) return errors.WithStack(&errors.Error{ Kind: ErrMustHavePrefix, Message: fmt.Sprintf("%s must start with %s", s.UserStr(provided), s.UserStrsOr(allAllowedPrefixes)), }) } func ErrorMustHaveSuffix(provided string, suffix string, suffixes ...string) error { allAllowedSuffixes := append([]string{suffix}, suffixes...) return errors.WithStack(&errors.Error{ Kind: ErrMustHaveSuffix, Message: fmt.Sprintf("%s must end with %s", s.UserStr(provided), s.UserStrsOr(allAllowedSuffixes)), }) } func ErrorCantHavePrefix(provided string, prefix string) error { return errors.WithStack(&errors.Error{ Kind: ErrCantHavePrefix, Message: fmt.Sprintf("%s cannot start with %s", s.UserStr(provided), s.UserStr(prefix)), }) } func ErrorCantHaveSuffix(provided string, suffix string) error { return errors.WithStack(&errors.Error{ Kind: ErrCantHaveSuffix, Message: fmt.Sprintf("%s cannot end with %s", s.UserStr(provided), s.UserStr(suffix)), }) } func ErrorInvalidInterface(provided interface{}, allowed interface{}, allowedVals ...interface{}) error { allAllowedVals := append([]interface{}{allowed}, allowedVals...) return errors.WithStack(&errors.Error{ Kind: ErrInvalidInterface, Message: fmt.Sprintf("invalid value (got %s, must be %s)", s.UserStr(provided), s.UserStrsOr(allAllowedVals)), }) } func ErrorInvalidFloat64(provided float64, allowed float64, allowedVals ...float64) error { allAllowedVals := append([]float64{allowed}, allowedVals...) return errors.WithStack(&errors.Error{ Kind: ErrInvalidFloat64, Message: fmt.Sprintf("invalid value (got %s, must be %s)", s.UserStr(provided), s.UserStrsOr(allAllowedVals)), }) } func ErrorInvalidFloat32(provided float32, allowed float32, allowedVals ...float32) error { allAllowedVals := append([]float32{allowed}, allowedVals...) return errors.WithStack(&errors.Error{ Kind: ErrInvalidFloat32, Message: fmt.Sprintf("invalid value (got %s, must be %s)", s.UserStr(provided), s.UserStrsOr(allAllowedVals)), }) } func ErrorInvalidInt64(provided int64, allowed int64, allowedVals ...int64) error { allAllowedVals := append([]int64{allowed}, allowedVals...) return errors.WithStack(&errors.Error{ Kind: ErrInvalidInt64, Message: fmt.Sprintf("invalid value (got %s, must be %s)", s.UserStr(provided), s.UserStrsOr(allAllowedVals)), }) } func ErrorInvalidInt32(provided int32, allowed int32, allowedVals ...int32) error { allAllowedVals := append([]int32{allowed}, allowedVals...) return errors.WithStack(&errors.Error{ Kind: ErrInvalidInt32, Message: fmt.Sprintf("invalid value (got %s, must be %s)", s.UserStr(provided), s.UserStrsOr(allAllowedVals)), }) } func ErrorInvalidInt(provided int, allowed int, allowedVals ...int) error { allAllowedVals := append([]int{allowed}, allowedVals...) return errors.WithStack(&errors.Error{ Kind: ErrInvalidInt, Message: fmt.Sprintf("invalid value (got %s, must be %s)", s.UserStr(provided), s.UserStrsOr(allAllowedVals)), }) } func ErrorInvalidStr(provided string, allowed string, allowedVals ...string) error { allAllowedVals := append([]string{allowed}, allowedVals...) return errors.WithStack(&errors.Error{ Kind: ErrInvalidStr, Message: fmt.Sprintf("invalid value (got %s, must be %s)", s.UserStr(provided), s.UserStrsOr(allAllowedVals)), }) } func ErrorDisallowedValue(provided interface{}) error { return errors.WithStack(&errors.Error{ Kind: ErrDisallowedValue, Message: fmt.Sprintf("%s is not allowed, please use a different value", s.UserStr(provided)), }) } func ErrorMustBeLessThanOrEqualTo(provided interface{}, boundary interface{}) error { return errors.WithStack(&errors.Error{ Kind: ErrMustBeLessThanOrEqualTo, Message: fmt.Sprintf("must be less than or equal to %s (got %s)", s.UserStr(boundary), s.UserStr(provided)), }) } func ErrorMustBeLessThan(provided interface{}, boundary interface{}) error { return errors.WithStack(&errors.Error{ Kind: ErrMustBeLessThan, Message: fmt.Sprintf("must be less than %s (got %s)", s.UserStr(boundary), s.UserStr(provided)), }) } func ErrorMustBeGreaterThanOrEqualTo(provided interface{}, boundary interface{}) error { return errors.WithStack(&errors.Error{ Kind: ErrMustBeGreaterThanOrEqualTo, Message: fmt.Sprintf("must be greater than or equal to %s (got %s)", s.UserStr(boundary), s.UserStr(provided)), }) } func ErrorMustBeGreaterThan(provided interface{}, boundary interface{}) error { return errors.WithStack(&errors.Error{ Kind: ErrMustBeGreaterThan, Message: fmt.Sprintf("must be greater than %s (got %s)", s.UserStr(boundary), s.UserStr(provided)), }) } func ErrorIsNotMultiple(provided interface{}, multiple interface{}) error { return errors.WithStack(&errors.Error{ Kind: ErrIsNotMultiple, Message: fmt.Sprintf("%s is not a multiple of %s", s.UserStr(provided), s.UserStr(multiple)), }) } func ErrorNonStringKeyFound(key interface{}) error { return errors.WithStack(&errors.Error{ Kind: ErrNonStringKeyFound, Message: fmt.Sprintf("non string key found: %s", s.ObjFlat(key)), }) } func ErrorInvalidPrimitiveType(provided interface{}, allowedType PrimitiveType, allowedTypes ...PrimitiveType) error { allAllowedTypes := append([]PrimitiveType{allowedType}, allowedTypes...) return errors.WithStack(&errors.Error{ Kind: ErrInvalidPrimitiveType, Message: fmt.Sprintf("%s: invalid type (expected %s)", s.UserStr(provided), s.StrsOr(PrimitiveTypes(allAllowedTypes).StringList())), }) } func ErrorDuplicatedValue(val interface{}) error { return errors.WithStack(&errors.Error{ Kind: ErrDuplicatedValue, Message: fmt.Sprintf("%s is duplicated", s.UserStr(val)), }) } func ErrorTooFewElements(minLength int) error { return errors.WithStack(&errors.Error{ Kind: ErrTooFewElements, Message: fmt.Sprintf("must contain at least %d elements", minLength), }) } func ErrorTooManyElements(maxLength int) error { return errors.WithStack(&errors.Error{ Kind: ErrTooManyElements, Message: fmt.Sprintf("must contain at most %d elements", maxLength), }) } func ErrorWrongNumberOfElements(invalidLengths []int) error { invalidElementsStr := "elements" if len(invalidLengths) == 1 && invalidLengths[0] == 1 { invalidElementsStr = "element" } invalidLengthStrs := make([]string, len(invalidLengths)) for i, length := range invalidLengths { invalidLengthStrs[i] = s.Int(length) } return errors.WithStack(&errors.Error{ Kind: ErrWrongNumberOfElements, Message: fmt.Sprintf("cannot contain %s %s", s.StrsOr(invalidLengthStrs), invalidElementsStr), }) } func ErrorCannotSetStructField() error { return errors.WithStack(&errors.Error{ Kind: ErrCannotSetStructField, Message: "unable to set struct field", }) } func ErrorCannotBeNull(isRequired bool) error { msg := "cannot be null" if !isRequired { msg = "cannot be null (specify a value, or remove the key to use the default value)" } return errors.WithStack(&errors.Error{ Kind: ErrCannotBeNull, Message: msg, }) } func ErrorCannotBeEmptyOrNull(isRequired bool) error { msg := "cannot be empty or null" if !isRequired { msg = "cannot be empty or null (specify a value, or remove the key to use the default value)" } return errors.WithStack(&errors.Error{ Kind: ErrCannotBeEmptyOrNull, Message: msg, }) } func ErrorCannotBeEmpty() error { return errors.WithStack(&errors.Error{ Kind: ErrCannotBeEmpty, Message: "cannot be empty", }) } func ErrorMustBeDefined(validValues ...interface{}) error { msg := "must be defined" if len(validValues) > 0 && !reflect.ValueOf(validValues[0]).IsNil() { // reflect is necessary here msg = fmt.Sprintf("must be defined, and set to %s", s.UserStrsOr(validValues)) } return errors.WithStack(&errors.Error{ Kind: ErrMustBeDefined, Message: msg, }) } func ErrorMapMustBeDefined(keys ...string) error { message := "must be defined" if len(keys) > 0 { message = fmt.Sprintf("must be defined, and contain the following keys: %s", s.UserStrsAnd(keys)) } return errors.WithStack(&errors.Error{ Kind: ErrMapMustBeDefined, Message: message, }) } func ErrorMustBeEmpty() error { return errors.WithStack(&errors.Error{ Kind: ErrMustBeEmpty, Message: "must be empty", }) } func ErrorEmailTooLong() error { return errors.WithStack(&errors.Error{ Kind: ErrEmailTooLong, Message: "email address exceeds maximum length", }) } func ErrorEmailInvalid() error { return errors.WithStack(&errors.Error{ Kind: ErrEmailInvalid, Message: "invalid email address", }) } func ErrorCortexResourceOnlyAllowed(invalidStr string) error { return errors.WithStack(&errors.Error{ Kind: ErrCortexResourceOnlyAllowed, Message: fmt.Sprintf("%s: only cortex resource references (which start with @) are allowed in this context", invalidStr), }) } func ErrorCortexResourceNotAllowed(resourceName string) error { return errors.WithStack(&errors.Error{ Kind: ErrCortexResourceNotAllowed, Message: fmt.Sprintf("@%s: cortex resource references (which start with @) are not allowed in this context", resourceName), }) } func ErrorImageVersionMismatch(image, tag, cortexVersion string) error { return errors.WithStack(&errors.Error{ Kind: ErrImageVersionMismatch, Message: fmt.Sprintf("the specified image (%s) has a tag (%s) which does not match your Cortex version (%s); please update the image tag, remove the image registry path from your configuration file (to use the default value), or update your CLI (pip install cortex==%s)", image, tag, cortexVersion, cortexVersion), }) } func ErrorFieldCantBeSpecified(errMsg string) error { message := errMsg if message == "" { message = "cannot be specified" } return errors.WithStack(&errors.Error{ Kind: ErrFieldCantBeSpecified, Message: message, }) } ================================================ FILE: pkg/lib/configreader/float32.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package configreader import ( "github.com/cortexlabs/cortex/pkg/lib/cast" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/exit" "github.com/cortexlabs/cortex/pkg/lib/files" "github.com/cortexlabs/cortex/pkg/lib/prompt" "github.com/cortexlabs/cortex/pkg/lib/slices" s "github.com/cortexlabs/cortex/pkg/lib/strings" ) type Float32Validation struct { Required bool Default float32 TreatNullAsZero bool // `: ` and `: null` will be read as `: 0.0` AllowedValues []float32 HiddenAllowedValues []float32 // allowed, but will not be listed as valid values (must be used in conjunction with AllowedValues) DisallowedValues []float32 CantBeSpecifiedErrStr *string GreaterThan *float32 GreaterThanOrEqualTo *float32 LessThan *float32 LessThanOrEqualTo *float32 Validator func(float32) (float32, error) } func Float32(inter interface{}, v *Float32Validation) (float32, error) { if inter == nil { if v.TreatNullAsZero { return ValidateFloat32Provided(0, v) } return 0, ErrorCannotBeNull(v.Required) } casted, castOk := cast.InterfaceToFloat32(inter) if !castOk { return 0, ErrorInvalidPrimitiveType(inter, PrimTypeFloat) } return ValidateFloat32Provided(casted, v) } func Float32FromInterfaceMap(key string, iMap map[string]interface{}, v *Float32Validation) (float32, error) { inter, ok := ReadInterfaceMapValue(key, iMap) if !ok { val, err := ValidateFloat32Missing(v) if err != nil { return 0, errors.Wrap(err, key) } return val, nil } val, err := Float32(inter, v) if err != nil { return 0, errors.Wrap(err, key) } return val, nil } func Float32FromStrMap(key string, sMap map[string]string, v *Float32Validation) (float32, error) { valStr, ok := sMap[key] if !ok || valStr == "" { val, err := ValidateFloat32Missing(v) if err != nil { return 0, errors.Wrap(err, key) } return val, nil } val, err := Float32FromStr(valStr, v) if err != nil { return 0, errors.Wrap(err, key) } return val, nil } func Float32FromStr(valStr string, v *Float32Validation) (float32, error) { if valStr == "" { return ValidateFloat32Missing(v) } casted, castOk := s.ParseFloat32(valStr) if !castOk { return 0, ErrorInvalidPrimitiveType(valStr, PrimTypeFloat) } return ValidateFloat32Provided(casted, v) } func Float32FromEnv(envVarName string, v *Float32Validation) (float32, error) { valStr := ReadEnvVar(envVarName) if valStr == nil || *valStr == "" { val, err := ValidateFloat32Missing(v) if err != nil { return 0, errors.Wrap(err, EnvVar(envVarName)) } return val, nil } val, err := Float32FromStr(*valStr, v) if err != nil { return 0, errors.Wrap(err, EnvVar(envVarName)) } return val, nil } func Float32FromFile(filePath string, v *Float32Validation) (float32, error) { if !files.IsFile(filePath) { val, err := ValidateFloat32Missing(v) if err != nil { return 0, errors.Wrap(err, filePath) } return val, nil } valStr, err := files.ReadFile(filePath) if err != nil { return 0, err } if len(valStr) == 0 { val, err := ValidateFloat32Missing(v) if err != nil { return 0, errors.Wrap(err, filePath) } return val, nil } val, err := Float32FromStr(valStr, v) if err != nil { return 0, errors.Wrap(err, filePath) } return val, nil } func Float32FromEnvOrFile(envVarName string, filePath string, v *Float32Validation) (float32, error) { valStr := ReadEnvVar(envVarName) if valStr != nil && *valStr != "" { return Float32FromEnv(envVarName, v) } return Float32FromFile(filePath, v) } func Float32FromPrompt(promptOpts *prompt.Options, v *Float32Validation) (float32, error) { promptOpts.DefaultStr = s.Float32(v.Default) valStr := prompt.Prompt(promptOpts) if valStr == "" { return ValidateFloat32Missing(v) } return Float32FromStr(valStr, v) } func ValidateFloat32Missing(v *Float32Validation) (float32, error) { if v.Required { return 0, ErrorMustBeDefined(v.AllowedValues) } return validateFloat32(v.Default, v) } func ValidateFloat32Provided(val float32, v *Float32Validation) (float32, error) { if v.CantBeSpecifiedErrStr != nil { return 0, ErrorFieldCantBeSpecified(*v.CantBeSpecifiedErrStr) } return validateFloat32(val, v) } func validateFloat32(val float32, v *Float32Validation) (float32, error) { err := ValidateFloat32Val(val, v) if err != nil { return 0, err } if v.Validator != nil { return v.Validator(val) } return val, nil } func ValidateFloat32Val(val float32, v *Float32Validation) error { if v.GreaterThan != nil { if val <= *v.GreaterThan { return ErrorMustBeGreaterThan(val, *v.GreaterThan) } } if v.GreaterThanOrEqualTo != nil { if val < *v.GreaterThanOrEqualTo { return ErrorMustBeGreaterThanOrEqualTo(val, *v.GreaterThanOrEqualTo) } } if v.LessThan != nil { if val >= *v.LessThan { return ErrorMustBeLessThan(val, *v.LessThan) } } if v.LessThanOrEqualTo != nil { if val > *v.LessThanOrEqualTo { return ErrorMustBeLessThanOrEqualTo(val, *v.LessThanOrEqualTo) } } if len(v.AllowedValues) > 0 { if !slices.HasFloat32(append(v.AllowedValues, v.HiddenAllowedValues...), val) { return ErrorInvalidFloat32(val, v.AllowedValues[0], v.AllowedValues[1:]...) } } if len(v.DisallowedValues) > 0 { if slices.HasFloat32(v.DisallowedValues, val) { return ErrorDisallowedValue(val) } } return nil } // // Musts // func MustFloat32FromEnv(envVarName string, v *Float32Validation) float32 { val, err := Float32FromEnv(envVarName, v) if err != nil { exit.Panic(err) } return val } func MustFloat32FromFile(filePath string, v *Float32Validation) float32 { val, err := Float32FromFile(filePath, v) if err != nil { exit.Panic(err) } return val } func MustFloat32FromEnvOrFile(envVarName string, filePath string, v *Float32Validation) float32 { val, err := Float32FromEnvOrFile(envVarName, filePath, v) if err != nil { exit.Panic(err) } return val } ================================================ FILE: pkg/lib/configreader/float32_list.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package configreader import ( "github.com/cortexlabs/cortex/pkg/lib/cast" "github.com/cortexlabs/cortex/pkg/lib/errors" ) type Float32ListValidation struct { Required bool Default []float32 AllowExplicitNull bool AllowEmpty bool CantBeSpecifiedErrStr *string CastSingleItem bool MinLength int MaxLength int InvalidLengths []int Validator func([]float32) ([]float32, error) } func Float32List(inter interface{}, v *Float32ListValidation) ([]float32, error) { casted, castOk := cast.InterfaceToFloat32Slice(inter) if !castOk { if v.CastSingleItem { castedItem, castOk := cast.InterfaceToFloat32(inter) if !castOk { return nil, ErrorInvalidPrimitiveType(inter, PrimTypeFloat, PrimTypeFloatList) } casted = []float32{castedItem} } else { return nil, ErrorInvalidPrimitiveType(inter, PrimTypeFloatList) } } return ValidateFloat32ListProvided(casted, v) } func Float32ListFromInterfaceMap(key string, iMap map[string]interface{}, v *Float32ListValidation) ([]float32, error) { inter, ok := ReadInterfaceMapValue(key, iMap) if !ok { val, err := ValidateFloat32ListMissing(v) if err != nil { return nil, errors.Wrap(err, key) } return val, nil } val, err := Float32List(inter, v) if err != nil { return nil, errors.Wrap(err, key) } return val, nil } func ValidateFloat32ListMissing(v *Float32ListValidation) ([]float32, error) { if v.Required { return nil, ErrorMustBeDefined() } return validateFloat32List(v.Default, v) } func ValidateFloat32ListProvided(val []float32, v *Float32ListValidation) ([]float32, error) { if v.CantBeSpecifiedErrStr != nil { return nil, ErrorFieldCantBeSpecified(*v.CantBeSpecifiedErrStr) } if !v.AllowExplicitNull && val == nil { return nil, ErrorCannotBeNull(v.Required) } return validateFloat32List(val, v) } func validateFloat32List(val []float32, v *Float32ListValidation) ([]float32, error) { if !v.AllowEmpty { if val != nil && len(val) == 0 { return nil, ErrorCannotBeEmpty() } } if v.MinLength != 0 { if len(val) < v.MinLength { return nil, ErrorTooFewElements(v.MinLength) } } if v.MaxLength != 0 { if len(val) > v.MaxLength { return nil, ErrorTooManyElements(v.MaxLength) } } for _, invalidLength := range v.InvalidLengths { if len(val) == invalidLength { return nil, ErrorWrongNumberOfElements(v.InvalidLengths) } } if v.Validator != nil { return v.Validator(val) } return val, nil } ================================================ FILE: pkg/lib/configreader/float32_ptr.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package configreader import ( "github.com/cortexlabs/cortex/pkg/lib/cast" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/files" "github.com/cortexlabs/cortex/pkg/lib/prompt" s "github.com/cortexlabs/cortex/pkg/lib/strings" ) type Float32PtrValidation struct { Required bool Default *float32 AllowExplicitNull bool AllowedValues []float32 HiddenAllowedValues []float32 // allowed, but will not be listed as valid values (must be used in conjunction with AllowedValues) DisallowedValues []float32 CantBeSpecifiedErrStr *string GreaterThan *float32 GreaterThanOrEqualTo *float32 LessThan *float32 LessThanOrEqualTo *float32 Validator func(float32) (float32, error) } func makeFloat32ValValidation(v *Float32PtrValidation) *Float32Validation { return &Float32Validation{ AllowedValues: v.AllowedValues, HiddenAllowedValues: v.HiddenAllowedValues, DisallowedValues: v.DisallowedValues, GreaterThan: v.GreaterThan, GreaterThanOrEqualTo: v.GreaterThanOrEqualTo, LessThan: v.LessThan, LessThanOrEqualTo: v.LessThanOrEqualTo, } } func Float32Ptr(inter interface{}, v *Float32PtrValidation) (*float32, error) { if inter == nil { return ValidateFloat32PtrProvided(nil, v) } casted, castOk := cast.InterfaceToFloat32(inter) if !castOk { return nil, ErrorInvalidPrimitiveType(inter, PrimTypeFloat) } return ValidateFloat32PtrProvided(&casted, v) } func Float32PtrFromInterfaceMap(key string, iMap map[string]interface{}, v *Float32PtrValidation) (*float32, error) { inter, ok := ReadInterfaceMapValue(key, iMap) if !ok { val, err := ValidateFloat32PtrMissing(v) if err != nil { return nil, errors.Wrap(err, key) } return val, nil } val, err := Float32Ptr(inter, v) if err != nil { return nil, errors.Wrap(err, key) } return val, nil } func Float32PtrFromStrMap(key string, sMap map[string]string, v *Float32PtrValidation) (*float32, error) { valStr, ok := sMap[key] if !ok || valStr == "" { val, err := ValidateFloat32PtrMissing(v) if err != nil { return nil, errors.Wrap(err, key) } return val, nil } val, err := Float32PtrFromStr(valStr, v) if err != nil { return nil, errors.Wrap(err, key) } return val, nil } func Float32PtrFromStr(valStr string, v *Float32PtrValidation) (*float32, error) { if valStr == "" { return ValidateFloat32PtrMissing(v) } casted, castOk := s.ParseFloat32(valStr) if !castOk { return nil, ErrorInvalidPrimitiveType(valStr, PrimTypeFloat) } return ValidateFloat32PtrProvided(&casted, v) } func Float32PtrFromEnv(envVarName string, v *Float32PtrValidation) (*float32, error) { valStr := ReadEnvVar(envVarName) if valStr == nil || *valStr == "" { val, err := ValidateFloat32PtrMissing(v) if err != nil { return nil, errors.Wrap(err, EnvVar(envVarName)) } return val, nil } val, err := Float32PtrFromStr(*valStr, v) if err != nil { return nil, errors.Wrap(err, EnvVar(envVarName)) } return val, nil } func Float32PtrFromFile(filePath string, v *Float32PtrValidation) (*float32, error) { if !files.IsFile(filePath) { val, err := ValidateFloat32PtrMissing(v) if err != nil { return nil, errors.Wrap(err, filePath) } return val, nil } valStr, err := files.ReadFile(filePath) if err != nil { return nil, err } if len(valStr) == 0 { val, err := ValidateFloat32PtrMissing(v) if err != nil { return nil, errors.Wrap(err, filePath) } return val, nil } val, err := Float32PtrFromStr(valStr, v) if err != nil { return nil, errors.Wrap(err, filePath) } return val, nil } func Float32PtrFromEnvOrFile(envVarName string, filePath string, v *Float32PtrValidation) (*float32, error) { valStr := ReadEnvVar(envVarName) if valStr != nil && *valStr != "" { return Float32PtrFromEnv(envVarName, v) } return Float32PtrFromFile(filePath, v) } func Float32PtrFromPrompt(promptOpts *prompt.Options, v *Float32PtrValidation) (*float32, error) { if v.Default != nil && promptOpts.DefaultStr == "" { promptOpts.DefaultStr = s.Float32(*v.Default) } valStr := prompt.Prompt(promptOpts) if valStr == "" { return ValidateFloat32PtrMissing(v) } return Float32PtrFromStr(valStr, v) } func ValidateFloat32PtrMissing(v *Float32PtrValidation) (*float32, error) { if v.Required { return nil, ErrorMustBeDefined(v.AllowedValues) } return validateFloat32Ptr(v.Default, v) } func ValidateFloat32PtrProvided(val *float32, v *Float32PtrValidation) (*float32, error) { if v.CantBeSpecifiedErrStr != nil { return nil, ErrorFieldCantBeSpecified(*v.CantBeSpecifiedErrStr) } if !v.AllowExplicitNull && val == nil { return nil, ErrorCannotBeNull(v.Required) } return validateFloat32Ptr(val, v) } func validateFloat32Ptr(val *float32, v *Float32PtrValidation) (*float32, error) { if val != nil { err := ValidateFloat32Val(*val, makeFloat32ValValidation(v)) if err != nil { return nil, err } } if val == nil { return val, nil } if v.Validator != nil { validated, err := v.Validator(*val) if err != nil { return nil, err } return &validated, nil } return val, nil } ================================================ FILE: pkg/lib/configreader/float64.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package configreader import ( "github.com/cortexlabs/cortex/pkg/lib/cast" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/exit" "github.com/cortexlabs/cortex/pkg/lib/files" "github.com/cortexlabs/cortex/pkg/lib/prompt" "github.com/cortexlabs/cortex/pkg/lib/slices" s "github.com/cortexlabs/cortex/pkg/lib/strings" ) type Float64Validation struct { Required bool Default float64 TreatNullAsZero bool // `: ` and `: null` will be read as `: 0.0` AllowedValues []float64 HiddenAllowedValues []float64 // allowed, but will not be listed as valid values (must be used in conjunction with AllowedValues) DisallowedValues []float64 CantBeSpecifiedErrStr *string GreaterThan *float64 GreaterThanOrEqualTo *float64 LessThan *float64 LessThanOrEqualTo *float64 Validator func(float64) (float64, error) } func Float64(inter interface{}, v *Float64Validation) (float64, error) { if inter == nil { if v.TreatNullAsZero { return ValidateFloat64Provided(0, v) } return 0, ErrorCannotBeNull(v.Required) } casted, castOk := cast.InterfaceToFloat64(inter) if !castOk { return 0, ErrorInvalidPrimitiveType(inter, PrimTypeFloat) } return ValidateFloat64Provided(casted, v) } func Float64FromInterfaceMap(key string, iMap map[string]interface{}, v *Float64Validation) (float64, error) { inter, ok := ReadInterfaceMapValue(key, iMap) if !ok { val, err := ValidateFloat64Missing(v) if err != nil { return 0, errors.Wrap(err, key) } return val, nil } val, err := Float64(inter, v) if err != nil { return 0, errors.Wrap(err, key) } return val, nil } func Float64FromStrMap(key string, sMap map[string]string, v *Float64Validation) (float64, error) { valStr, ok := sMap[key] if !ok || valStr == "" { val, err := ValidateFloat64Missing(v) if err != nil { return 0, errors.Wrap(err, key) } return val, nil } val, err := Float64FromStr(valStr, v) if err != nil { return 0, errors.Wrap(err, key) } return val, nil } func Float64FromStr(valStr string, v *Float64Validation) (float64, error) { if valStr == "" { return ValidateFloat64Missing(v) } casted, castOk := s.ParseFloat64(valStr) if !castOk { return 0, ErrorInvalidPrimitiveType(valStr, PrimTypeFloat) } return ValidateFloat64Provided(casted, v) } func Float64FromEnv(envVarName string, v *Float64Validation) (float64, error) { valStr := ReadEnvVar(envVarName) if valStr == nil || *valStr == "" { val, err := ValidateFloat64Missing(v) if err != nil { return 0, errors.Wrap(err, EnvVar(envVarName)) } return val, nil } val, err := Float64FromStr(*valStr, v) if err != nil { return 0, errors.Wrap(err, EnvVar(envVarName)) } return val, nil } func Float64FromFile(filePath string, v *Float64Validation) (float64, error) { if !files.IsFile(filePath) { val, err := ValidateFloat64Missing(v) if err != nil { return 0, errors.Wrap(err, filePath) } return val, nil } valStr, err := files.ReadFile(filePath) if err != nil { return 0, err } if len(valStr) == 0 { val, err := ValidateFloat64Missing(v) if err != nil { return 0, errors.Wrap(err, filePath) } return val, nil } val, err := Float64FromStr(valStr, v) if err != nil { return 0, errors.Wrap(err, filePath) } return val, nil } func Float64FromEnvOrFile(envVarName string, filePath string, v *Float64Validation) (float64, error) { valStr := ReadEnvVar(envVarName) if valStr != nil && *valStr != "" { return Float64FromEnv(envVarName, v) } return Float64FromFile(filePath, v) } func Float64FromPrompt(promptOpts *prompt.Options, v *Float64Validation) (float64, error) { promptOpts.DefaultStr = s.Float64(v.Default) valStr := prompt.Prompt(promptOpts) if valStr == "" { return ValidateFloat64Missing(v) } return Float64FromStr(valStr, v) } func ValidateFloat64Missing(v *Float64Validation) (float64, error) { if v.Required { return 0, ErrorMustBeDefined(v.AllowedValues) } return validateFloat64(v.Default, v) } func ValidateFloat64Provided(val float64, v *Float64Validation) (float64, error) { if v.CantBeSpecifiedErrStr != nil { return 0, ErrorFieldCantBeSpecified(*v.CantBeSpecifiedErrStr) } return validateFloat64(val, v) } func validateFloat64(val float64, v *Float64Validation) (float64, error) { err := ValidateFloat64Val(val, v) if err != nil { return 0, err } if v.Validator != nil { return v.Validator(val) } return val, nil } func ValidateFloat64Val(val float64, v *Float64Validation) error { if v.GreaterThan != nil { if val <= *v.GreaterThan { return ErrorMustBeGreaterThan(val, *v.GreaterThan) } } if v.GreaterThanOrEqualTo != nil { if val < *v.GreaterThanOrEqualTo { return ErrorMustBeGreaterThanOrEqualTo(val, *v.GreaterThanOrEqualTo) } } if v.LessThan != nil { if val >= *v.LessThan { return ErrorMustBeLessThan(val, *v.LessThan) } } if v.LessThanOrEqualTo != nil { if val > *v.LessThanOrEqualTo { return ErrorMustBeLessThanOrEqualTo(val, *v.LessThanOrEqualTo) } } if len(v.AllowedValues) > 0 { if !slices.HasFloat64(append(v.AllowedValues, v.HiddenAllowedValues...), val) { return ErrorInvalidFloat64(val, v.AllowedValues[0], v.AllowedValues[1:]...) } } if len(v.DisallowedValues) > 0 { if slices.HasFloat64(v.DisallowedValues, val) { return ErrorDisallowedValue(val) } } return nil } // // Musts // func MustFloat64FromEnv(envVarName string, v *Float64Validation) float64 { val, err := Float64FromEnv(envVarName, v) if err != nil { exit.Panic(err) } return val } func MustFloat64FromFile(filePath string, v *Float64Validation) float64 { val, err := Float64FromFile(filePath, v) if err != nil { exit.Panic(err) } return val } func MustFloat64FromEnvOrFile(envVarName string, filePath string, v *Float64Validation) float64 { val, err := Float64FromEnvOrFile(envVarName, filePath, v) if err != nil { exit.Panic(err) } return val } ================================================ FILE: pkg/lib/configreader/float64_list.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package configreader import ( "github.com/cortexlabs/cortex/pkg/lib/cast" "github.com/cortexlabs/cortex/pkg/lib/errors" ) type Float64ListValidation struct { Required bool Default []float64 AllowExplicitNull bool AllowEmpty bool CantBeSpecifiedErrStr *string CastSingleItem bool MinLength int MaxLength int InvalidLengths []int Validator func([]float64) ([]float64, error) } func Float64List(inter interface{}, v *Float64ListValidation) ([]float64, error) { casted, castOk := cast.InterfaceToFloat64Slice(inter) if !castOk { if v.CastSingleItem { castedItem, castOk := cast.InterfaceToFloat64(inter) if !castOk { return nil, ErrorInvalidPrimitiveType(inter, PrimTypeFloat, PrimTypeFloatList) } casted = []float64{castedItem} } else { return nil, ErrorInvalidPrimitiveType(inter, PrimTypeFloatList) } } return ValidateFloat64ListProvided(casted, v) } func Float64ListFromInterfaceMap(key string, iMap map[string]interface{}, v *Float64ListValidation) ([]float64, error) { inter, ok := ReadInterfaceMapValue(key, iMap) if !ok { val, err := ValidateFloat64ListMissing(v) if err != nil { return nil, errors.Wrap(err, key) } return val, nil } val, err := Float64List(inter, v) if err != nil { return nil, errors.Wrap(err, key) } return val, nil } func ValidateFloat64ListMissing(v *Float64ListValidation) ([]float64, error) { if v.Required { return nil, ErrorMustBeDefined() } return validateFloat64List(v.Default, v) } func ValidateFloat64ListProvided(val []float64, v *Float64ListValidation) ([]float64, error) { if v.CantBeSpecifiedErrStr != nil { return nil, ErrorFieldCantBeSpecified(*v.CantBeSpecifiedErrStr) } if !v.AllowExplicitNull && val == nil { return nil, ErrorCannotBeNull(v.Required) } return validateFloat64List(val, v) } func validateFloat64List(val []float64, v *Float64ListValidation) ([]float64, error) { if !v.AllowEmpty { if val != nil && len(val) == 0 { return nil, ErrorCannotBeEmpty() } } if v.MinLength != 0 { if len(val) < v.MinLength { return nil, ErrorTooFewElements(v.MinLength) } } if v.MaxLength != 0 { if len(val) > v.MaxLength { return nil, ErrorTooManyElements(v.MaxLength) } } for _, invalidLength := range v.InvalidLengths { if len(val) == invalidLength { return nil, ErrorWrongNumberOfElements(v.InvalidLengths) } } if v.Validator != nil { return v.Validator(val) } return val, nil } ================================================ FILE: pkg/lib/configreader/float64_ptr.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package configreader import ( "github.com/cortexlabs/cortex/pkg/lib/cast" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/files" "github.com/cortexlabs/cortex/pkg/lib/prompt" s "github.com/cortexlabs/cortex/pkg/lib/strings" ) type Float64PtrValidation struct { Required bool Default *float64 AllowExplicitNull bool AllowedValues []float64 HiddenAllowedValues []float64 // allowed, but will not be listed as valid values (must be used in conjunction with AllowedValues) DisallowedValues []float64 CantBeSpecifiedErrStr *string GreaterThan *float64 GreaterThanOrEqualTo *float64 LessThan *float64 LessThanOrEqualTo *float64 Validator func(float64) (float64, error) } func makeFloat64ValValidation(v *Float64PtrValidation) *Float64Validation { return &Float64Validation{ AllowedValues: v.AllowedValues, HiddenAllowedValues: v.HiddenAllowedValues, DisallowedValues: v.DisallowedValues, GreaterThan: v.GreaterThan, GreaterThanOrEqualTo: v.GreaterThanOrEqualTo, LessThan: v.LessThan, LessThanOrEqualTo: v.LessThanOrEqualTo, } } func Float64Ptr(inter interface{}, v *Float64PtrValidation) (*float64, error) { if inter == nil { return ValidateFloat64PtrProvided(nil, v) } casted, castOk := cast.InterfaceToFloat64(inter) if !castOk { return nil, ErrorInvalidPrimitiveType(inter, PrimTypeFloat) } return ValidateFloat64PtrProvided(&casted, v) } func Float64PtrFromInterfaceMap(key string, iMap map[string]interface{}, v *Float64PtrValidation) (*float64, error) { inter, ok := ReadInterfaceMapValue(key, iMap) if !ok { val, err := ValidateFloat64PtrMissing(v) if err != nil { return nil, errors.Wrap(err, key) } return val, nil } val, err := Float64Ptr(inter, v) if err != nil { return nil, errors.Wrap(err, key) } return val, nil } func Float64PtrFromStrMap(key string, sMap map[string]string, v *Float64PtrValidation) (*float64, error) { valStr, ok := sMap[key] if !ok || valStr == "" { val, err := ValidateFloat64PtrMissing(v) if err != nil { return nil, errors.Wrap(err, key) } return val, nil } val, err := Float64PtrFromStr(valStr, v) if err != nil { return nil, errors.Wrap(err, key) } return val, nil } func Float64PtrFromStr(valStr string, v *Float64PtrValidation) (*float64, error) { if valStr == "" { return ValidateFloat64PtrMissing(v) } casted, castOk := s.ParseFloat64(valStr) if !castOk { return nil, ErrorInvalidPrimitiveType(valStr, PrimTypeFloat) } return ValidateFloat64PtrProvided(&casted, v) } func Float64PtrFromEnv(envVarName string, v *Float64PtrValidation) (*float64, error) { valStr := ReadEnvVar(envVarName) if valStr == nil || *valStr == "" { val, err := ValidateFloat64PtrMissing(v) if err != nil { return nil, errors.Wrap(err, EnvVar(envVarName)) } return val, nil } val, err := Float64PtrFromStr(*valStr, v) if err != nil { return nil, errors.Wrap(err, EnvVar(envVarName)) } return val, nil } func Float64PtrFromFile(filePath string, v *Float64PtrValidation) (*float64, error) { if !files.IsFile(filePath) { val, err := ValidateFloat64PtrMissing(v) if err != nil { return nil, errors.Wrap(err, filePath) } return val, nil } valStr, err := files.ReadFile(filePath) if err != nil { return nil, err } if len(valStr) == 0 { val, err := ValidateFloat64PtrMissing(v) if err != nil { return nil, errors.Wrap(err, filePath) } return val, nil } val, err := Float64PtrFromStr(valStr, v) if err != nil { return nil, errors.Wrap(err, filePath) } return val, nil } func Float64PtrFromEnvOrFile(envVarName string, filePath string, v *Float64PtrValidation) (*float64, error) { valStr := ReadEnvVar(envVarName) if valStr != nil && *valStr != "" { return Float64PtrFromEnv(envVarName, v) } return Float64PtrFromFile(filePath, v) } func Float64PtrFromPrompt(promptOpts *prompt.Options, v *Float64PtrValidation) (*float64, error) { if v.Default != nil && promptOpts.DefaultStr == "" { promptOpts.DefaultStr = s.Float64(*v.Default) } valStr := prompt.Prompt(promptOpts) if valStr == "" { return ValidateFloat64PtrMissing(v) } return Float64PtrFromStr(valStr, v) } func ValidateFloat64PtrMissing(v *Float64PtrValidation) (*float64, error) { if v.Required { return nil, ErrorMustBeDefined(v.AllowedValues) } return validateFloat64Ptr(v.Default, v) } func ValidateFloat64PtrProvided(val *float64, v *Float64PtrValidation) (*float64, error) { if v.CantBeSpecifiedErrStr != nil { return nil, ErrorFieldCantBeSpecified(*v.CantBeSpecifiedErrStr) } if !v.AllowExplicitNull && val == nil { return nil, ErrorCannotBeNull(v.Required) } return validateFloat64Ptr(val, v) } func validateFloat64Ptr(val *float64, v *Float64PtrValidation) (*float64, error) { if val != nil { err := ValidateFloat64Val(*val, makeFloat64ValValidation(v)) if err != nil { return nil, err } } if val == nil { return val, nil } if v.Validator != nil { validated, err := v.Validator(*val) if err != nil { return nil, err } return &validated, nil } return val, nil } ================================================ FILE: pkg/lib/configreader/int.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package configreader import ( "github.com/cortexlabs/cortex/pkg/lib/cast" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/exit" "github.com/cortexlabs/cortex/pkg/lib/files" "github.com/cortexlabs/cortex/pkg/lib/prompt" "github.com/cortexlabs/cortex/pkg/lib/slices" s "github.com/cortexlabs/cortex/pkg/lib/strings" ) type IntValidation struct { Required bool Default int TreatNullAsZero bool // `: ` and `: null` will be read as `: 0` AllowedValues []int HiddenAllowedValues []int // allowed, but will not be listed as valid values (must be used in conjunction with AllowedValues) DisallowedValues []int CantBeSpecifiedErrStr *string GreaterThan *int GreaterThanOrEqualTo *int LessThan *int LessThanOrEqualTo *int Validator func(int) (int, error) } func Int(inter interface{}, v *IntValidation) (int, error) { if inter == nil { if v.TreatNullAsZero { return ValidateIntProvided(0, v) } return 0, ErrorCannotBeNull(v.Required) } casted, castOk := cast.InterfaceToInt(inter) if !castOk { return 0, ErrorInvalidPrimitiveType(inter, PrimTypeInt) } return ValidateIntProvided(casted, v) } func IntFromInterfaceMap(key string, iMap map[string]interface{}, v *IntValidation) (int, error) { inter, ok := ReadInterfaceMapValue(key, iMap) if !ok { val, err := ValidateIntMissing(v) if err != nil { return 0, errors.Wrap(err, key) } return val, nil } val, err := Int(inter, v) if err != nil { return 0, errors.Wrap(err, key) } return val, nil } func IntFromStrMap(key string, sMap map[string]string, v *IntValidation) (int, error) { valStr, ok := sMap[key] if !ok || valStr == "" { val, err := ValidateIntMissing(v) if err != nil { return 0, errors.Wrap(err, key) } return val, nil } val, err := IntFromStr(valStr, v) if err != nil { return 0, errors.Wrap(err, key) } return val, nil } func IntFromStr(valStr string, v *IntValidation) (int, error) { if valStr == "" { return ValidateIntMissing(v) } casted, castOk := s.ParseInt(valStr) if !castOk { return 0, ErrorInvalidPrimitiveType(valStr, PrimTypeInt) } return ValidateIntProvided(casted, v) } func IntFromEnv(envVarName string, v *IntValidation) (int, error) { valStr := ReadEnvVar(envVarName) if valStr == nil || *valStr == "" { val, err := ValidateIntMissing(v) if err != nil { return 0, errors.Wrap(err, EnvVar(envVarName)) } return val, nil } val, err := IntFromStr(*valStr, v) if err != nil { return 0, errors.Wrap(err, EnvVar(envVarName)) } return val, nil } func IntFromFile(filePath string, v *IntValidation) (int, error) { if !files.IsFile(filePath) { val, err := ValidateIntMissing(v) if err != nil { return 0, errors.Wrap(err, filePath) } return val, nil } valStr, err := files.ReadFile(filePath) if err != nil { return 0, err } if len(valStr) == 0 { val, err := ValidateIntMissing(v) if err != nil { return 0, errors.Wrap(err, filePath) } return val, nil } val, err := IntFromStr(valStr, v) if err != nil { return 0, errors.Wrap(err, filePath) } return val, nil } func IntFromEnvOrFile(envVarName string, filePath string, v *IntValidation) (int, error) { valStr := ReadEnvVar(envVarName) if valStr != nil && *valStr != "" { return IntFromEnv(envVarName, v) } return IntFromFile(filePath, v) } func IntFromPrompt(promptOpts *prompt.Options, v *IntValidation) (int, error) { promptOpts.DefaultStr = s.Int(v.Default) valStr := prompt.Prompt(promptOpts) if valStr == "" { return ValidateIntMissing(v) } return IntFromStr(valStr, v) } func ValidateIntMissing(v *IntValidation) (int, error) { if v.Required { return 0, ErrorMustBeDefined(v.AllowedValues) } return validateInt(v.Default, v) } func ValidateIntProvided(val int, v *IntValidation) (int, error) { if v.CantBeSpecifiedErrStr != nil { return 0, ErrorFieldCantBeSpecified(*v.CantBeSpecifiedErrStr) } return validateInt(val, v) } func validateInt(val int, v *IntValidation) (int, error) { err := ValidateIntVal(val, v) if err != nil { return 0, err } if v.Validator != nil { return v.Validator(val) } return val, nil } func ValidateIntVal(val int, v *IntValidation) error { if v.GreaterThan != nil { if val <= *v.GreaterThan { return ErrorMustBeGreaterThan(val, *v.GreaterThan) } } if v.GreaterThanOrEqualTo != nil { if val < *v.GreaterThanOrEqualTo { return ErrorMustBeGreaterThanOrEqualTo(val, *v.GreaterThanOrEqualTo) } } if v.LessThan != nil { if val >= *v.LessThan { return ErrorMustBeLessThan(val, *v.LessThan) } } if v.LessThanOrEqualTo != nil { if val > *v.LessThanOrEqualTo { return ErrorMustBeLessThanOrEqualTo(val, *v.LessThanOrEqualTo) } } if len(v.AllowedValues) > 0 { if !slices.HasInt(append(v.AllowedValues, v.HiddenAllowedValues...), val) { return ErrorInvalidInt(val, v.AllowedValues[0], v.AllowedValues[1:]...) } } if len(v.DisallowedValues) > 0 { if slices.HasInt(v.DisallowedValues, val) { return ErrorDisallowedValue(val) } } return nil } // // Musts // func MustIntFromEnv(envVarName string, v *IntValidation) int { val, err := IntFromEnv(envVarName, v) if err != nil { exit.Panic(err) } return val } func MustIntFromFile(filePath string, v *IntValidation) int { val, err := IntFromFile(filePath, v) if err != nil { exit.Panic(err) } return val } func MustIntFromEnvOrFile(envVarName string, filePath string, v *IntValidation) int { val, err := IntFromEnvOrFile(envVarName, filePath, v) if err != nil { exit.Panic(err) } return val } ================================================ FILE: pkg/lib/configreader/int32.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package configreader import ( "github.com/cortexlabs/cortex/pkg/lib/cast" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/exit" "github.com/cortexlabs/cortex/pkg/lib/files" "github.com/cortexlabs/cortex/pkg/lib/prompt" "github.com/cortexlabs/cortex/pkg/lib/slices" s "github.com/cortexlabs/cortex/pkg/lib/strings" ) type Int32Validation struct { Required bool Default int32 TreatNullAsZero bool // `: ` and `: null` will be read as `: 0` AllowedValues []int32 HiddenAllowedValues []int32 // allowed, but will not be listed as valid values (must be used in conjunction with AllowedValues) DisallowedValues []int32 CantBeSpecifiedErrStr *string GreaterThan *int32 GreaterThanOrEqualTo *int32 LessThan *int32 LessThanOrEqualTo *int32 Validator func(int32) (int32, error) } func Int32(inter interface{}, v *Int32Validation) (int32, error) { if inter == nil { if v.TreatNullAsZero { return ValidateInt32Provided(0, v) } return 0, ErrorCannotBeNull(v.Required) } casted, castOk := cast.InterfaceToInt32(inter) if !castOk { return 0, ErrorInvalidPrimitiveType(inter, PrimTypeInt) } return ValidateInt32Provided(casted, v) } func Int32FromInterfaceMap(key string, iMap map[string]interface{}, v *Int32Validation) (int32, error) { inter, ok := ReadInterfaceMapValue(key, iMap) if !ok { val, err := ValidateInt32Missing(v) if err != nil { return 0, errors.Wrap(err, key) } return val, nil } val, err := Int32(inter, v) if err != nil { return 0, errors.Wrap(err, key) } return val, nil } func Int32FromStrMap(key string, sMap map[string]string, v *Int32Validation) (int32, error) { valStr, ok := sMap[key] if !ok || valStr == "" { val, err := ValidateInt32Missing(v) if err != nil { return 0, errors.Wrap(err, key) } return val, nil } val, err := Int32FromStr(valStr, v) if err != nil { return 0, errors.Wrap(err, key) } return val, nil } func Int32FromStr(valStr string, v *Int32Validation) (int32, error) { if valStr == "" { return ValidateInt32Missing(v) } casted, castOk := s.ParseInt32(valStr) if !castOk { return 0, ErrorInvalidPrimitiveType(valStr, PrimTypeInt) } return ValidateInt32Provided(casted, v) } func Int32FromEnv(envVarName string, v *Int32Validation) (int32, error) { valStr := ReadEnvVar(envVarName) if valStr == nil || *valStr == "" { val, err := ValidateInt32Missing(v) if err != nil { return 0, errors.Wrap(err, EnvVar(envVarName)) } return val, nil } val, err := Int32FromStr(*valStr, v) if err != nil { return 0, errors.Wrap(err, EnvVar(envVarName)) } return val, nil } func Int32FromFile(filePath string, v *Int32Validation) (int32, error) { if !files.IsFile(filePath) { val, err := ValidateInt32Missing(v) if err != nil { return 0, errors.Wrap(err, filePath) } return val, nil } valStr, err := files.ReadFile(filePath) if err != nil { return 0, err } if len(valStr) == 0 { val, err := ValidateInt32Missing(v) if err != nil { return 0, errors.Wrap(err, filePath) } return val, nil } val, err := Int32FromStr(valStr, v) if err != nil { return 0, errors.Wrap(err, filePath) } return val, nil } func Int32FromEnvOrFile(envVarName string, filePath string, v *Int32Validation) (int32, error) { valStr := ReadEnvVar(envVarName) if valStr != nil && *valStr != "" { return Int32FromEnv(envVarName, v) } return Int32FromFile(filePath, v) } func Int32FromPrompt(promptOpts *prompt.Options, v *Int32Validation) (int32, error) { promptOpts.DefaultStr = s.Int32(v.Default) valStr := prompt.Prompt(promptOpts) if valStr == "" { return ValidateInt32Missing(v) } return Int32FromStr(valStr, v) } func ValidateInt32Missing(v *Int32Validation) (int32, error) { if v.Required { return 0, ErrorMustBeDefined(v.AllowedValues) } return validateInt32(v.Default, v) } func ValidateInt32Provided(val int32, v *Int32Validation) (int32, error) { if v.CantBeSpecifiedErrStr != nil { return 0, ErrorFieldCantBeSpecified(*v.CantBeSpecifiedErrStr) } return validateInt32(val, v) } func validateInt32(val int32, v *Int32Validation) (int32, error) { err := ValidateInt32Val(val, v) if err != nil { return 0, err } if v.Validator != nil { return v.Validator(val) } return val, nil } func ValidateInt32Val(val int32, v *Int32Validation) error { if v.GreaterThan != nil { if val <= *v.GreaterThan { return ErrorMustBeGreaterThan(val, *v.GreaterThan) } } if v.GreaterThanOrEqualTo != nil { if val < *v.GreaterThanOrEqualTo { return ErrorMustBeGreaterThanOrEqualTo(val, *v.GreaterThanOrEqualTo) } } if v.LessThan != nil { if val >= *v.LessThan { return ErrorMustBeLessThan(val, *v.LessThan) } } if v.LessThanOrEqualTo != nil { if val > *v.LessThanOrEqualTo { return ErrorMustBeLessThanOrEqualTo(val, *v.LessThanOrEqualTo) } } if len(v.AllowedValues) > 0 { if !slices.HasInt32(append(v.AllowedValues, v.HiddenAllowedValues...), val) { return ErrorInvalidInt32(val, v.AllowedValues[0], v.AllowedValues[1:]...) } } if len(v.DisallowedValues) > 0 { if slices.HasInt32(v.DisallowedValues, val) { return ErrorDisallowedValue(val) } } return nil } // // Musts // func MustInt32FromEnv(envVarName string, v *Int32Validation) int32 { val, err := Int32FromEnv(envVarName, v) if err != nil { exit.Panic(err) } return val } func MustInt32FromFile(filePath string, v *Int32Validation) int32 { val, err := Int32FromFile(filePath, v) if err != nil { exit.Panic(err) } return val } func MustInt32FromEnvOrFile(envVarName string, filePath string, v *Int32Validation) int32 { val, err := Int32FromEnvOrFile(envVarName, filePath, v) if err != nil { exit.Panic(err) } return val } ================================================ FILE: pkg/lib/configreader/int32_list.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package configreader import ( "github.com/cortexlabs/cortex/pkg/lib/cast" "github.com/cortexlabs/cortex/pkg/lib/errors" ) type Int32ListValidation struct { Required bool Default []int32 AllowExplicitNull bool AllowEmpty bool CantBeSpecifiedErrStr *string CastSingleItem bool MinLength int MaxLength int InvalidLengths []int Validator func([]int32) ([]int32, error) } func Int32List(inter interface{}, v *Int32ListValidation) ([]int32, error) { casted, castOk := cast.InterfaceToInt32Slice(inter) if !castOk { if v.CastSingleItem { castedItem, castOk := cast.InterfaceToInt32(inter) if !castOk { return nil, ErrorInvalidPrimitiveType(inter, PrimTypeInt, PrimTypeIntList) } casted = []int32{castedItem} } else { return nil, ErrorInvalidPrimitiveType(inter, PrimTypeIntList) } } return ValidateInt32ListProvided(casted, v) } func Int32ListFromInterfaceMap(key string, iMap map[string]interface{}, v *Int32ListValidation) ([]int32, error) { inter, ok := ReadInterfaceMapValue(key, iMap) if !ok { val, err := ValidateInt32ListMissing(v) if err != nil { return nil, errors.Wrap(err, key) } return val, nil } val, err := Int32List(inter, v) if err != nil { return nil, errors.Wrap(err, key) } return val, nil } func ValidateInt32ListMissing(v *Int32ListValidation) ([]int32, error) { if v.Required { return nil, ErrorMustBeDefined() } return validateInt32List(v.Default, v) } func ValidateInt32ListProvided(val []int32, v *Int32ListValidation) ([]int32, error) { if v.CantBeSpecifiedErrStr != nil { return nil, ErrorFieldCantBeSpecified(*v.CantBeSpecifiedErrStr) } if !v.AllowExplicitNull && val == nil { return nil, ErrorCannotBeNull(v.Required) } return validateInt32List(val, v) } func validateInt32List(val []int32, v *Int32ListValidation) ([]int32, error) { if !v.AllowEmpty { if val != nil && len(val) == 0 { return nil, ErrorCannotBeEmpty() } } if v.MinLength != 0 { if len(val) < v.MinLength { return nil, ErrorTooFewElements(v.MinLength) } } if v.MaxLength != 0 { if len(val) > v.MaxLength { return nil, ErrorTooManyElements(v.MaxLength) } } for _, invalidLength := range v.InvalidLengths { if len(val) == invalidLength { return nil, ErrorWrongNumberOfElements(v.InvalidLengths) } } if v.Validator != nil { return v.Validator(val) } return val, nil } ================================================ FILE: pkg/lib/configreader/int32_ptr.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package configreader import ( "github.com/cortexlabs/cortex/pkg/lib/cast" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/files" "github.com/cortexlabs/cortex/pkg/lib/prompt" s "github.com/cortexlabs/cortex/pkg/lib/strings" ) type Int32PtrValidation struct { Required bool Default *int32 AllowExplicitNull bool AllowedValues []int32 HiddenAllowedValues []int32 // allowed, but will not be listed as valid values (must be used in conjunction with AllowedValues) DisallowedValues []int32 CantBeSpecifiedErrStr *string GreaterThan *int32 GreaterThanOrEqualTo *int32 LessThan *int32 LessThanOrEqualTo *int32 Validator func(int32) (int32, error) } func makeInt32ValValidation(v *Int32PtrValidation) *Int32Validation { return &Int32Validation{ AllowedValues: v.AllowedValues, HiddenAllowedValues: v.HiddenAllowedValues, DisallowedValues: v.DisallowedValues, GreaterThan: v.GreaterThan, GreaterThanOrEqualTo: v.GreaterThanOrEqualTo, LessThan: v.LessThan, LessThanOrEqualTo: v.LessThanOrEqualTo, } } func Int32Ptr(inter interface{}, v *Int32PtrValidation) (*int32, error) { if inter == nil { return ValidateInt32PtrProvided(nil, v) } casted, castOk := cast.InterfaceToInt32(inter) if !castOk { return nil, ErrorInvalidPrimitiveType(inter, PrimTypeInt) } return ValidateInt32PtrProvided(&casted, v) } func Int32PtrFromInterfaceMap(key string, iMap map[string]interface{}, v *Int32PtrValidation) (*int32, error) { inter, ok := ReadInterfaceMapValue(key, iMap) if !ok { val, err := ValidateInt32PtrMissing(v) if err != nil { return nil, errors.Wrap(err, key) } return val, nil } val, err := Int32Ptr(inter, v) if err != nil { return nil, errors.Wrap(err, key) } return val, nil } func Int32PtrFromStrMap(key string, sMap map[string]string, v *Int32PtrValidation) (*int32, error) { valStr, ok := sMap[key] if !ok || valStr == "" { val, err := ValidateInt32PtrMissing(v) if err != nil { return nil, errors.Wrap(err, key) } return val, nil } val, err := Int32PtrFromStr(valStr, v) if err != nil { return nil, errors.Wrap(err, key) } return val, nil } func Int32PtrFromStr(valStr string, v *Int32PtrValidation) (*int32, error) { if valStr == "" { return ValidateInt32PtrMissing(v) } casted, castOk := s.ParseInt32(valStr) if !castOk { return nil, ErrorInvalidPrimitiveType(valStr, PrimTypeInt) } return ValidateInt32PtrProvided(&casted, v) } func Int32PtrFromEnv(envVarName string, v *Int32PtrValidation) (*int32, error) { valStr := ReadEnvVar(envVarName) if valStr == nil || *valStr == "" { val, err := ValidateInt32PtrMissing(v) if err != nil { return nil, errors.Wrap(err, EnvVar(envVarName)) } return val, nil } val, err := Int32PtrFromStr(*valStr, v) if err != nil { return nil, errors.Wrap(err, EnvVar(envVarName)) } return val, nil } func Int32PtrFromFile(filePath string, v *Int32PtrValidation) (*int32, error) { if !files.IsFile(filePath) { val, err := ValidateInt32PtrMissing(v) if err != nil { return nil, errors.Wrap(err, filePath) } return val, nil } valStr, err := files.ReadFile(filePath) if err != nil { return nil, err } if len(valStr) == 0 { val, err := ValidateInt32PtrMissing(v) if err != nil { return nil, errors.Wrap(err, filePath) } return val, nil } val, err := Int32PtrFromStr(valStr, v) if err != nil { return nil, errors.Wrap(err, filePath) } return val, nil } func Int32PtrFromEnvOrFile(envVarName string, filePath string, v *Int32PtrValidation) (*int32, error) { valStr := ReadEnvVar(envVarName) if valStr != nil && *valStr != "" { return Int32PtrFromEnv(envVarName, v) } return Int32PtrFromFile(filePath, v) } func Int32PtrFromPrompt(promptOpts *prompt.Options, v *Int32PtrValidation) (*int32, error) { if v.Default != nil && promptOpts.DefaultStr == "" { promptOpts.DefaultStr = s.Int32(*v.Default) } valStr := prompt.Prompt(promptOpts) if valStr == "" { return ValidateInt32PtrMissing(v) } return Int32PtrFromStr(valStr, v) } func ValidateInt32PtrMissing(v *Int32PtrValidation) (*int32, error) { if v.Required { return nil, ErrorMustBeDefined(v.AllowedValues) } return validateInt32Ptr(v.Default, v) } func ValidateInt32PtrProvided(val *int32, v *Int32PtrValidation) (*int32, error) { if v.CantBeSpecifiedErrStr != nil { return nil, ErrorFieldCantBeSpecified(*v.CantBeSpecifiedErrStr) } if !v.AllowExplicitNull && val == nil { return nil, ErrorCannotBeNull(v.Required) } return validateInt32Ptr(val, v) } func validateInt32Ptr(val *int32, v *Int32PtrValidation) (*int32, error) { if val != nil { err := ValidateInt32Val(*val, makeInt32ValValidation(v)) if err != nil { return nil, err } } if val == nil { return val, nil } if v.Validator != nil { validated, err := v.Validator(*val) if err != nil { return nil, err } return &validated, nil } return val, nil } ================================================ FILE: pkg/lib/configreader/int64.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package configreader import ( "github.com/cortexlabs/cortex/pkg/lib/cast" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/exit" "github.com/cortexlabs/cortex/pkg/lib/files" "github.com/cortexlabs/cortex/pkg/lib/prompt" "github.com/cortexlabs/cortex/pkg/lib/slices" s "github.com/cortexlabs/cortex/pkg/lib/strings" ) type Int64Validation struct { Required bool Default int64 TreatNullAsZero bool // `: ` and `: null` will be read as `: 0` AllowedValues []int64 HiddenAllowedValues []int64 // allowed, but will not be listed as valid values (must be used in conjunction with AllowedValues) DisallowedValues []int64 CantBeSpecifiedErrStr *string GreaterThan *int64 GreaterThanOrEqualTo *int64 LessThan *int64 LessThanOrEqualTo *int64 Validator func(int64) (int64, error) } func Int64(inter interface{}, v *Int64Validation) (int64, error) { if inter == nil { if v.TreatNullAsZero { return ValidateInt64Provided(0, v) } return 0, ErrorCannotBeNull(v.Required) } casted, castOk := cast.InterfaceToInt64(inter) if !castOk { return 0, ErrorInvalidPrimitiveType(inter, PrimTypeInt) } return ValidateInt64Provided(casted, v) } func Int64FromInterfaceMap(key string, iMap map[string]interface{}, v *Int64Validation) (int64, error) { inter, ok := ReadInterfaceMapValue(key, iMap) if !ok { val, err := ValidateInt64Missing(v) if err != nil { return 0, errors.Wrap(err, key) } return val, nil } val, err := Int64(inter, v) if err != nil { return 0, errors.Wrap(err, key) } return val, nil } func Int64FromStrMap(key string, sMap map[string]string, v *Int64Validation) (int64, error) { valStr, ok := sMap[key] if !ok || valStr == "" { val, err := ValidateInt64Missing(v) if err != nil { return 0, errors.Wrap(err, key) } return val, nil } val, err := Int64FromStr(valStr, v) if err != nil { return 0, errors.Wrap(err, key) } return val, nil } func Int64FromStr(valStr string, v *Int64Validation) (int64, error) { if valStr == "" { return ValidateInt64Missing(v) } casted, castOk := s.ParseInt64(valStr) if !castOk { return 0, ErrorInvalidPrimitiveType(valStr, PrimTypeInt) } return ValidateInt64Provided(casted, v) } func Int64FromEnv(envVarName string, v *Int64Validation) (int64, error) { valStr := ReadEnvVar(envVarName) if valStr == nil || *valStr == "" { val, err := ValidateInt64Missing(v) if err != nil { return 0, errors.Wrap(err, EnvVar(envVarName)) } return val, nil } val, err := Int64FromStr(*valStr, v) if err != nil { return 0, errors.Wrap(err, EnvVar(envVarName)) } return val, nil } func Int64FromFile(filePath string, v *Int64Validation) (int64, error) { if !files.IsFile(filePath) { val, err := ValidateInt64Missing(v) if err != nil { return 0, errors.Wrap(err, filePath) } return val, nil } valStr, err := files.ReadFile(filePath) if err != nil { return 0, err } if len(valStr) == 0 { val, err := ValidateInt64Missing(v) if err != nil { return 0, errors.Wrap(err, filePath) } return val, nil } val, err := Int64FromStr(valStr, v) if err != nil { return 0, errors.Wrap(err, filePath) } return val, nil } func Int64FromEnvOrFile(envVarName string, filePath string, v *Int64Validation) (int64, error) { valStr := ReadEnvVar(envVarName) if valStr != nil && *valStr != "" { return Int64FromEnv(envVarName, v) } return Int64FromFile(filePath, v) } func Int64FromPrompt(promptOpts *prompt.Options, v *Int64Validation) (int64, error) { promptOpts.DefaultStr = s.Int64(v.Default) valStr := prompt.Prompt(promptOpts) if valStr == "" { return ValidateInt64Missing(v) } return Int64FromStr(valStr, v) } func ValidateInt64Missing(v *Int64Validation) (int64, error) { if v.Required { return 0, ErrorMustBeDefined(v.AllowedValues) } return validateInt64(v.Default, v) } func ValidateInt64Provided(val int64, v *Int64Validation) (int64, error) { if v.CantBeSpecifiedErrStr != nil { return 0, ErrorFieldCantBeSpecified(*v.CantBeSpecifiedErrStr) } return validateInt64(val, v) } func validateInt64(val int64, v *Int64Validation) (int64, error) { err := ValidateInt64Val(val, v) if err != nil { return 0, err } if v.Validator != nil { return v.Validator(val) } return val, nil } func ValidateInt64Val(val int64, v *Int64Validation) error { if v.GreaterThan != nil { if val <= *v.GreaterThan { return ErrorMustBeGreaterThan(val, *v.GreaterThan) } } if v.GreaterThanOrEqualTo != nil { if val < *v.GreaterThanOrEqualTo { return ErrorMustBeGreaterThanOrEqualTo(val, *v.GreaterThanOrEqualTo) } } if v.LessThan != nil { if val >= *v.LessThan { return ErrorMustBeLessThan(val, *v.LessThan) } } if v.LessThanOrEqualTo != nil { if val > *v.LessThanOrEqualTo { return ErrorMustBeLessThanOrEqualTo(val, *v.LessThanOrEqualTo) } } if len(v.AllowedValues) > 0 { if !slices.HasInt64(append(v.AllowedValues, v.HiddenAllowedValues...), val) { return ErrorInvalidInt64(val, v.AllowedValues[0], v.AllowedValues[1:]...) } } if len(v.DisallowedValues) > 0 { if slices.HasInt64(v.DisallowedValues, val) { return ErrorDisallowedValue(val) } } return nil } // // Musts // func MustInt64FromEnv(envVarName string, v *Int64Validation) int64 { val, err := Int64FromEnv(envVarName, v) if err != nil { exit.Panic(err) } return val } func MustInt64FromFile(filePath string, v *Int64Validation) int64 { val, err := Int64FromFile(filePath, v) if err != nil { exit.Panic(err) } return val } func MustInt64FromEnvOrFile(envVarName string, filePath string, v *Int64Validation) int64 { val, err := Int64FromEnvOrFile(envVarName, filePath, v) if err != nil { exit.Panic(err) } return val } ================================================ FILE: pkg/lib/configreader/int64_list.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package configreader import ( "github.com/cortexlabs/cortex/pkg/lib/cast" "github.com/cortexlabs/cortex/pkg/lib/errors" ) type Int64ListValidation struct { Required bool Default []int64 AllowExplicitNull bool AllowEmpty bool CantBeSpecifiedErrStr *string CastSingleItem bool MinLength int MaxLength int InvalidLengths []int Validator func([]int64) ([]int64, error) } func Int64List(inter interface{}, v *Int64ListValidation) ([]int64, error) { casted, castOk := cast.InterfaceToInt64Slice(inter) if !castOk { if v.CastSingleItem { castedItem, castOk := cast.InterfaceToInt64(inter) if !castOk { return nil, ErrorInvalidPrimitiveType(inter, PrimTypeInt, PrimTypeIntList) } casted = []int64{castedItem} } else { return nil, ErrorInvalidPrimitiveType(inter, PrimTypeIntList) } } return ValidateInt64ListProvided(casted, v) } func Int64ListFromInterfaceMap(key string, iMap map[string]interface{}, v *Int64ListValidation) ([]int64, error) { inter, ok := ReadInterfaceMapValue(key, iMap) if !ok { val, err := ValidateInt64ListMissing(v) if err != nil { return nil, errors.Wrap(err, key) } return val, nil } val, err := Int64List(inter, v) if err != nil { return nil, errors.Wrap(err, key) } return val, nil } func ValidateInt64ListMissing(v *Int64ListValidation) ([]int64, error) { if v.Required { return nil, ErrorMustBeDefined() } return validateInt64List(v.Default, v) } func ValidateInt64ListProvided(val []int64, v *Int64ListValidation) ([]int64, error) { if v.CantBeSpecifiedErrStr != nil { return nil, ErrorFieldCantBeSpecified(*v.CantBeSpecifiedErrStr) } if !v.AllowExplicitNull && val == nil { return nil, ErrorCannotBeNull(v.Required) } return validateInt64List(val, v) } func validateInt64List(val []int64, v *Int64ListValidation) ([]int64, error) { if !v.AllowEmpty { if val != nil && len(val) == 0 { return nil, ErrorCannotBeEmpty() } } if v.MinLength != 0 { if len(val) < v.MinLength { return nil, ErrorTooFewElements(v.MinLength) } } if v.MaxLength != 0 { if len(val) > v.MaxLength { return nil, ErrorTooManyElements(v.MaxLength) } } for _, invalidLength := range v.InvalidLengths { if len(val) == invalidLength { return nil, ErrorWrongNumberOfElements(v.InvalidLengths) } } if v.Validator != nil { return v.Validator(val) } return val, nil } ================================================ FILE: pkg/lib/configreader/int64_ptr.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package configreader import ( "github.com/cortexlabs/cortex/pkg/lib/cast" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/files" "github.com/cortexlabs/cortex/pkg/lib/prompt" s "github.com/cortexlabs/cortex/pkg/lib/strings" ) type Int64PtrValidation struct { Required bool Default *int64 AllowExplicitNull bool AllowedValues []int64 HiddenAllowedValues []int64 // allowed, but will not be listed as valid values (must be used in conjunction with AllowedValues) DisallowedValues []int64 CantBeSpecifiedErrStr *string GreaterThan *int64 GreaterThanOrEqualTo *int64 LessThan *int64 LessThanOrEqualTo *int64 Validator func(int64) (int64, error) } func makeInt64ValValidation(v *Int64PtrValidation) *Int64Validation { return &Int64Validation{ AllowedValues: v.AllowedValues, HiddenAllowedValues: v.HiddenAllowedValues, DisallowedValues: v.DisallowedValues, GreaterThan: v.GreaterThan, GreaterThanOrEqualTo: v.GreaterThanOrEqualTo, LessThan: v.LessThan, LessThanOrEqualTo: v.LessThanOrEqualTo, } } func Int64Ptr(inter interface{}, v *Int64PtrValidation) (*int64, error) { if inter == nil { return ValidateInt64PtrProvided(nil, v) } casted, castOk := cast.InterfaceToInt64(inter) if !castOk { return nil, ErrorInvalidPrimitiveType(inter, PrimTypeInt) } return ValidateInt64PtrProvided(&casted, v) } func Int64PtrFromInterfaceMap(key string, iMap map[string]interface{}, v *Int64PtrValidation) (*int64, error) { inter, ok := ReadInterfaceMapValue(key, iMap) if !ok { val, err := ValidateInt64PtrMissing(v) if err != nil { return nil, errors.Wrap(err, key) } return val, nil } val, err := Int64Ptr(inter, v) if err != nil { return nil, errors.Wrap(err, key) } return val, nil } func Int64PtrFromStrMap(key string, sMap map[string]string, v *Int64PtrValidation) (*int64, error) { valStr, ok := sMap[key] if !ok || valStr == "" { val, err := ValidateInt64PtrMissing(v) if err != nil { return nil, errors.Wrap(err, key) } return val, nil } val, err := Int64PtrFromStr(valStr, v) if err != nil { return nil, errors.Wrap(err, key) } return val, nil } func Int64PtrFromStr(valStr string, v *Int64PtrValidation) (*int64, error) { if valStr == "" { return ValidateInt64PtrMissing(v) } casted, castOk := s.ParseInt64(valStr) if !castOk { return nil, ErrorInvalidPrimitiveType(valStr, PrimTypeInt) } return ValidateInt64PtrProvided(&casted, v) } func Int64PtrFromEnv(envVarName string, v *Int64PtrValidation) (*int64, error) { valStr := ReadEnvVar(envVarName) if valStr == nil || *valStr == "" { val, err := ValidateInt64PtrMissing(v) if err != nil { return nil, errors.Wrap(err, EnvVar(envVarName)) } return val, nil } val, err := Int64PtrFromStr(*valStr, v) if err != nil { return nil, errors.Wrap(err, EnvVar(envVarName)) } return val, nil } func Int64PtrFromFile(filePath string, v *Int64PtrValidation) (*int64, error) { if !files.IsFile(filePath) { val, err := ValidateInt64PtrMissing(v) if err != nil { return nil, errors.Wrap(err, filePath) } return val, nil } valStr, err := files.ReadFile(filePath) if err != nil { return nil, err } if len(valStr) == 0 { val, err := ValidateInt64PtrMissing(v) if err != nil { return nil, errors.Wrap(err, filePath) } return val, nil } val, err := Int64PtrFromStr(valStr, v) if err != nil { return nil, errors.Wrap(err, filePath) } return val, nil } func Int64PtrFromEnvOrFile(envVarName string, filePath string, v *Int64PtrValidation) (*int64, error) { valStr := ReadEnvVar(envVarName) if valStr != nil && *valStr != "" { return Int64PtrFromEnv(envVarName, v) } return Int64PtrFromFile(filePath, v) } func Int64PtrFromPrompt(promptOpts *prompt.Options, v *Int64PtrValidation) (*int64, error) { if v.Default != nil && promptOpts.DefaultStr == "" { promptOpts.DefaultStr = s.Int64(*v.Default) } valStr := prompt.Prompt(promptOpts) if valStr == "" { return ValidateInt64PtrMissing(v) } return Int64PtrFromStr(valStr, v) } func ValidateInt64PtrMissing(v *Int64PtrValidation) (*int64, error) { if v.Required { return nil, ErrorMustBeDefined(v.AllowedValues) } return validateInt64Ptr(v.Default, v) } func ValidateInt64PtrProvided(val *int64, v *Int64PtrValidation) (*int64, error) { if v.CantBeSpecifiedErrStr != nil { return nil, ErrorFieldCantBeSpecified(*v.CantBeSpecifiedErrStr) } if !v.AllowExplicitNull && val == nil { return nil, ErrorCannotBeNull(v.Required) } return validateInt64Ptr(val, v) } func validateInt64Ptr(val *int64, v *Int64PtrValidation) (*int64, error) { if val != nil { err := ValidateInt64Val(*val, makeInt64ValValidation(v)) if err != nil { return nil, err } } if val == nil { return val, nil } if v.Validator != nil { validated, err := v.Validator(*val) if err != nil { return nil, err } return &validated, nil } return val, nil } ================================================ FILE: pkg/lib/configreader/int_list.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package configreader import ( "github.com/cortexlabs/cortex/pkg/lib/cast" "github.com/cortexlabs/cortex/pkg/lib/errors" ) type IntListValidation struct { Required bool Default []int AllowExplicitNull bool AllowEmpty bool CantBeSpecifiedErrStr *string CastSingleItem bool MinLength int MaxLength int InvalidLengths []int Validator func([]int) ([]int, error) } func IntList(inter interface{}, v *IntListValidation) ([]int, error) { casted, castOk := cast.InterfaceToIntSlice(inter) if !castOk { if v.CastSingleItem { castedItem, castOk := cast.InterfaceToInt(inter) if !castOk { return nil, ErrorInvalidPrimitiveType(inter, PrimTypeInt, PrimTypeIntList) } casted = []int{castedItem} } else { return nil, ErrorInvalidPrimitiveType(inter, PrimTypeIntList) } } return ValidateIntListProvided(casted, v) } func IntListFromInterfaceMap(key string, iMap map[string]interface{}, v *IntListValidation) ([]int, error) { inter, ok := ReadInterfaceMapValue(key, iMap) if !ok { val, err := ValidateIntListMissing(v) if err != nil { return nil, errors.Wrap(err, key) } return val, nil } val, err := IntList(inter, v) if err != nil { return nil, errors.Wrap(err, key) } return val, nil } func ValidateIntListMissing(v *IntListValidation) ([]int, error) { if v.Required { return nil, ErrorMustBeDefined() } return validateIntList(v.Default, v) } func ValidateIntListProvided(val []int, v *IntListValidation) ([]int, error) { if v.CantBeSpecifiedErrStr != nil { return nil, ErrorFieldCantBeSpecified(*v.CantBeSpecifiedErrStr) } if !v.AllowExplicitNull && val == nil { return nil, ErrorCannotBeNull(v.Required) } return validateIntList(val, v) } func validateIntList(val []int, v *IntListValidation) ([]int, error) { if !v.AllowEmpty { if val != nil && len(val) == 0 { return nil, ErrorCannotBeEmpty() } } if v.MinLength != 0 { if len(val) < v.MinLength { return nil, ErrorTooFewElements(v.MinLength) } } if v.MaxLength != 0 { if len(val) > v.MaxLength { return nil, ErrorTooManyElements(v.MaxLength) } } for _, invalidLength := range v.InvalidLengths { if len(val) == invalidLength { return nil, ErrorWrongNumberOfElements(v.InvalidLengths) } } if v.Validator != nil { return v.Validator(val) } return val, nil } ================================================ FILE: pkg/lib/configreader/int_ptr.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package configreader import ( "github.com/cortexlabs/cortex/pkg/lib/cast" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/files" "github.com/cortexlabs/cortex/pkg/lib/prompt" s "github.com/cortexlabs/cortex/pkg/lib/strings" ) type IntPtrValidation struct { Required bool Default *int AllowExplicitNull bool AllowedValues []int HiddenAllowedValues []int // allowed, but will not be listed as valid values (must be used in conjunction with AllowedValues) DisallowedValues []int CantBeSpecifiedErrStr *string GreaterThan *int GreaterThanOrEqualTo *int LessThan *int LessThanOrEqualTo *int Validator func(int) (int, error) } func makeIntValValidation(v *IntPtrValidation) *IntValidation { return &IntValidation{ AllowedValues: v.AllowedValues, HiddenAllowedValues: v.HiddenAllowedValues, DisallowedValues: v.DisallowedValues, GreaterThan: v.GreaterThan, GreaterThanOrEqualTo: v.GreaterThanOrEqualTo, LessThan: v.LessThan, LessThanOrEqualTo: v.LessThanOrEqualTo, } } func IntPtr(inter interface{}, v *IntPtrValidation) (*int, error) { if inter == nil { return ValidateIntPtrProvided(nil, v) } casted, castOk := cast.InterfaceToInt(inter) if !castOk { return nil, ErrorInvalidPrimitiveType(inter, PrimTypeInt) } return ValidateIntPtrProvided(&casted, v) } func IntPtrFromInterfaceMap(key string, iMap map[string]interface{}, v *IntPtrValidation) (*int, error) { inter, ok := ReadInterfaceMapValue(key, iMap) if !ok { val, err := ValidateIntPtrMissing(v) if err != nil { return nil, errors.Wrap(err, key) } return val, nil } val, err := IntPtr(inter, v) if err != nil { return nil, errors.Wrap(err, key) } return val, nil } func IntPtrFromStrMap(key string, sMap map[string]string, v *IntPtrValidation) (*int, error) { valStr, ok := sMap[key] if !ok || valStr == "" { val, err := ValidateIntPtrMissing(v) if err != nil { return nil, errors.Wrap(err, key) } return val, nil } val, err := IntPtrFromStr(valStr, v) if err != nil { return nil, errors.Wrap(err, key) } return val, nil } func IntPtrFromStr(valStr string, v *IntPtrValidation) (*int, error) { if valStr == "" { return ValidateIntPtrMissing(v) } casted, castOk := s.ParseInt(valStr) if !castOk { return nil, ErrorInvalidPrimitiveType(valStr, PrimTypeInt) } return ValidateIntPtrProvided(&casted, v) } func IntPtrFromEnv(envVarName string, v *IntPtrValidation) (*int, error) { valStr := ReadEnvVar(envVarName) if valStr == nil || *valStr == "" { val, err := ValidateIntPtrMissing(v) if err != nil { return nil, errors.Wrap(err, EnvVar(envVarName)) } return val, nil } val, err := IntPtrFromStr(*valStr, v) if err != nil { return nil, errors.Wrap(err, EnvVar(envVarName)) } return val, nil } func IntPtrFromFile(filePath string, v *IntPtrValidation) (*int, error) { if !files.IsFile(filePath) { val, err := ValidateIntPtrMissing(v) if err != nil { return nil, errors.Wrap(err, filePath) } return val, nil } valStr, err := files.ReadFile(filePath) if err != nil { return nil, err } if len(valStr) == 0 { val, err := ValidateIntPtrMissing(v) if err != nil { return nil, errors.Wrap(err, filePath) } return val, nil } val, err := IntPtrFromStr(valStr, v) if err != nil { return nil, errors.Wrap(err, filePath) } return val, nil } func IntPtrFromEnvOrFile(envVarName string, filePath string, v *IntPtrValidation) (*int, error) { valStr := ReadEnvVar(envVarName) if valStr != nil && *valStr != "" { return IntPtrFromEnv(envVarName, v) } return IntPtrFromFile(filePath, v) } func IntPtrFromPrompt(promptOpts *prompt.Options, v *IntPtrValidation) (*int, error) { if v.Default != nil && promptOpts.DefaultStr == "" { promptOpts.DefaultStr = s.Int(*v.Default) } valStr := prompt.Prompt(promptOpts) if valStr == "" { return ValidateIntPtrMissing(v) } return IntPtrFromStr(valStr, v) } func ValidateIntPtrMissing(v *IntPtrValidation) (*int, error) { if v.Required { return nil, ErrorMustBeDefined(v.AllowedValues) } return validateIntPtr(v.Default, v) } func ValidateIntPtrProvided(val *int, v *IntPtrValidation) (*int, error) { if v.CantBeSpecifiedErrStr != nil { return nil, ErrorFieldCantBeSpecified(*v.CantBeSpecifiedErrStr) } if !v.AllowExplicitNull && val == nil { return nil, ErrorCannotBeNull(v.Required) } return validateIntPtr(val, v) } func validateIntPtr(val *int, v *IntPtrValidation) (*int, error) { if val != nil { err := ValidateIntVal(*val, makeIntValValidation(v)) if err != nil { return nil, err } } if val == nil { return val, nil } if v.Validator != nil { validated, err := v.Validator(*val) if err != nil { return nil, err } return &validated, nil } return val, nil } ================================================ FILE: pkg/lib/configreader/interface.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package configreader import ( "github.com/cortexlabs/cortex/pkg/lib/cast" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/maps" "github.com/cortexlabs/cortex/pkg/lib/pointer" "github.com/cortexlabs/cortex/pkg/lib/sets/strset" s "github.com/cortexlabs/cortex/pkg/lib/strings" "github.com/cortexlabs/yaml" ) type InterfaceValidation struct { Required bool Default interface{} AllowExplicitNull bool CantBeSpecifiedErrStr *string AllowCortexResources bool RequireCortexResources bool Validator func(interface{}) (interface{}, error) } func Interface(inter interface{}, v *InterfaceValidation) (interface{}, error) { return ValidateInterfaceProvided(inter, v) } func InterfaceFromInterfaceMap(key string, iMap map[string]interface{}, v *InterfaceValidation) (interface{}, error) { inter, ok := ReadInterfaceMapValue(key, iMap) if !ok { val, err := ValidateInterfaceMissing(v) if err != nil { return nil, errors.Wrap(err, key) } return val, nil } val, err := ValidateInterfaceProvided(inter, v) if err != nil { return nil, errors.Wrap(err, key) } return val, nil } func ValidateInterfaceMissing(v *InterfaceValidation) (interface{}, error) { if v.Required { return nil, ErrorMustBeDefined() } return validateInterface(v.Default, v) } func ValidateInterfaceProvided(val interface{}, v *InterfaceValidation) (interface{}, error) { if v.CantBeSpecifiedErrStr != nil { return nil, ErrorFieldCantBeSpecified(*v.CantBeSpecifiedErrStr) } if !v.AllowExplicitNull && val == nil { return nil, ErrorCannotBeNull(v.Required) } return validateInterface(val, v) } func validateInterface(val interface{}, v *InterfaceValidation) (interface{}, error) { if v.RequireCortexResources { if err := checkOnlyCortexResources(val); err != nil { return nil, err } } else if !v.AllowCortexResources { if err := checkNoCortexResources(val); err != nil { return nil, err } } if v.Validator != nil { return v.Validator(val) } return val, nil } func checkNoCortexResources(obj interface{}) error { if objStr, ok := obj.(string); ok { if resourceName, ok := yaml.ExtractAtSymbolText(objStr); ok { return ErrorCortexResourceNotAllowed(resourceName) } } if objSlice, ok := cast.InterfaceToInterfaceSlice(obj); ok { for i, objItem := range objSlice { if err := checkNoCortexResources(objItem); err != nil { return errors.Wrap(err, s.Index(i)) } } } if objMap, ok := cast.InterfaceToInterfaceInterfaceMap(obj); ok { for k, v := range objMap { if err := checkNoCortexResources(k); err != nil { return err } if err := checkNoCortexResources(v); err != nil { return errors.Wrap(err, s.UserStrStripped(k)) } } } return nil } func checkOnlyCortexResources(obj interface{}) error { if objStr, ok := obj.(string); ok { if _, ok := yaml.ExtractAtSymbolText(objStr); !ok { return ErrorCortexResourceOnlyAllowed(objStr) } } if objSlice, ok := cast.InterfaceToInterfaceSlice(obj); ok { for i, objItem := range objSlice { if err := checkOnlyCortexResources(objItem); err != nil { return errors.Wrap(err, s.Index(i)) } } } if objMap, ok := cast.InterfaceToInterfaceInterfaceMap(obj); ok { for k, v := range objMap { if err := checkOnlyCortexResources(k); err != nil { return err } if err := checkOnlyCortexResources(v); err != nil { return errors.Wrap(err, s.UserStrStripped(k)) } } } return nil } // FlattenAllStrValues assumes that the order for maps is deterministic func FlattenAllStrValues(obj interface{}) ([]string, error) { obj = pointer.IndirectSafe(obj) flattened := []string{} if objStr, ok := obj.(string); ok { return append(flattened, objStr), nil } if objSlice, ok := cast.InterfaceToInterfaceSlice(obj); ok { for i, elem := range objSlice { subFlattened, err := FlattenAllStrValues(elem) if err != nil { return nil, errors.Wrap(err, s.Index(i)) } flattened = append(flattened, subFlattened...) } return flattened, nil } if objMap, ok := cast.InterfaceToStrInterfaceMap(obj); ok { for _, key := range maps.InterfaceMapSortedKeys(objMap) { subFlattened, err := FlattenAllStrValues(objMap[key]) if err != nil { return nil, errors.Wrap(err, s.UserStrStripped(key)) } flattened = append(flattened, subFlattened...) } return flattened, nil } return nil, ErrorInvalidPrimitiveType(obj, PrimTypeString, PrimTypeList, PrimTypeMap) } func FlattenAllStrValuesAsSet(obj interface{}) (strset.Set, error) { strs, err := FlattenAllStrValues(obj) if err != nil { return nil, err } set := strset.New() for _, str := range strs { set.Add(str) } return set, nil } ================================================ FILE: pkg/lib/configreader/interface_map.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package configreader import ( "github.com/cortexlabs/cortex/pkg/lib/cast" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/slices" ) type InterfaceMapValidation struct { Required bool Default map[string]interface{} AllowExplicitNull bool AllowEmpty bool CantBeSpecifiedErrStr *string ConvertNullToEmpty bool ScalarsOnly bool StringLeavesOnly bool StringKeysOnly bool // Useful for ensuring this field is JSON parsable; validates that all maps and nested maps only use string keys AllowedLeafValues []string AllowCortexResources bool RequireCortexResources bool Validator func(map[string]interface{}) (map[string]interface{}, error) } func InterfaceMap(inter interface{}, v *InterfaceMapValidation) (map[string]interface{}, error) { casted, castOk := cast.InterfaceToStrInterfaceMap(inter) if !castOk { return nil, ErrorInvalidPrimitiveType(inter, PrimTypeMap) } return ValidateInterfaceMapProvided(casted, v) } func InterfaceMapFromInterfaceMap(key string, iMap map[string]interface{}, v *InterfaceMapValidation) (map[string]interface{}, error) { inter, ok := ReadInterfaceMapValue(key, iMap) if !ok { val, err := ValidateInterfaceMapMissing(v) if err != nil { return nil, errors.Wrap(err, key) } return val, nil } val, err := InterfaceMap(inter, v) if err != nil { return nil, errors.Wrap(err, key) } return val, nil } func ValidateInterfaceMapMissing(v *InterfaceMapValidation) (map[string]interface{}, error) { if v.Required { return nil, ErrorMustBeDefined() } return validateInterfaceMap(v.Default, v) } func ValidateInterfaceMapProvided(val map[string]interface{}, v *InterfaceMapValidation) (map[string]interface{}, error) { if v.CantBeSpecifiedErrStr != nil { return nil, ErrorFieldCantBeSpecified(*v.CantBeSpecifiedErrStr) } if !v.AllowExplicitNull && val == nil { return nil, ErrorCannotBeNull(v.Required) } return validateInterfaceMap(val, v) } func validateInterfaceMap(val map[string]interface{}, v *InterfaceMapValidation) (map[string]interface{}, error) { if v.RequireCortexResources { if err := checkOnlyCortexResources(val); err != nil { return nil, err } } else if !v.AllowCortexResources { if err := checkNoCortexResources(val); err != nil { return nil, err } } if !v.AllowEmpty { if val != nil && len(val) == 0 { return nil, ErrorCannotBeEmpty() } } if v.ScalarsOnly { for k, v := range val { if !cast.IsScalarType(v) { return nil, errors.Wrap(ErrorInvalidPrimitiveType(v, PrimTypeString, PrimTypeInt, PrimTypeFloat, PrimTypeBool), k) } } } if v.StringLeavesOnly { _, err := FlattenAllStrValues(val) if err != nil { return nil, err } } if v.AllowedLeafValues != nil { leafVals, err := FlattenAllStrValues(val) if err != nil { return nil, err } for _, leafVal := range leafVals { if !slices.HasString(v.AllowedLeafValues, leafVal) { return nil, ErrorInvalidStr(leafVal, v.AllowedLeafValues[0], v.AllowedLeafValues[1:]...) } } } if v.StringKeysOnly { for key, value := range val { m, ok := cast.InterfaceToInterfaceInterfaceMap(value) if !ok { continue } stringToIntMap := map[string]interface{}{} for kInterface, vInterface := range m { kString, ok := kInterface.(string) if !ok { return nil, errors.Wrap(ErrorNonStringKeyFound(kInterface), key) } stringToIntMap[kString] = vInterface } _, err := validateInterfaceMap(stringToIntMap, v) if err != nil { return nil, errors.Wrap(err, key) } val[key] = stringToIntMap } } if v.Validator != nil { return v.Validator(val) } if val == nil && v.ConvertNullToEmpty { val = make(map[string]interface{}) } return val, nil } ================================================ FILE: pkg/lib/configreader/interface_map_list.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package configreader import ( "github.com/cortexlabs/cortex/pkg/lib/cast" "github.com/cortexlabs/cortex/pkg/lib/errors" ) type InterfaceMapListValidation struct { Required bool Default []map[string]interface{} AllowExplicitNull bool AllowEmpty bool CantBeSpecifiedErrStr *string CastSingleItem bool MinLength int MaxLength int InvalidLengths []int AllowCortexResources bool RequireCortexResources bool Validator func([]map[string]interface{}) ([]map[string]interface{}, error) } func InterfaceMapList(inter interface{}, v *InterfaceMapListValidation) ([]map[string]interface{}, error) { casted, castOk := cast.InterfaceToStrInterfaceMapSlice(inter) if !castOk { if v.CastSingleItem { castedItem, castOk := cast.InterfaceToStrInterfaceMap(inter) if !castOk { return nil, ErrorInvalidPrimitiveType(inter, PrimTypeMap, PrimTypeMapList) } casted = []map[string]interface{}{castedItem} } else { return nil, ErrorInvalidPrimitiveType(inter, PrimTypeMapList) } } return ValidateInterfaceMapListProvided(casted, v) } func InterfaceMapListFromInterfaceMap(key string, iMap map[string]interface{}, v *InterfaceMapListValidation) ([]map[string]interface{}, error) { inter, ok := ReadInterfaceMapValue(key, iMap) if !ok { val, err := ValidateInterfaceMapListMissing(v) if err != nil { return nil, errors.Wrap(err, key) } return val, nil } val, err := InterfaceMapList(inter, v) if err != nil { return nil, errors.Wrap(err, key) } return val, nil } func ValidateInterfaceMapListMissing(v *InterfaceMapListValidation) ([]map[string]interface{}, error) { if v.Required { return nil, ErrorMustBeDefined() } return validateInterfaceMapList(v.Default, v) } func ValidateInterfaceMapListProvided(val []map[string]interface{}, v *InterfaceMapListValidation) ([]map[string]interface{}, error) { if v.CantBeSpecifiedErrStr != nil { return nil, ErrorFieldCantBeSpecified(*v.CantBeSpecifiedErrStr) } if !v.AllowExplicitNull && val == nil { return nil, ErrorCannotBeNull(v.Required) } return validateInterfaceMapList(val, v) } func validateInterfaceMapList(val []map[string]interface{}, v *InterfaceMapListValidation) ([]map[string]interface{}, error) { if v.RequireCortexResources { if err := checkOnlyCortexResources(val); err != nil { return nil, err } } else if !v.AllowCortexResources { if err := checkNoCortexResources(val); err != nil { return nil, err } } if !v.AllowEmpty { if val != nil && len(val) == 0 { return nil, ErrorCannotBeEmpty() } } if v.MinLength != 0 { if len(val) < v.MinLength { return nil, ErrorTooFewElements(v.MinLength) } } if v.MaxLength != 0 { if len(val) > v.MaxLength { return nil, ErrorTooManyElements(v.MaxLength) } } for _, invalidLength := range v.InvalidLengths { if len(val) == invalidLength { return nil, ErrorWrongNumberOfElements(v.InvalidLengths) } } if v.Validator != nil { return v.Validator(val) } return val, nil } ================================================ FILE: pkg/lib/configreader/interface_test.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package configreader import ( "testing" "github.com/stretchr/testify/require" ) func TestFlattenAllStrValues(t *testing.T) { var input interface{} var expected []string input = "test" expected = []string{"test"} CheckFlattenAllStrValues(input, expected, t) input = []interface{}{"test1", "test2", "test3"} expected = []string{"test1", "test2", "test3"} CheckFlattenAllStrValues(input, expected, t) input = map[interface{}]interface{}{ "k1": "test1", "k2": "test2", "k3": "test3", } expected = []string{"test1", "test2", "test3"} CheckFlattenAllStrValues(input, expected, t) input = map[interface{}]interface{}{ "k1": []interface{}{"test1"}, "k2": "test2", "k3": []interface{}{"test3", "test4"}, } expected = []string{"test1", "test2", "test3", "test4"} CheckFlattenAllStrValues(input, expected, t) input = map[string]interface{}{ "k1": []interface{}{"test1"}, "k2": "test2", "k3": []interface{}{"test3", "test4"}, "k4": map[string]interface{}{ "k2": "test6", "k1": []interface{}{"test5"}, "k3": []interface{}{"test7", "test8"}, }, } expected = []string{"test1", "test2", "test3", "test4", "test5", "test6", "test7", "test8"} CheckFlattenAllStrValues(input, expected, t) input = MustReadYAMLStr( ` test: key1: [test5] key2: test6 key3: [test7, test8] a2: key1: [test1] key2: test2 key3: [test3, test4] z: - key1: [test9] key2: test10 key3: [test11, test12] - key1: [test13] key2: test14 key3: [test15, test16] `) expected = []string{"test1", "test2", "test3", "test4", "test5", "test6", "test7", "test8", "test9", "test10", "test11", "test12", "test13", "test14", "test15", "test16"} CheckFlattenAllStrValues(input, expected, t) } func CheckFlattenAllStrValues(obj interface{}, expected []string, t *testing.T) { flattened, err := FlattenAllStrValues(obj) require.NoError(t, err) require.Equal(t, expected, flattened) } ================================================ FILE: pkg/lib/configreader/reader.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package configreader import ( "fmt" "os" "path/filepath" "reflect" "strings" "github.com/cortexlabs/cortex/pkg/lib/cast" "github.com/cortexlabs/cortex/pkg/lib/debug" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/exit" "github.com/cortexlabs/cortex/pkg/lib/files" "github.com/cortexlabs/cortex/pkg/lib/json" "github.com/cortexlabs/cortex/pkg/lib/maps" "github.com/cortexlabs/cortex/pkg/lib/prompt" "github.com/cortexlabs/cortex/pkg/lib/slices" s "github.com/cortexlabs/cortex/pkg/lib/strings" "github.com/cortexlabs/yaml" ) type StructFieldValidation struct { StructField string // Required (can be omitted to skip writing of value to struct) Key string // Required, or defaults to json key or "StructField" DefaultField string // Optional. Will set the default to the runtime value of this field DefaultDependentFields []string // Optional. Will be passed in to DefaultDependentFieldsFunc. Dependent fields must be listed first in the `[]*cr.StructFieldValidation`. DefaultDependentFieldsFunc func([]interface{}) interface{} // Optional. Will be called with DefaultDependentFields // Provide one of the following: StringValidation *StringValidation StringPtrValidation *StringPtrValidation StringListValidation *StringListValidation BoolValidation *BoolValidation BoolPtrValidation *BoolPtrValidation BoolListValidation *BoolListValidation IntValidation *IntValidation IntPtrValidation *IntPtrValidation IntListValidation *IntListValidation Int32Validation *Int32Validation Int32PtrValidation *Int32PtrValidation Int32ListValidation *Int32ListValidation Int64Validation *Int64Validation Int64PtrValidation *Int64PtrValidation Int64ListValidation *Int64ListValidation Float32Validation *Float32Validation Float32PtrValidation *Float32PtrValidation Float32ListValidation *Float32ListValidation Float64Validation *Float64Validation Float64PtrValidation *Float64PtrValidation Float64ListValidation *Float64ListValidation StringMapValidation *StringMapValidation InterfaceMapValidation *InterfaceMapValidation InterfaceMapListValidation *InterfaceMapListValidation InterfaceValidation *InterfaceValidation StructValidation *StructValidation StructListValidation *StructListValidation InterfaceStructValidation *InterfaceStructValidation InterfaceStructListValidation *InterfaceStructListValidation Nil bool // Additional parsing step for StringValidation or StringPtrValidation Parser func(string) (interface{}, error) } type StructValidation struct { StructFieldValidations []*StructFieldValidation Required bool AllowExplicitNull bool TreatNullAsEmpty bool // If explicit null or if it's top level and the file is empty, treat as empty map DefaultNil bool // If this struct is nested and its key is not defined, set it to nil instead of defaults or erroring (e.g. if any subfields are required) CantBeSpecifiedErrStr *string ShortCircuit bool AllowExtraFields bool } type StructListValidation struct { StructValidation *StructValidation Required bool AllowExplicitNull bool TreatNullAsEmpty bool // If explicit null or if it's top level and the file is empty, treat as empty list MinLength int MaxLength int InvalidLengths []int CantBeSpecifiedErrStr *string ShortCircuit bool } type InterfaceStructValidation struct { TypeKey string // required TypeStructField string // optional (will set this field if present) InterfaceStructTypes map[string]*InterfaceStructType // specify this or ParsedInterfaceStructTypes ParsedInterfaceStructTypes map[interface{}]*InterfaceStructType // must specify Parser if using this Parser func(string) (interface{}, error) Required bool AllowExplicitNull bool TreatNullAsEmpty bool // If explicit null or if it's top level and the file is empty, treat as empty map CantBeSpecifiedErrStr *string ShortCircuit bool AllowExtraFields bool } type InterfaceStructType struct { Type interface{} // e.g. (*MyType)(nil) StructFieldValidations []*StructFieldValidation } type InterfaceStructListValidation struct { InterfaceStructValidation *InterfaceStructValidation Required bool AllowExplicitNull bool TreatNullAsEmpty bool // If explicit null or if it's top level and the file is empty, treat as empty map CantBeSpecifiedErrStr *string ShortCircuit bool } func Struct(dest interface{}, inter interface{}, v *StructValidation) []error { allowedFields := []string{} allErrs := []error{} var ok bool if inter == nil { if v.TreatNullAsEmpty { inter = make(map[interface{}]interface{}, 0) } else { if !v.AllowExplicitNull { return []error{ErrorCannotBeEmptyOrNull(v.Required)} } return nil } } interMap, ok := cast.InterfaceToStrInterfaceMap(inter) if !ok { return []error{ErrorInvalidPrimitiveType(inter, PrimTypeMap)} } for _, structFieldValidation := range v.StructFieldValidations { key := inferKey(reflect.TypeOf(dest), structFieldValidation.StructField, structFieldValidation.Key) allowedFields = append(allowedFields, key) if structFieldValidation.Nil == true { continue } var err error var errs []error var val interface{} if structFieldValidation.StringValidation != nil { validation := *structFieldValidation.StringValidation updateValidation(&validation, dest, structFieldValidation) val, err = StringFromInterfaceMap(key, interMap, &validation) if err == nil && structFieldValidation.Parser != nil { val, err = structFieldValidation.Parser(val.(string)) err = errors.Wrap(err, key) } } else if structFieldValidation.StringPtrValidation != nil { validation := *structFieldValidation.StringPtrValidation updateValidation(&validation, dest, structFieldValidation) val, err = StringPtrFromInterfaceMap(key, interMap, &validation) if err == nil && structFieldValidation.Parser != nil { if val.(*string) == nil { val = nil } else { val, err = structFieldValidation.Parser(*val.(*string)) if err == nil && val != nil { valValue := reflect.ValueOf(val) valPtrValue := reflect.New(valValue.Type()) valPtrValue.Elem().Set(valValue) val = valPtrValue.Interface() } else { val = nil err = errors.Wrap(err, key) } } } } else if structFieldValidation.StringListValidation != nil { validation := *structFieldValidation.StringListValidation updateValidation(&validation, dest, structFieldValidation) val, err = StringListFromInterfaceMap(key, interMap, &validation) } else if structFieldValidation.BoolValidation != nil { validation := *structFieldValidation.BoolValidation updateValidation(&validation, dest, structFieldValidation) val, err = BoolFromInterfaceMap(key, interMap, &validation) } else if structFieldValidation.BoolPtrValidation != nil { validation := *structFieldValidation.BoolPtrValidation updateValidation(&validation, dest, structFieldValidation) val, err = BoolPtrFromInterfaceMap(key, interMap, &validation) } else if structFieldValidation.BoolListValidation != nil { validation := *structFieldValidation.BoolListValidation updateValidation(&validation, dest, structFieldValidation) val, err = BoolListFromInterfaceMap(key, interMap, &validation) } else if structFieldValidation.IntValidation != nil { validation := *structFieldValidation.IntValidation updateValidation(&validation, dest, structFieldValidation) val, err = IntFromInterfaceMap(key, interMap, &validation) } else if structFieldValidation.IntPtrValidation != nil { validation := *structFieldValidation.IntPtrValidation updateValidation(&validation, dest, structFieldValidation) val, err = IntPtrFromInterfaceMap(key, interMap, &validation) } else if structFieldValidation.IntListValidation != nil { validation := *structFieldValidation.IntListValidation updateValidation(&validation, dest, structFieldValidation) val, err = IntListFromInterfaceMap(key, interMap, &validation) } else if structFieldValidation.Int32Validation != nil { validation := *structFieldValidation.Int32Validation updateValidation(&validation, dest, structFieldValidation) val, err = Int32FromInterfaceMap(key, interMap, &validation) } else if structFieldValidation.Int32PtrValidation != nil { validation := *structFieldValidation.Int32PtrValidation updateValidation(&validation, dest, structFieldValidation) val, err = Int32PtrFromInterfaceMap(key, interMap, &validation) } else if structFieldValidation.Int32ListValidation != nil { validation := *structFieldValidation.Int32ListValidation updateValidation(&validation, dest, structFieldValidation) val, err = Int32ListFromInterfaceMap(key, interMap, &validation) } else if structFieldValidation.Int64Validation != nil { validation := *structFieldValidation.Int64Validation updateValidation(&validation, dest, structFieldValidation) val, err = Int64FromInterfaceMap(key, interMap, &validation) } else if structFieldValidation.Int64PtrValidation != nil { validation := *structFieldValidation.Int64PtrValidation updateValidation(&validation, dest, structFieldValidation) val, err = Int64PtrFromInterfaceMap(key, interMap, &validation) } else if structFieldValidation.Int64ListValidation != nil { validation := *structFieldValidation.Int64ListValidation updateValidation(&validation, dest, structFieldValidation) val, err = Int64ListFromInterfaceMap(key, interMap, &validation) } else if structFieldValidation.Float32Validation != nil { validation := *structFieldValidation.Float32Validation updateValidation(&validation, dest, structFieldValidation) val, err = Float32FromInterfaceMap(key, interMap, &validation) } else if structFieldValidation.Float32PtrValidation != nil { validation := *structFieldValidation.Float32PtrValidation updateValidation(&validation, dest, structFieldValidation) val, err = Float32PtrFromInterfaceMap(key, interMap, &validation) } else if structFieldValidation.Float64Validation != nil { validation := *structFieldValidation.Float64Validation updateValidation(&validation, dest, structFieldValidation) val, err = Float64FromInterfaceMap(key, interMap, &validation) } else if structFieldValidation.Float64PtrValidation != nil { validation := *structFieldValidation.Float64PtrValidation updateValidation(&validation, dest, structFieldValidation) val, err = Float64PtrFromInterfaceMap(key, interMap, &validation) } else if structFieldValidation.Float32ListValidation != nil { validation := *structFieldValidation.Float32ListValidation updateValidation(&validation, dest, structFieldValidation) val, err = Float32ListFromInterfaceMap(key, interMap, &validation) } else if structFieldValidation.Float64ListValidation != nil { validation := *structFieldValidation.Float64ListValidation updateValidation(&validation, dest, structFieldValidation) val, err = Float64ListFromInterfaceMap(key, interMap, &validation) } else if structFieldValidation.StringMapValidation != nil { validation := *structFieldValidation.StringMapValidation updateValidation(&validation, dest, structFieldValidation) val, err = StringMapFromInterfaceMap(key, interMap, &validation) } else if structFieldValidation.InterfaceMapValidation != nil { validation := *structFieldValidation.InterfaceMapValidation updateValidation(&validation, dest, structFieldValidation) val, err = InterfaceMapFromInterfaceMap(key, interMap, &validation) } else if structFieldValidation.InterfaceMapListValidation != nil { validation := *structFieldValidation.InterfaceMapListValidation updateValidation(&validation, dest, structFieldValidation) val, err = InterfaceMapListFromInterfaceMap(key, interMap, &validation) } else if structFieldValidation.InterfaceValidation != nil { validation := *structFieldValidation.InterfaceValidation updateValidation(&validation, dest, structFieldValidation) val, err = InterfaceFromInterfaceMap(key, interMap, &validation) } else if structFieldValidation.StructValidation != nil { validation := *structFieldValidation.StructValidation updateValidation(&validation, dest, structFieldValidation) nestedType := reflect.ValueOf(dest).Elem().FieldByName(structFieldValidation.StructField).Type() interMapVal, ok := ReadInterfaceMapValue(key, interMap) if ok && validation.CantBeSpecifiedErrStr != nil { err = errors.Wrap(ErrorFieldCantBeSpecified(*validation.CantBeSpecifiedErrStr), key) } else if !ok && validation.Required { err = errors.Wrap(ErrorMustBeDefined(), key) } else if !ok && validation.DefaultNil { val = nil } else { if !ok { interMapVal = make(map[string]interface{}) // Here validation.DefaultNil == false, so create an empty map to hold the nested default values } val = reflect.New(nestedType.Elem()).Interface() errs = Struct(val, interMapVal, &validation) if interMapVal == nil { val = nil // If the object was nil, set val to nil rather than a pointer to the initialized zero value } errs = errors.WrapAll(errs, key) } } else if structFieldValidation.StructListValidation != nil { validation := *structFieldValidation.StructListValidation updateValidation(&validation, dest, structFieldValidation) nestedType := reflect.ValueOf(dest).Elem().FieldByName(structFieldValidation.StructField).Type() interMapVal, ok := ReadInterfaceMapValue(key, interMap) if ok && validation.CantBeSpecifiedErrStr != nil { err = errors.Wrap(ErrorFieldCantBeSpecified(*validation.CantBeSpecifiedErrStr), key) } else if !ok && validation.Required { err = errors.Wrap(ErrorMustBeDefined(), key) } else { val = reflect.Indirect(reflect.New(nestedType)).Interface() val, errs = StructList(val, interMapVal, &validation) errs = errors.WrapAll(errs, key) } } else if structFieldValidation.InterfaceStructValidation != nil { validation := *structFieldValidation.InterfaceStructValidation updateValidation(&validation, dest, structFieldValidation) interMapVal, ok := ReadInterfaceMapValue(key, interMap) if ok && validation.CantBeSpecifiedErrStr != nil { err = errors.Wrap(ErrorFieldCantBeSpecified(*validation.CantBeSpecifiedErrStr), key) } else if !ok && validation.Required { err = errors.Wrap(ErrorMustBeDefined(), key) } else { val, errs = InterfaceStruct(interMapVal, &validation) errs = errors.WrapAll(errs, key) } } else if structFieldValidation.InterfaceStructListValidation != nil { validation := *structFieldValidation.InterfaceStructListValidation updateValidation(&validation, dest, structFieldValidation) nestedType := reflect.ValueOf(dest).Elem().FieldByName(structFieldValidation.StructField).Type() interMapVal, ok := ReadInterfaceMapValue(key, interMap) if ok && validation.CantBeSpecifiedErrStr != nil { err = errors.Wrap(ErrorFieldCantBeSpecified(*validation.CantBeSpecifiedErrStr), key) } else if !ok && validation.Required { err = errors.Wrap(ErrorMustBeDefined(), key) } else { val = reflect.Indirect(reflect.New(nestedType)).Interface() val, errs = InterfaceStructList(val, interMapVal, &validation) errs = errors.WrapAll(errs, key) } } else { exit.Panic(ErrorUnsupportedFieldValidation()) } allErrs, _ = errors.AddError(allErrs, err) allErrs, _ = errors.AddErrors(allErrs, errs) if errors.HasError(allErrs) { if v.ShortCircuit { return allErrs } continue } if structFieldValidation.StructField != "" { if val == nil { err = setFieldNil(dest, structFieldValidation.StructField) } else { err = setField(val, dest, structFieldValidation.StructField) } if allErrs, ok = errors.AddError(allErrs, err, key); ok { if v.ShortCircuit { return allErrs } } } } if !v.AllowExtraFields { extraFields := slices.SubtractStrSlice(maps.InterfaceMapKeys(interMap), allowedFields) for _, extraField := range extraFields { allErrs = append(allErrs, ErrorUnsupportedKey(extraField)) } } if errors.HasError(allErrs) { return allErrs } return nil } func StructList(dest interface{}, inter interface{}, v *StructListValidation) (interface{}, []error) { if inter == nil { if v.TreatNullAsEmpty { inter = make([]interface{}, 0) } else { if !v.AllowExplicitNull { return nil, []error{ErrorCannotBeEmptyOrNull(v.Required)} } return nil, nil } } interSlice, ok := cast.InterfaceToInterfaceSlice(inter) if !ok { return nil, []error{ErrorInvalidPrimitiveType(inter, PrimTypeList)} } if v.MinLength != 0 { if len(interSlice) < v.MinLength { return nil, []error{ErrorTooFewElements(v.MinLength)} } } if v.MaxLength != 0 { if len(interSlice) > v.MaxLength { return nil, []error{ErrorTooManyElements(v.MaxLength)} } } for _, invalidLength := range v.InvalidLengths { if len(interSlice) == invalidLength { return nil, []error{ErrorWrongNumberOfElements(v.InvalidLengths)} } } errs := []error{} for i, interItem := range interSlice { val := reflect.New(reflect.ValueOf(dest).Type().Elem().Elem()).Interface() subErrs := Struct(val, interItem, v.StructValidation) var ok bool if errs, ok = errors.AddErrors(errs, subErrs, s.Index(i)); ok { if v.ShortCircuit { return nil, errs } continue } if interItem == nil { val = nil // If the object was nil, set val to nil rather than a pointer to the initialized zero value } dest = appendVal(dest, val) } return dest, errs } func InterfaceStruct(inter interface{}, v *InterfaceStructValidation) (interface{}, []error) { if inter == nil { if v.TreatNullAsEmpty { inter = make(map[interface{}]interface{}, 0) } else { if !v.AllowExplicitNull { return nil, []error{ErrorCannotBeEmptyOrNull(v.Required)} } return nil, nil } } interMap, ok := cast.InterfaceToStrInterfaceMap(inter) if !ok { return nil, []error{ErrorInvalidPrimitiveType(inter, PrimTypeMap)} } var validTypeStrs []string if v.InterfaceStructTypes != nil { for typeStr := range v.InterfaceStructTypes { validTypeStrs = append(validTypeStrs, typeStr) } } typeStrValidation := &StringValidation{ Required: true, AllowedValues: validTypeStrs, } typeStr, err := StringFromInterfaceMap(v.TypeKey, interMap, typeStrValidation) if err != nil { return nil, []error{err} } var typeObj interface{} if v.Parser != nil { typeObj, err = v.Parser(typeStr) if err != nil { return nil, []error{errors.Wrap(err, v.TypeKey)} } } var typeFieldValidation *StructFieldValidation if v.TypeStructField == "" { typeFieldValidation = &StructFieldValidation{ Key: v.TypeKey, Nil: true, } } else { typeFieldValidation = &StructFieldValidation{ Key: v.TypeKey, StructField: v.TypeStructField, StringValidation: typeStrValidation, Parser: v.Parser, } } var structType *InterfaceStructType if v.InterfaceStructTypes != nil { structType = v.InterfaceStructTypes[typeStr] } else { structType = v.ParsedInterfaceStructTypes[typeObj] if structType == nil { // This error case may or may not be handled by v.Parser() var validTypeObjs []interface{} for typeObj := range v.ParsedInterfaceStructTypes { validTypeObjs = append(validTypeObjs, typeObj) } return nil, []error{errors.Wrap(ErrorInvalidInterface(typeStr, validTypeObjs[0], validTypeObjs[1:]...), v.TypeKey)} } } val := reflect.New(reflect.TypeOf(structType.Type).Elem()).Interface() structValidation := &StructValidation{ StructFieldValidations: append(structType.StructFieldValidations, typeFieldValidation), Required: v.Required, AllowExplicitNull: v.AllowExplicitNull, ShortCircuit: v.ShortCircuit, AllowExtraFields: v.AllowExtraFields, } errs := Struct(val, inter, structValidation) return val, errs } func InterfaceStructList(dest interface{}, inter interface{}, v *InterfaceStructListValidation) (interface{}, []error) { if inter == nil { if v.TreatNullAsEmpty { inter = make([]interface{}, 0) } else { if !v.AllowExplicitNull { return nil, []error{ErrorCannotBeEmptyOrNull(v.Required)} } return nil, nil } } interSlice, ok := cast.InterfaceToInterfaceSlice(inter) if !ok { return nil, []error{ErrorInvalidPrimitiveType(inter, PrimTypeList)} } errs := []error{} for i, interItem := range interSlice { val, subErrs := InterfaceStruct(interItem, v.InterfaceStructValidation) var ok bool if errs, ok = errors.AddErrors(errs, subErrs, s.Index(i)); ok { if v.ShortCircuit { return nil, errs } continue } dest = appendVal(dest, val) } return dest, errs } func updateValidation(validation interface{}, dest interface{}, structFieldValidation *StructFieldValidation) { if structFieldValidation.DefaultField != "" { runtimeVal := reflect.ValueOf(dest).Elem().FieldByName(structFieldValidation.DefaultField).Interface() setField(runtimeVal, validation, "Default") } else if structFieldValidation.DefaultDependentFieldsFunc != nil { runtimeVals := make([]interface{}, len(structFieldValidation.DefaultDependentFields)) for i, fieldName := range structFieldValidation.DefaultDependentFields { runtimeVals[i] = reflect.ValueOf(dest).Elem().FieldByName(fieldName).Interface() } val := structFieldValidation.DefaultDependentFieldsFunc(runtimeVals) setField(val, validation, "Default") } } func ReadInterfaceMapValue(name string, interMap map[string]interface{}) (interface{}, bool) { if interMap == nil { return nil, false } val, ok := interMap[name] if !ok { return nil, false } return val, true } // // Prompt // type PromptItemValidation struct { StructField string // Required PromptOpts *prompt.Options // Required // Provide one of the following: StringValidation *StringValidation StringPtrValidation *StringPtrValidation BoolValidation *BoolValidation BoolPtrValidation *BoolPtrValidation IntValidation *IntValidation IntPtrValidation *IntPtrValidation Int32Validation *Int32Validation Int32PtrValidation *Int32PtrValidation Int64Validation *Int64Validation Int64PtrValidation *Int64PtrValidation Float32Validation *Float32Validation Float32PtrValidation *Float32PtrValidation Float64Validation *Float64Validation Float64PtrValidation *Float64PtrValidation // Additional parsing step for StringValidation or StringPtrValidation Parser func(string) (interface{}, error) } type PromptValidation struct { PromptItemValidations []*PromptItemValidation SkipNonEmptyFields bool // skips fields that are not zero-valued SkipNonNilFields bool // skips pointer fields that are not nil PrintNewLineIfPrompted bool // prints an extra new line at the end if any questions were asked } func ReadPrompt(dest interface{}, promptValidation *PromptValidation) error { var val interface{} var err error shouldPrintTrailingNewLine := false // Validate any skipped fields first, so that any errors are returned before prompting if promptValidation.SkipNonEmptyFields { for _, promptItemValidation := range promptValidation.PromptItemValidations { v := reflect.ValueOf(dest).Elem().FieldByName(promptItemValidation.StructField) if !v.IsZero() { if promptItemValidation.StringValidation != nil && promptItemValidation.Parser == nil { if _, err := ValidateStringProvided(v.Interface().(string), promptItemValidation.StringValidation); err != nil { return errors.Wrap(err, inferPromptFieldName(reflect.TypeOf(dest), promptItemValidation.StructField)) } } else if promptItemValidation.StringPtrValidation != nil && promptItemValidation.Parser == nil { if _, err := ValidateStringPtrProvided(v.Interface().(*string), promptItemValidation.StringPtrValidation); err != nil { return errors.Wrap(err, inferPromptFieldName(reflect.TypeOf(dest), promptItemValidation.StructField)) } } else if promptItemValidation.BoolValidation != nil { if _, err := ValidateBoolProvided(v.Interface().(bool), promptItemValidation.BoolValidation); err != nil { return errors.Wrap(err, inferPromptFieldName(reflect.TypeOf(dest), promptItemValidation.StructField)) } } else if promptItemValidation.BoolPtrValidation != nil { if _, err := ValidateBoolPtrProvided(v.Interface().(*bool), promptItemValidation.BoolPtrValidation); err != nil { return errors.Wrap(err, inferPromptFieldName(reflect.TypeOf(dest), promptItemValidation.StructField)) } } else if promptItemValidation.IntValidation != nil { if _, err := ValidateIntProvided(v.Interface().(int), promptItemValidation.IntValidation); err != nil { return errors.Wrap(err, inferPromptFieldName(reflect.TypeOf(dest), promptItemValidation.StructField)) } } else if promptItemValidation.IntPtrValidation != nil { if _, err := ValidateIntPtrProvided(v.Interface().(*int), promptItemValidation.IntPtrValidation); err != nil { return errors.Wrap(err, inferPromptFieldName(reflect.TypeOf(dest), promptItemValidation.StructField)) } } else if promptItemValidation.Int32Validation != nil { if _, err := ValidateInt32Provided(v.Interface().(int32), promptItemValidation.Int32Validation); err != nil { return errors.Wrap(err, inferPromptFieldName(reflect.TypeOf(dest), promptItemValidation.StructField)) } } else if promptItemValidation.Int32PtrValidation != nil { if _, err := ValidateInt32PtrProvided(v.Interface().(*int32), promptItemValidation.Int32PtrValidation); err != nil { return errors.Wrap(err, inferPromptFieldName(reflect.TypeOf(dest), promptItemValidation.StructField)) } } else if promptItemValidation.Int64Validation != nil { if _, err := ValidateInt64Provided(v.Interface().(int64), promptItemValidation.Int64Validation); err != nil { return errors.Wrap(err, inferPromptFieldName(reflect.TypeOf(dest), promptItemValidation.StructField)) } } else if promptItemValidation.Int64PtrValidation != nil { if _, err := ValidateInt64PtrProvided(v.Interface().(*int64), promptItemValidation.Int64PtrValidation); err != nil { return errors.Wrap(err, inferPromptFieldName(reflect.TypeOf(dest), promptItemValidation.StructField)) } } else if promptItemValidation.Float32Validation != nil { if _, err := ValidateFloat32Provided(v.Interface().(float32), promptItemValidation.Float32Validation); err != nil { return errors.Wrap(err, inferPromptFieldName(reflect.TypeOf(dest), promptItemValidation.StructField)) } } else if promptItemValidation.Float32PtrValidation != nil { if _, err := ValidateFloat32PtrProvided(v.Interface().(*float32), promptItemValidation.Float32PtrValidation); err != nil { return errors.Wrap(err, inferPromptFieldName(reflect.TypeOf(dest), promptItemValidation.StructField)) } } else if promptItemValidation.Float64Validation != nil { if _, err := ValidateFloat64Provided(v.Interface().(float64), promptItemValidation.Float64Validation); err != nil { return errors.Wrap(err, inferPromptFieldName(reflect.TypeOf(dest), promptItemValidation.StructField)) } } else if promptItemValidation.Float64PtrValidation != nil { if _, err := ValidateFloat64PtrProvided(v.Interface().(*float64), promptItemValidation.Float64PtrValidation); err != nil { return errors.Wrap(err, inferPromptFieldName(reflect.TypeOf(dest), promptItemValidation.StructField)) } } } } } for _, promptItemValidation := range promptValidation.PromptItemValidations { if promptValidation.SkipNonEmptyFields { v := reflect.ValueOf(dest).Elem().FieldByName(promptItemValidation.StructField) if !v.IsZero() { continue } } else if promptValidation.SkipNonNilFields { v := reflect.ValueOf(dest).Elem().FieldByName(promptItemValidation.StructField) if v.Kind() == reflect.Ptr && !v.IsNil() { continue } } if promptValidation.PrintNewLineIfPrompted { shouldPrintTrailingNewLine = true } for { if promptItemValidation.StringValidation != nil { val, err = StringFromPrompt(promptItemValidation.PromptOpts, promptItemValidation.StringValidation) if err == nil && promptItemValidation.Parser != nil { val, err = promptItemValidation.Parser(val.(string)) } } else if promptItemValidation.StringPtrValidation != nil { val, err = StringPtrFromPrompt(promptItemValidation.PromptOpts, promptItemValidation.StringPtrValidation) if err == nil && promptItemValidation.Parser != nil { if val.(*string) == nil { val = nil } else { val, err = promptItemValidation.Parser(*val.(*string)) if err == nil && val != nil { valValue := reflect.ValueOf(val) valPtrValue := reflect.New(valValue.Type()) valPtrValue.Elem().Set(valValue) val = valPtrValue.Interface() } else { val = nil // err is already set } } } } else if promptItemValidation.BoolValidation != nil { val, err = BoolFromPrompt(promptItemValidation.PromptOpts, promptItemValidation.BoolValidation) } else if promptItemValidation.BoolPtrValidation != nil { val, err = BoolPtrFromPrompt(promptItemValidation.PromptOpts, promptItemValidation.BoolPtrValidation) } else if promptItemValidation.IntValidation != nil { val, err = IntFromPrompt(promptItemValidation.PromptOpts, promptItemValidation.IntValidation) } else if promptItemValidation.IntPtrValidation != nil { val, err = IntPtrFromPrompt(promptItemValidation.PromptOpts, promptItemValidation.IntPtrValidation) } else if promptItemValidation.Int32Validation != nil { val, err = Int32FromPrompt(promptItemValidation.PromptOpts, promptItemValidation.Int32Validation) } else if promptItemValidation.Int32PtrValidation != nil { val, err = Int32PtrFromPrompt(promptItemValidation.PromptOpts, promptItemValidation.Int32PtrValidation) } else if promptItemValidation.Int64Validation != nil { val, err = Int64FromPrompt(promptItemValidation.PromptOpts, promptItemValidation.Int64Validation) } else if promptItemValidation.Int64PtrValidation != nil { val, err = Int64PtrFromPrompt(promptItemValidation.PromptOpts, promptItemValidation.Int64PtrValidation) } else if promptItemValidation.Float32Validation != nil { val, err = Float32FromPrompt(promptItemValidation.PromptOpts, promptItemValidation.Float32Validation) } else if promptItemValidation.Float32PtrValidation != nil { val, err = Float32PtrFromPrompt(promptItemValidation.PromptOpts, promptItemValidation.Float32PtrValidation) } else if promptItemValidation.Float64Validation != nil { val, err = Float64FromPrompt(promptItemValidation.PromptOpts, promptItemValidation.Float64Validation) } else if promptItemValidation.Float64PtrValidation != nil { val, err = Float64PtrFromPrompt(promptItemValidation.PromptOpts, promptItemValidation.Float64PtrValidation) } else { exit.Panic(ErrorUnsupportedFieldValidation()) } if err == nil { break } if promptItemValidation.PromptOpts.SkipTrailingNewline { fmt.Printf("error: %s\n", errors.Message(err)) } else { fmt.Printf("error: %s\n\n", errors.Message(err)) } } if val == nil { err = setFieldNil(dest, promptItemValidation.StructField) } else { err = setField(val, dest, promptItemValidation.StructField) } if err != nil { return err } } if shouldPrintTrailingNewLine { fmt.Println() } return nil } // Reads a string map into a struct func StructFromStringMap(dest interface{}, strMap map[string]string, v *StructValidation) []error { allowedFields := []string{} allErrs := []error{} var ok bool if strMap == nil { if v.TreatNullAsEmpty { strMap = make(map[string]string, 0) } else { if !v.AllowExplicitNull { return []error{ErrorCannotBeEmptyOrNull(v.Required)} } return nil } } for _, structFieldValidation := range v.StructFieldValidations { key := inferKey(reflect.TypeOf(dest), structFieldValidation.StructField, structFieldValidation.Key) allowedFields = append(allowedFields, key) if structFieldValidation.Nil == true { continue } strMapVal, keyExists := strMap[key] var err error var errs []error var val interface{} if structFieldValidation.StringValidation != nil { validation := *structFieldValidation.StringValidation updateValidation(&validation, dest, structFieldValidation) if keyExists { val, err = StringFromStr(strMapVal, &validation) } else { val, err = ValidateStringMissing(&validation) } if err == nil && structFieldValidation.Parser != nil { val, err = structFieldValidation.Parser(val.(string)) } } else if structFieldValidation.StringPtrValidation != nil { validation := *structFieldValidation.StringPtrValidation updateValidation(&validation, dest, structFieldValidation) if keyExists { val, err = StringPtrFromStr(strMapVal, &validation) } else { val, err = ValidateStringPtrMissing(&validation) } if err == nil && structFieldValidation.Parser != nil { if val.(*string) == nil { val = nil } else { val, err = structFieldValidation.Parser(*val.(*string)) if err == nil && val != nil { valValue := reflect.ValueOf(val) valPtrValue := reflect.New(valValue.Type()) valPtrValue.Elem().Set(valValue) val = valPtrValue.Interface() } else { val = nil } } } } else if structFieldValidation.BoolValidation != nil { validation := *structFieldValidation.BoolValidation updateValidation(&validation, dest, structFieldValidation) if keyExists { val, err = BoolFromStr(strMapVal, &validation) } else { val, err = ValidateBoolMissing(&validation) } } else if structFieldValidation.BoolPtrValidation != nil { validation := *structFieldValidation.BoolPtrValidation updateValidation(&validation, dest, structFieldValidation) if keyExists { val, err = BoolPtrFromStr(strMapVal, &validation) } else { val, err = ValidateBoolPtrMissing(&validation) } } else if structFieldValidation.IntValidation != nil { validation := *structFieldValidation.IntValidation updateValidation(&validation, dest, structFieldValidation) if keyExists { val, err = IntFromStr(strMapVal, &validation) } else { val, err = ValidateIntMissing(&validation) } } else if structFieldValidation.IntPtrValidation != nil { validation := *structFieldValidation.IntPtrValidation updateValidation(&validation, dest, structFieldValidation) if keyExists { val, err = IntPtrFromStr(strMapVal, &validation) } else { val, err = ValidateIntPtrMissing(&validation) } } else if structFieldValidation.Int32Validation != nil { validation := *structFieldValidation.Int32Validation updateValidation(&validation, dest, structFieldValidation) if keyExists { val, err = Int32FromStr(strMapVal, &validation) } else { val, err = ValidateInt32Missing(&validation) } } else if structFieldValidation.Int32PtrValidation != nil { validation := *structFieldValidation.Int32PtrValidation updateValidation(&validation, dest, structFieldValidation) if keyExists { val, err = Int32PtrFromStr(strMapVal, &validation) } else { val, err = ValidateInt32PtrMissing(&validation) } } else if structFieldValidation.Int64Validation != nil { validation := *structFieldValidation.Int64Validation updateValidation(&validation, dest, structFieldValidation) if keyExists { val, err = Int64FromStr(strMapVal, &validation) } else { val, err = ValidateInt64Missing(&validation) } } else if structFieldValidation.Int64PtrValidation != nil { validation := *structFieldValidation.Int64PtrValidation updateValidation(&validation, dest, structFieldValidation) if keyExists { val, err = Int64PtrFromStr(strMapVal, &validation) } else { val, err = ValidateInt64PtrMissing(&validation) } } else if structFieldValidation.Float32Validation != nil { validation := *structFieldValidation.Float32Validation updateValidation(&validation, dest, structFieldValidation) if keyExists { val, err = Float32FromStr(strMapVal, &validation) } else { val, err = ValidateFloat32Missing(&validation) } } else if structFieldValidation.Float32PtrValidation != nil { validation := *structFieldValidation.Float32PtrValidation updateValidation(&validation, dest, structFieldValidation) if keyExists { val, err = Float32PtrFromStr(strMapVal, &validation) } else { val, err = ValidateFloat32PtrMissing(&validation) } } else if structFieldValidation.Float64Validation != nil { validation := *structFieldValidation.Float64Validation updateValidation(&validation, dest, structFieldValidation) if keyExists { val, err = Float64FromStr(strMapVal, &validation) } else { val, err = ValidateFloat64Missing(&validation) } } else if structFieldValidation.Float64PtrValidation != nil { validation := *structFieldValidation.Float64PtrValidation updateValidation(&validation, dest, structFieldValidation) if keyExists { val, err = Float64PtrFromStr(strMapVal, &validation) } else { val, err = ValidateFloat64PtrMissing(&validation) } } else { exit.Panic(ErrorUnsupportedFieldValidation()) } err = errors.Wrap(err, key) errs = errors.WrapAll(errs, key) allErrs, _ = errors.AddError(allErrs, err) allErrs, _ = errors.AddErrors(allErrs, errs) if errors.HasError(allErrs) { if v.ShortCircuit { return allErrs } continue } if val == nil { err = setFieldNil(dest, structFieldValidation.StructField) } else { err = setField(val, dest, structFieldValidation.StructField) } if allErrs, ok = errors.AddError(allErrs, err, key); ok { if v.ShortCircuit { return allErrs } } } if !v.AllowExtraFields { extraFields := slices.SubtractStrSlice(maps.StrMapKeysString(strMap), allowedFields) for _, extraField := range extraFields { allErrs = append(allErrs, ErrorUnsupportedKey(extraField)) } } if errors.HasError(allErrs) { return allErrs } return nil } // Reads a directory of files into a struct, where each file name is the key and the contents is the value func StructFromFiles(dest interface{}, dirPath string, v *StructValidation) []error { strMap := map[string]string{} fileNames, err := files.ListDir(dirPath, true) if err != nil { return []error{err} } for _, fileName := range fileNames { fileBytes, err := files.ReadFileBytes(filepath.Join(dirPath, fileName)) if err != nil { return []error{err} } strMap[fileName] = strings.TrimSpace(string(fileBytes)) } return StructFromStringMap(dest, strMap, v) } // // Environment variable // func ReadEnvVar(envVarName string) *string { envVar, envVarIsSet := os.LookupEnv(envVarName) if envVarIsSet { return &envVar } return nil } // // JSON and YAML Config // func ParseYAMLFile(dest interface{}, validation *StructValidation, filePath string) []error { fileInterface, err := ReadYAMLFile(filePath) if err != nil { return []error{err} } errs := Struct(dest, fileInterface, validation) if errors.HasError(errs) { return errors.WrapAll(errs, filePath) } return nil } func ParseYAMLBytes(dest interface{}, validation *StructValidation, data []byte) error { fileInterface, err := ReadYAMLBytes(data) if err != nil { return err } errs := Struct(dest, fileInterface, validation) if errors.HasError(errs) { return errors.FirstError(errs...) } return nil } func ReadYAMLFile(filePath string) (interface{}, error) { fileBytes, err := files.ReadFileBytes(filePath) if err != nil { return nil, err } fileInterface, err := ReadYAMLBytes(fileBytes) if err != nil { return nil, errors.Wrap(err, filePath) } return fileInterface, nil } func ReadYAMLFileStrMap(filePath string) (map[string]interface{}, error) { parsed, err := ReadYAMLFile(filePath) if err != nil { return nil, err } casted, ok := cast.InterfaceToStrInterfaceMap(parsed) if !ok { return nil, ErrorInvalidPrimitiveType(parsed, PrimTypeMap) } return casted, nil } func ReadYAMLBytes(yamlBytes []byte) (interface{}, error) { if len(yamlBytes) == 0 { return nil, nil } var parsed interface{} err := yaml.Unmarshal(yamlBytes, &parsed) if err != nil { return nil, ErrorInvalidYAML(err) } return parsed, nil } func ReadJSONBytes(jsonBytes []byte) (interface{}, error) { if len(jsonBytes) == 0 { return nil, nil } var parsed interface{} err := json.DecodeWithNumber(jsonBytes, &parsed) if err != nil { return nil, err } return parsed, nil } func MustReadYAMLStr(yamlStr string) interface{} { parsed, err := ReadYAMLBytes([]byte(yamlStr)) if err != nil { exit.Panic(err) } return parsed } func MustReadYAMLStrMap(yamlStr string) map[string]interface{} { parsed, err := ReadYAMLBytes([]byte(yamlStr)) if err != nil { exit.Panic(err) } casted, ok := cast.InterfaceToStrInterfaceMap(parsed) if !ok { exit.Panic(ErrorInvalidPrimitiveType(parsed, PrimTypeMap)) } return casted } func MustReadJSONStr(jsonStr string) interface{} { parsed, err := ReadJSONBytes([]byte(jsonStr)) if err != nil { exit.Panic(err) } return parsed } // // Helpers // func appendVal(slice interface{}, val interface{}) interface{} { return reflect.Append(reflect.ValueOf(slice), reflect.ValueOf(val)).Interface() } // destStruct must be a pointer to a struct func setField(val interface{}, destStruct interface{}, fieldName string) error { v := reflect.ValueOf(destStruct).Elem().FieldByName(fieldName) if !v.IsValid() || !v.CanSet() { debug.Ppg(val) debug.Ppg(destStruct) return errors.Wrap(ErrorCannotSetStructField(), fieldName) } if val == nil { // Check for nil-able types if v.Kind() == reflect.Chan || v.Kind() == reflect.Func || v.Kind() == reflect.Interface || v.Kind() == reflect.Map || v.Kind() == reflect.Ptr || v.Kind() == reflect.Slice { v.Set(reflect.Zero(v.Type())) return nil } debug.Ppg(val) debug.Ppg(destStruct) return errors.Wrap(ErrorCannotSetStructField(), fieldName) } if !reflect.ValueOf(val).Type().AssignableTo(v.Type()) { debug.Ppg(val) debug.Ppg(destStruct) return errors.Wrap(ErrorCannotSetStructField(), fieldName) } v.Set(reflect.ValueOf(val)) return nil } // destStruct must be a pointer to a struct func setFirstField(val interface{}, destStruct interface{}) error { v := reflect.ValueOf(destStruct).Elem().FieldByIndex([]int{0}) if !v.IsValid() || !v.CanSet() { debug.Ppg(val) debug.Ppg(destStruct) return errors.Wrap(ErrorCannotSetStructField(), "first field") } v.Set(reflect.ValueOf(val)) return nil } // destStruct must be a pointer to a struct func setFieldNil(destStruct interface{}, fieldName string) error { v := reflect.ValueOf(destStruct).Elem().FieldByName(fieldName) if !v.IsValid() || !v.CanSet() { debug.Ppg(destStruct) return errors.Wrap(ErrorCannotSetStructField(), fieldName) } v.Set(reflect.Zero(v.Type())) return nil } // destStruct must be a pointer to a struct func setFieldIfExists(val interface{}, destStruct interface{}, fieldName string) bool { if structHasKey(destStruct, fieldName) { err := setField(val, destStruct, fieldName) return err == nil } return false } // structVal must be a pointer to a struct func structHasKey(val interface{}, fieldName string) bool { v := reflect.ValueOf(val).Elem().FieldByName(fieldName) if v.IsValid() && v.CanSet() { return true } return false } func inferKey(structType reflect.Type, typeStructField string, typeKey string) string { if typeKey != "" { return typeKey } field, _ := structType.Elem().FieldByName(typeStructField) tag, ok := getTagFieldName(field) if ok { return tag } return typeStructField } func inferPromptFieldName(structType reflect.Type, typeStructField string) string { field, _ := structType.Elem().FieldByName(typeStructField) tag, ok := getTagFieldName(field) if ok { return tag } return typeStructField } func getTagFieldName(field reflect.StructField) (string, bool) { tag, ok := field.Tag.Lookup("json") if ok { return strings.Split(tag, ",")[0], true } tag, ok = field.Tag.Lookup("yaml") if ok { return strings.Split(tag, ",")[0], true } return "", false } ================================================ FILE: pkg/lib/configreader/reader_test.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package configreader import ( "fmt" "reflect" "testing" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/stretchr/testify/require" ) type SimpleConfig struct { Key1 bool `json:"key1,omitempty"` Key2 bool `json:"key2"` } func TestSimple(t *testing.T) { structValidation := &StructValidation{ StructFieldValidations: []*StructFieldValidation{ { // Key: "key1", StructField: "Key1", BoolValidation: &BoolValidation{ Required: true, }, }, { // Key: "key2", StructField: "Key2", BoolValidation: &BoolValidation{ Default: true, }, }, }, Required: true, ShortCircuit: true, } configData := MustReadYAMLStr( ` key1: true `) expected := &SimpleConfig{ Key1: true, Key2: true, } testConfig(structValidation, configData, expected, t) } type NestedConfig struct { Key0 float64 `json:"key0"` Key1 *Nested1 `json:"key1"` Key2 *Nested2 `json:"key2"` } type Nested1 struct { Key11 int32 `json:"key11"` } type Nested2 struct { Key21 string `json:"key21"` Key22 *Nested3 `json:"key22"` } type Nested3 struct { Key31 int `json:"key31"` } func TestNested(t *testing.T) { structValidation := &StructValidation{ StructFieldValidations: []*StructFieldValidation{ { // Key: "key0", StructField: "Key0", Float64Validation: &Float64Validation{}, }, { // Key: "key1", StructField: "Key1", StructValidation: &StructValidation{ StructFieldValidations: []*StructFieldValidation{ { // Key: "key11", StructField: "Key11", Int32Validation: &Int32Validation{}, }, }, Required: true, ShortCircuit: true, }, }, { // Key: "key2", StructField: "Key2", StructValidation: &StructValidation{ StructFieldValidations: []*StructFieldValidation{ { // Key: "key21", StructField: "Key21", StringValidation: &StringValidation{}, }, { // Key: "key22", StructField: "Key22", StructValidation: &StructValidation{ StructFieldValidations: []*StructFieldValidation{ { // Key: "key31", StructField: "Key31", IntValidation: &IntValidation{}, }, }, Required: true, ShortCircuit: true, }, }, }, Required: true, ShortCircuit: true, }, }, }, Required: true, ShortCircuit: true, } configData := MustReadYAMLStr( ` key0: 1.1 key1: key11: 2 key2: key21: test key22: key31: 0 `) expected := &NestedConfig{ Key0: 1.1, Key1: &Nested1{ Key11: 2, }, Key2: &Nested2{ Key21: "test", Key22: &Nested3{ Key31: 0, }, }, } testConfig(structValidation, configData, expected, t) } type NestedListConfig struct { Key0 float64 `json:"key0"` Key1 *NestedList1 `json:"key1"` } type NestedList1 struct { Key11 []*NestedList2 `json:"key11"` } type NestedList2 struct { KeyA string `json:"keyA"` KeyB int `json:"keyB"` KeyC []float64 `json:"keyC"` KeyD *NestedList3 `json:"keyD"` } type NestedList3 struct { KeyX string `json:"keyX"` KeyY string `json:"keyY"` } func TestNestedList(t *testing.T) { structValidation := &StructValidation{ StructFieldValidations: []*StructFieldValidation{ { // Key: "key0", StructField: "Key0", Float64Validation: &Float64Validation{}, }, { // Key: "key1", StructField: "Key1", StructValidation: &StructValidation{ StructFieldValidations: []*StructFieldValidation{ { // Key: "key11", StructField: "Key11", StructListValidation: &StructListValidation{ StructValidation: &StructValidation{ StructFieldValidations: []*StructFieldValidation{ { // Key: "keyA", StructField: "KeyA", StringValidation: &StringValidation{}, }, { // Key: "keyB", StructField: "KeyB", IntValidation: &IntValidation{}, }, { // Key: "keyC", StructField: "KeyC", Float64ListValidation: &Float64ListValidation{}, }, { // Key: "keyD", StructField: "KeyD", StructValidation: &StructValidation{ StructFieldValidations: []*StructFieldValidation{ { // Key: "keyX", StructField: "KeyX", StringValidation: &StringValidation{}, }, { // Key: "keyY", StructField: "KeyY", StringValidation: &StringValidation{}, }, }, }, }, }, }, }, }, }, }, }, }, Required: true, ShortCircuit: true, } configData := MustReadYAMLStr( ` key0: 1.1 key1: key11: - keyA: A keyB: 0 keyC: - 0.1 - 0.2 keyD: keyX: test1 keyY: test2 - keyA: X keyB: 1 keyC: - 1.1 - 1.2 - 1.3 keyD: keyX: test3 keyY: test4 `) expected := &NestedListConfig{ Key0: 1.1, Key1: &NestedList1{ Key11: []*NestedList2{ { KeyA: "A", KeyB: 0, KeyC: []float64{float64(0.1), float64(0.2)}, KeyD: &NestedList3{ KeyX: "test1", KeyY: "test2", }, }, { KeyA: "X", KeyB: 1, KeyC: []float64{float64(1.1), float64(1.2), float64(1.3)}, KeyD: &NestedList3{ KeyX: "test3", KeyY: "test4", }, }, }, }, } testConfig(structValidation, configData, expected, t) } type Typed interface { GetType() string } type Typed1 struct { Key0 string `json:"key0"` Key1 string `json:"key1"` } type Typed1WithType struct { Type string `json:"type"` Key0 string `json:"key0"` Key1 string `json:"key1"` } func (t *Typed1) GetType() string { return "type1" } func (t *Typed1WithType) GetType() string { return "type1" } type Typed2 struct { KeyA int `json:"keyA"` KeyB int `json:"keyB"` } type Typed2WithType struct { Type string `json:"type"` KeyA int `json:"keyA"` KeyB int `json:"keyB"` } func (t *Typed2) GetType() string { return "type2" } func (t *Typed2WithType) GetType() string { return "type2" } type TypedConfig struct { Typed `json:"typed"` } var _interfaceStructValidation = &InterfaceStructValidation{ TypeKey: "type", InterfaceStructTypes: map[string]*InterfaceStructType{ "type1": { Type: (*Typed1)(nil), StructFieldValidations: []*StructFieldValidation{ { // Key: "key0", StructField: "Key0", StringValidation: &StringValidation{}, }, { // Key: "key1", StructField: "Key1", StringValidation: &StringValidation{}, }, }, }, "type2": { Type: (*Typed2)(nil), StructFieldValidations: []*StructFieldValidation{ { // Key: "keyA", StructField: "KeyA", IntValidation: &IntValidation{}, }, { // Key: "keyB", StructField: "KeyB", IntValidation: &IntValidation{}, }, }, }, }, } var _interfaceStructValidationWithTypeKeyConfig = &InterfaceStructValidation{ TypeKey: "type", TypeStructField: "Type", InterfaceStructTypes: map[string]*InterfaceStructType{ "type1": { Type: (*Typed1WithType)(nil), StructFieldValidations: []*StructFieldValidation{ { // Key: "key0", StructField: "Key0", StringValidation: &StringValidation{}, }, { // Key: "key1", StructField: "Key1", StringValidation: &StringValidation{}, }, }, }, "type2": { Type: (*Typed2WithType)(nil), StructFieldValidations: []*StructFieldValidation{ { // Key: "keyA", StructField: "KeyA", IntValidation: &IntValidation{}, }, { // Key: "keyB", StructField: "KeyB", IntValidation: &IntValidation{}, }, }, }, }, } func TestInterface(t *testing.T) { structValidation := &StructValidation{ StructFieldValidations: []*StructFieldValidation{ { // Key: "typed", StructField: "Typed", InterfaceStructValidation: _interfaceStructValidation, }, }, } configDataType1 := MustReadYAMLStr( ` typed: type: type1 key0: testA key1: testB `) configDataType2 := MustReadYAMLStr( ` typed: type: type2 keyA: 0 keyB: 1 `) expectedType1 := &TypedConfig{ Typed: &Typed1{ Key0: "testA", Key1: "testB", }, } expectedType2 := &TypedConfig{ Typed: &Typed2{ KeyA: 0, KeyB: 1, }, } testConfig(structValidation, configDataType1, expectedType1, t) testConfig(structValidation, configDataType2, expectedType2, t) structValidation = &StructValidation{ StructFieldValidations: []*StructFieldValidation{ { // Key: "typed", StructField: "Typed", InterfaceStructValidation: _interfaceStructValidationWithTypeKeyConfig, }, }, } expectedTypeWithTypeKey1 := &TypedConfig{ Typed: &Typed1WithType{ Type: "type1", Key0: "testA", Key1: "testB", }, } expectedTypeWithTypeKey2 := &TypedConfig{ Typed: &Typed2WithType{ Type: "type2", KeyA: 0, KeyB: 1, }, } testConfig(structValidation, configDataType1, expectedTypeWithTypeKey1, t) testConfig(structValidation, configDataType2, expectedTypeWithTypeKey2, t) } type TypedListConfig struct { Typeds []Typed `json:"typeds"` } func TestInterfaceList(t *testing.T) { structValidation := &StructValidation{ StructFieldValidations: []*StructFieldValidation{ { // Key: "typeds", StructField: "Typeds", InterfaceStructListValidation: &InterfaceStructListValidation{ InterfaceStructValidation: _interfaceStructValidation, }, }, }, } configData := MustReadYAMLStr( ` typeds: - type: type1 key0: testA key1: testB - type: type2 keyA: 0 keyB: 1 - type: type1 key0: test1 key1: test2 - type: type2 keyA: 0 keyB: -1 `) expected := &TypedListConfig{ Typeds: []Typed{ &Typed1{ Key0: "testA", Key1: "testB", }, &Typed2{ KeyA: 0, KeyB: 1, }, &Typed1{ Key0: "test1", Key1: "test2", }, &Typed2{ KeyA: 0, KeyB: -1, }, }, } testConfig(structValidation, configData, expected, t) structValidation = &StructValidation{ StructFieldValidations: []*StructFieldValidation{ { // Key: "typeds", StructField: "Typeds", InterfaceStructListValidation: &InterfaceStructListValidation{ InterfaceStructValidation: _interfaceStructValidationWithTypeKeyConfig, }, }, }, } expected = &TypedListConfig{ Typeds: []Typed{ &Typed1WithType{ Type: "type1", Key0: "testA", Key1: "testB", }, &Typed2WithType{ Type: "type2", KeyA: 0, KeyB: 1, }, &Typed1WithType{ Type: "type1", Key0: "test1", Key1: "test2", }, &Typed2WithType{ Type: "type2", KeyA: 0, KeyB: -1, }, }, } testConfig(structValidation, configData, expected, t) } type NullableConfig struct { Key1 *string `json:"key1"` Key2 []string `json:"key2"` Key3 interface{} `json:"key3"` } type NullableParentConfig struct { KeyA *NullableConfig `json:"key_a"` } func TestDefaultNull(t *testing.T) { configDataEmpty := MustReadYAMLStr(``) configDataNull := MustReadYAMLStr(`Null`) configDataEmptyMap := MustReadYAMLStr(`{}`) configDataNullValues := MustReadYAMLStr( ` key1: null key2: null key3: null `) configDataParentNullValues := MustReadYAMLStr( ` key_a: key1: null key2: null key3: null `) configDataParentNull := MustReadYAMLStr( ` key_a: null `) structFieldValidations := []*StructFieldValidation{ { StructField: "Key1", StringPtrValidation: &StringPtrValidation{ AllowExplicitNull: true, }, }, { StructField: "Key2", StringListValidation: &StringListValidation{ Default: []string{"key2"}, AllowExplicitNull: true, }, }, { StructField: "Key3", InterfaceValidation: &InterfaceValidation{ Default: "key3", AllowExplicitNull: true, }, }, } // AllowExplicitNull = true structValidation := &StructValidation{ StructFieldValidations: structFieldValidations, AllowExplicitNull: true, } var expected interface{} expected = &NullableConfig{} testConfig(structValidation, configDataEmpty, expected, t) testConfig(structValidation, configDataNull, expected, t) expected = &NullableConfig{ Key1: nil, Key2: []string{"key2"}, Key3: "key3", } testConfig(structValidation, configDataEmptyMap, expected, t) expected = &NullableConfig{ Key1: nil, Key2: nil, Key3: nil, } testConfig(structValidation, configDataNullValues, expected, t) // AllowExplicitNull = false structValidation = &StructValidation{ StructFieldValidations: structFieldValidations, AllowExplicitNull: false, } expected = &NullableConfig{} testConfigError(structValidation, configDataEmpty, expected, t) testConfigError(structValidation, configDataNull, expected, t) expected = &NullableConfig{ Key1: nil, Key2: []string{"key2"}, Key3: "key3", } testConfig(structValidation, configDataEmptyMap, expected, t) expected = &NullableConfig{ Key1: nil, Key2: nil, Key3: nil, } testConfig(structValidation, configDataNullValues, expected, t) // parent, AllowExplicitNull = true on both structValidation = &StructValidation{ StructFieldValidations: []*StructFieldValidation{ { StructField: "KeyA", StructValidation: &StructValidation{ StructFieldValidations: structFieldValidations, AllowExplicitNull: true, }, }, }, AllowExplicitNull: true, } expected = &NullableParentConfig{} testConfig(structValidation, configDataEmpty, expected, t) testConfig(structValidation, configDataNull, expected, t) expected = &NullableParentConfig{ KeyA: &NullableConfig{ Key1: nil, Key2: []string{"key2"}, Key3: "key3", }, } testConfig(structValidation, configDataEmptyMap, expected, t) expected = &NullableParentConfig{} testConfig(structValidation, configDataParentNull, expected, t) expected = &NullableParentConfig{ KeyA: &NullableConfig{ Key1: nil, Key2: nil, Key3: nil, }, } testConfig(structValidation, configDataParentNullValues, expected, t) // parent, AllowExplicitNull = false on both structValidation = &StructValidation{ StructFieldValidations: []*StructFieldValidation{ { StructField: "KeyA", StructValidation: &StructValidation{ StructFieldValidations: structFieldValidations, AllowExplicitNull: false, }, }, }, AllowExplicitNull: false, } expected = &NullableParentConfig{} testConfigError(structValidation, configDataEmpty, expected, t) testConfigError(structValidation, configDataNull, expected, t) expected = &NullableParentConfig{ KeyA: &NullableConfig{ Key1: nil, Key2: []string{"key2"}, Key3: "key3", }, } testConfig(structValidation, configDataEmptyMap, expected, t) testConfigError(structValidation, configDataParentNull, expected, t) expected = &NullableParentConfig{ KeyA: &NullableConfig{ Key1: nil, Key2: nil, Key3: nil, }, } testConfig(structValidation, configDataParentNullValues, expected, t) // parent, AllowExplicitNull = true on child, DefaultNil = true structValidation = &StructValidation{ StructFieldValidations: []*StructFieldValidation{ { StructField: "KeyA", StructValidation: &StructValidation{ StructFieldValidations: structFieldValidations, AllowExplicitNull: true, DefaultNil: true, }, }, }, AllowExplicitNull: false, } expected = &NullableParentConfig{} testConfigError(structValidation, configDataEmpty, expected, t) testConfigError(structValidation, configDataNull, expected, t) expected = &NullableParentConfig{} testConfig(structValidation, configDataEmptyMap, expected, t) expected = &NullableParentConfig{} testConfig(structValidation, configDataParentNull, expected, t) expected = &NullableParentConfig{ KeyA: &NullableConfig{ Key1: nil, Key2: nil, Key3: nil, }, } testConfig(structValidation, configDataParentNullValues, expected, t) // parent, AllowExplicitNull = false on both, DefaultNil = true structValidation = &StructValidation{ StructFieldValidations: []*StructFieldValidation{ { StructField: "KeyA", StructValidation: &StructValidation{ StructFieldValidations: structFieldValidations, AllowExplicitNull: false, DefaultNil: true, }, }, }, AllowExplicitNull: false, } expected = &NullableParentConfig{} testConfigError(structValidation, configDataEmpty, expected, t) testConfigError(structValidation, configDataNull, expected, t) expected = &NullableParentConfig{} testConfig(structValidation, configDataEmptyMap, expected, t) expected = &NullableParentConfig{} testConfigError(structValidation, configDataParentNull, expected, t) expected = &NullableParentConfig{ KeyA: &NullableConfig{ Key1: nil, Key2: nil, Key3: nil, }, } testConfig(structValidation, configDataParentNullValues, expected, t) } type DefaultConfig struct { Key1 bool `json:"key1"` Key2 string `json:"key2"` Key3 string `json:"key3"` } func TestDefaultField(t *testing.T) { structValidation := &StructValidation{ StructFieldValidations: []*StructFieldValidation{ { StructField: "Key1", BoolValidation: &BoolValidation{}, }, { StructField: "Key2", StringValidation: &StringValidation{}, }, { StructField: "Key3", DefaultField: "Key2", StringValidation: &StringValidation{}, }, }, } configData := MustReadYAMLStr( ` key1: true key2: "key2" key3: "key3" `) expected := &DefaultConfig{ Key1: true, Key2: "key2", Key3: "key3", } testConfig(structValidation, configData, expected, t) configData = MustReadYAMLStr( ` key1: true key2: "key2" `) expected = &DefaultConfig{ Key1: true, Key2: "key2", Key3: "key2", } testConfig(structValidation, configData, expected, t) structValidation = &StructValidation{ StructFieldValidations: []*StructFieldValidation{ { StructField: "Key1", BoolValidation: &BoolValidation{}, }, { StructField: "Key2", StringValidation: &StringValidation{}, }, { StructField: "Key3", DefaultDependentFields: []string{"Key2"}, DefaultDependentFieldsFunc: func(vals []interface{}) interface{} { return vals[0].(string) + ".py" }, StringValidation: &StringValidation{}, }, }, } configData = MustReadYAMLStr( ` key1: true key2: "key2" `) expected = &DefaultConfig{ Key1: true, Key2: "key2", Key3: "key2.py", } testConfig(structValidation, configData, expected, t) structValidation = &StructValidation{ StructFieldValidations: []*StructFieldValidation{ { StructField: "Key1", BoolValidation: &BoolValidation{}, }, { StructField: "Key2", StringValidation: &StringValidation{}, }, { StructField: "Key3", DefaultDependentFields: []string{"Key1"}, DefaultDependentFieldsFunc: func(vals []interface{}) interface{} { if vals[0].(bool) { return "It was true" } return "It was false" }, StringValidation: &StringValidation{}, }, }, } configData = MustReadYAMLStr( ` key1: true key2: "key2" `) expected = &DefaultConfig{ Key1: true, Key2: "key2", Key3: "It was true", } testConfig(structValidation, configData, expected, t) structValidation = &StructValidation{ StructFieldValidations: []*StructFieldValidation{ { StructField: "Key2", StringValidation: &StringValidation{}, }, { StructField: "Key3", StringValidation: &StringValidation{}, }, { StructField: "Key1", DefaultDependentFields: []string{"Key2"}, DefaultDependentFieldsFunc: func(vals []interface{}) interface{} { if vals[0].(string) == "key2" { return true } return false }, BoolValidation: &BoolValidation{}, }, }, } configData = MustReadYAMLStr( ` key2: "key2" key3: "key3" `) expected = &DefaultConfig{ Key1: true, Key2: "key2", Key3: "key3", } testConfig(structValidation, configData, expected, t) configData = MustReadYAMLStr( ` key2: "test" key3: "key3" `) expected = &DefaultConfig{ Key1: false, Key2: "test", Key3: "key3", } testConfig(structValidation, configData, expected, t) } func testConfig(structValidation *StructValidation, configData interface{}, expected interface{}, t *testing.T) { config := reflect.New(reflect.TypeOf(expected).Elem()).Interface() errs := Struct(config, configData, structValidation) if errs != nil { for _, err := range errs { fmt.Println("ERROR: " + errors.Message(err)) } } require.Empty(t, errs) require.Equal(t, expected, config) } func testConfigError(structValidation *StructValidation, configData interface{}, expectedTypeInstance interface{}, t *testing.T) { config := reflect.New(reflect.TypeOf(expectedTypeInstance).Elem()).Interface() errs := Struct(config, configData, structValidation) require.NotEmpty(t, errs) } ================================================ FILE: pkg/lib/configreader/string.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package configreader import ( "fmt" "strings" "github.com/cortexlabs/cortex/pkg/lib/cast" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/exit" "github.com/cortexlabs/cortex/pkg/lib/files" "github.com/cortexlabs/cortex/pkg/lib/prompt" "github.com/cortexlabs/cortex/pkg/lib/regex" "github.com/cortexlabs/cortex/pkg/lib/slices" s "github.com/cortexlabs/cortex/pkg/lib/strings" "github.com/cortexlabs/cortex/pkg/lib/urls" ) type StringValidation struct { Required bool Default string AllowEmpty bool // Allow `: ""` TreatNullAsEmpty bool // `: ` and `: null` will be read as `: ""` AllowedValues []string HiddenAllowedValues []string // allowed, but will not be listed as valid values (must be used in conjunction with AllowedValues) DisallowedValues []string CantBeSpecifiedErrStr *string Prefix string Suffix string InvalidPrefixes []string InvalidSuffixes []string AllowedPrefixes []string AllowedSuffixes []string MaxLength int MinLength int DisallowLeadingWhitespace bool DisallowTrailingWhitespace bool AlphaNumericDashDotUnderscoreOrEmpty bool AlphaNumericDashDotUnderscore bool AlphaNumericDashUnderscoreOrEmpty bool AlphaNumericDashUnderscore bool AlphaNumericDotUnderscore bool AlphaNumericDash bool AWSTag bool DNS1035 bool DNS1123 bool CastInt bool CastNumeric bool CastScalar bool AllowCortexResources bool RequireCortexResources bool DockerImage bool DockerImageOrEmpty bool Validator func(string) (string, error) } func EnvVar(envVarName string) string { return fmt.Sprintf("environment variable \"%s\"", envVarName) } func String(inter interface{}, v *StringValidation) (string, error) { if inter == nil { if v.TreatNullAsEmpty { return ValidateStringProvided("", v) } return "", ErrorCannotBeNull(v.Required) } casted, castOk := inter.(string) if !castOk { if v.CastScalar { if !cast.IsScalarType(inter) { return "", ErrorInvalidPrimitiveType(inter, PrimTypeString, PrimTypeInt, PrimTypeFloat, PrimTypeBool) } casted = s.ObjFlatNoQuotes(inter) } else if v.CastNumeric { if !cast.IsNumericType(inter) { return "", ErrorInvalidPrimitiveType(inter, PrimTypeString, PrimTypeInt, PrimTypeFloat) } casted = s.ObjFlatNoQuotes(inter) } else if v.CastInt { if !cast.IsIntType(inter) { return "", ErrorInvalidPrimitiveType(inter, PrimTypeString, PrimTypeInt) } casted = s.ObjFlatNoQuotes(inter) } else { return "", ErrorInvalidPrimitiveType(inter, PrimTypeString) } } return ValidateStringProvided(casted, v) } func StringFromInterfaceMap(key string, iMap map[string]interface{}, v *StringValidation) (string, error) { inter, ok := ReadInterfaceMapValue(key, iMap) if !ok { val, err := ValidateStringMissing(v) if err != nil { return "", errors.Wrap(err, key) } return val, nil } val, err := String(inter, v) if err != nil { return "", errors.Wrap(err, key) } return val, nil } func StringFromStrMap(key string, sMap map[string]string, v *StringValidation) (string, error) { valStr, ok := sMap[key] if !ok { val, err := ValidateStringMissing(v) if err != nil { return "", errors.Wrap(err, key) } return val, nil } val, err := StringFromStr(valStr, v) if err != nil { return "", errors.Wrap(err, key) } return val, nil } func StringFromStr(valStr string, v *StringValidation) (string, error) { return ValidateStringProvided(valStr, v) } func StringFromEnv(envVarName string, v *StringValidation) (string, error) { valStr := ReadEnvVar(envVarName) if valStr == nil { val, err := ValidateStringMissing(v) if err != nil { return "", errors.Wrap(err, EnvVar(envVarName)) } return val, nil } val, err := StringFromStr(*valStr, v) if err != nil { return "", errors.Wrap(err, EnvVar(envVarName)) } return val, nil } func StringFromFile(filePath string, v *StringValidation) (string, error) { if !files.IsFile(filePath) { val, err := ValidateStringMissing(v) if err != nil { return "", errors.Wrap(err, filePath) } return val, nil } valStr, err := files.ReadFile(filePath) if err != nil { return "", err } if len(valStr) == 0 { val, err := ValidateStringMissing(v) if err != nil { return "", errors.Wrap(err, filePath) } return val, nil } val, err := StringFromStr(valStr, v) if err != nil { return "", errors.Wrap(err, filePath) } return val, nil } func StringFromEnvOrFile(envVarName string, filePath string, v *StringValidation) (string, error) { valStr := ReadEnvVar(envVarName) if valStr != nil { return StringFromEnv(envVarName, v) } return StringFromFile(filePath, v) } func StringFromPrompt(promptOpts *prompt.Options, v *StringValidation) (string, error) { promptOpts.DefaultStr = v.Default valStr := prompt.Prompt(promptOpts) if valStr == "" { // Treat empty prompt value as missing return ValidateStringMissing(v) } return StringFromStr(valStr, v) } func ValidateStringMissing(v *StringValidation) (string, error) { if v.Required { return "", ErrorMustBeDefined(v.AllowedValues) } return validateString(v.Default, v) } func ValidateStringProvided(val string, v *StringValidation) (string, error) { if v.CantBeSpecifiedErrStr != nil { return "", ErrorFieldCantBeSpecified(*v.CantBeSpecifiedErrStr) } return validateString(val, v) } func validateString(val string, v *StringValidation) (string, error) { err := ValidateStringVal(val, v) if err != nil { return "", err } if v.Validator != nil { return v.Validator(val) } return val, nil } func ValidateStringVal(val string, v *StringValidation) error { if v.RequireCortexResources { if err := checkOnlyCortexResources(val); err != nil { return err } } else if !v.AllowCortexResources { if err := checkNoCortexResources(val); err != nil { return err } } if !v.AllowEmpty { if len(val) == 0 { return ErrorCannotBeEmpty() } } if len(v.AllowedValues) > 0 { if !slices.HasString(append(v.AllowedValues, v.HiddenAllowedValues...), val) { return ErrorInvalidStr(val, v.AllowedValues[0], v.AllowedValues[1:]...) } } if len(v.DisallowedValues) > 0 { if slices.HasString(v.DisallowedValues, val) { return ErrorDisallowedValue(val) } } if v.MaxLength > 0 && len(val) > v.MaxLength { return ErrorTooLong(val, v.MaxLength) } if v.MinLength > 0 && len(val) < v.MinLength { return ErrorTooShort(val, v.MinLength) } if v.Prefix != "" { if !strings.HasPrefix(val, v.Prefix) { return ErrorMustHavePrefix(val, v.Prefix) } } if v.Suffix != "" { if !strings.HasSuffix(val, v.Suffix) { return ErrorMustHaveSuffix(val, v.Suffix) } } for _, invalidPrefix := range v.InvalidPrefixes { if strings.HasPrefix(val, invalidPrefix) { return ErrorCantHavePrefix(val, invalidPrefix) } } for _, invalidSuffix := range v.InvalidSuffixes { if strings.HasSuffix(val, invalidSuffix) { return ErrorCantHaveSuffix(val, invalidSuffix) } } if len(v.AllowedPrefixes) > 0 { matchedPrefixes := 0 for _, allowedPrefix := range v.AllowedPrefixes { if strings.HasPrefix(val, allowedPrefix) { matchedPrefixes++ } } if matchedPrefixes == 0 { return ErrorMustHavePrefix(val, v.AllowedPrefixes[0], v.AllowedPrefixes[1:]...) } } if len(v.AllowedSuffixes) > 0 { matchedSuffixes := 0 for _, allowedSuffix := range v.AllowedSuffixes { if strings.HasSuffix(val, allowedSuffix) { matchedSuffixes++ } } if matchedSuffixes == 0 { return ErrorMustHaveSuffix(val, v.AllowedSuffixes[0], v.AllowedSuffixes[1:]...) } } if v.DisallowLeadingWhitespace { if regex.HasLeadingWhitespace(val) { return ErrorLeadingWhitespace(val) } } if v.DisallowTrailingWhitespace { if regex.HasTrailingWhitespace(val) { return ErrorTrailingWhitespace(val) } } if v.AlphaNumericDashDotUnderscore { if !regex.IsAlphaNumericDashDotUnderscore(val) { return ErrorAlphaNumericDashDotUnderscore(val) } } if v.AlphaNumericDashUnderscore { if !regex.IsAlphaNumericDashUnderscore(val) { return ErrorAlphaNumericDashUnderscore(val) } } if v.AlphaNumericDotUnderscore { if !regex.IsAlphaNumericDotUnderscore(val) { return ErrorAlphaNumericDotUnderscore(val) } } if v.AlphaNumericDash { if !regex.IsAlphaNumericDash(val) { return ErrorAlphaNumericDash(val) } } if v.AlphaNumericDashUnderscoreOrEmpty { if !regex.IsAlphaNumericDashUnderscore(val) && val != "" { return ErrorAlphaNumericDashUnderscore(val) } } if v.AlphaNumericDashDotUnderscoreOrEmpty { if !regex.IsAlphaNumericDashDotUnderscore(val) && val != "" { return ErrorAlphaNumericDashDotUnderscore(val) } } if v.AWSTag { if !regex.IsValidAWSTag(val) && val != "" { return ErrorInvalidAWSTag(val) } } if v.DockerImage { if !regex.IsValidDockerImage(val) { return ErrorInvalidDockerImage(val) } } if v.DockerImageOrEmpty { if !regex.IsValidDockerImage(val) && val != "" { return ErrorInvalidDockerImage(val) } } if v.DNS1035 { if err := urls.CheckDNS1035(val); err != nil { return err } } if v.DNS1123 { if err := urls.CheckDNS1123(val); err != nil { return err } } return nil } // // Musts // func MustStringFromEnv(envVarName string, v *StringValidation) string { val, err := StringFromEnv(envVarName, v) if err != nil { exit.Panic(err) } return val } func MustStringFromFile(filePath string, v *StringValidation) string { val, err := StringFromFile(filePath, v) if err != nil { exit.Panic(err) } return val } func MustStringFromEnvOrFile(envVarName string, filePath string, v *StringValidation) string { val, err := StringFromEnvOrFile(envVarName, filePath, v) if err != nil { exit.Panic(err) } return val } ================================================ FILE: pkg/lib/configreader/string_list.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package configreader import ( "github.com/cortexlabs/cortex/pkg/lib/cast" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/slices" s "github.com/cortexlabs/cortex/pkg/lib/strings" ) type StringListValidation struct { Required bool Default []string AllowExplicitNull bool AllowEmpty bool CantBeSpecifiedErrStr *string CastSingleItem bool DisallowDups bool MinLength int MaxLength int InvalidLengths []int AllowCortexResources bool RequireCortexResources bool ElementStringValidation *StringValidation // Required, Default, AllowEmpty, TreatNullAsEmpty & Validator fields not applicable here Validator func([]string) ([]string, error) } func StringList(inter interface{}, v *StringListValidation) ([]string, error) { casted, castOk := cast.InterfaceToStrSlice(inter) if !castOk { if v.CastSingleItem { castedItem, castOk := inter.(string) if !castOk { return nil, ErrorInvalidPrimitiveType(inter, PrimTypeString, PrimTypeStringList) } casted = []string{castedItem} } else { return nil, ErrorInvalidPrimitiveType(inter, PrimTypeStringList) } } return ValidateStringListProvided(casted, v) } func StringListFromInterfaceMap(key string, iMap map[string]interface{}, v *StringListValidation) ([]string, error) { inter, ok := ReadInterfaceMapValue(key, iMap) if !ok { val, err := ValidateStringListMissing(v) if err != nil { return nil, errors.Wrap(err, key) } return val, nil } val, err := StringList(inter, v) if err != nil { return nil, errors.Wrap(err, key) } return val, nil } func ValidateStringListMissing(v *StringListValidation) ([]string, error) { if v.Required { return nil, ErrorMustBeDefined() } return validateStringList(v.Default, v) } func ValidateStringListProvided(val []string, v *StringListValidation) ([]string, error) { if v.CantBeSpecifiedErrStr != nil { return nil, ErrorFieldCantBeSpecified(*v.CantBeSpecifiedErrStr) } if !v.AllowExplicitNull && val == nil { return nil, ErrorCannotBeNull(v.Required) } return validateStringList(val, v) } func validateStringList(val []string, v *StringListValidation) ([]string, error) { if v.RequireCortexResources { if err := checkOnlyCortexResources(val); err != nil { return nil, err } } else if !v.AllowCortexResources { if err := checkNoCortexResources(val); err != nil { return nil, err } } if !v.AllowEmpty { if val != nil && len(val) == 0 { return nil, ErrorCannotBeEmpty() } } if v.DisallowDups { if dups := slices.FindDuplicateStrs(val); len(dups) > 0 { return nil, ErrorDuplicatedValue(dups[0]) } } if v.MinLength != 0 { if len(val) < v.MinLength { return nil, ErrorTooFewElements(v.MinLength) } } if v.MaxLength != 0 { if len(val) > v.MaxLength { return nil, ErrorTooManyElements(v.MaxLength) } } for _, invalidLength := range v.InvalidLengths { if len(val) == invalidLength { return nil, ErrorWrongNumberOfElements(v.InvalidLengths) } } if v.ElementStringValidation != nil { for i, element := range val { err := ValidateStringVal(element, v.ElementStringValidation) if err != nil { return nil, errors.Wrap(err, s.Index(i)) } } } if v.Validator != nil { return v.Validator(val) } return val, nil } ================================================ FILE: pkg/lib/configreader/string_map.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package configreader import ( "github.com/cortexlabs/cortex/pkg/lib/cast" "github.com/cortexlabs/cortex/pkg/lib/errors" ) type StringMapValidation struct { Required bool Default map[string]string AllowExplicitNull bool AllowEmpty bool ConvertNullToEmpty bool CantBeSpecifiedErrStr *string KeyStringValidator *StringValidation ValueStringValidator *StringValidation Validator func(map[string]string) (map[string]string, error) } func StringMap(inter interface{}, v *StringMapValidation) (map[string]string, error) { casted, castOk := cast.InterfaceToStrStrMap(inter) if !castOk { return nil, ErrorInvalidPrimitiveType(inter, PrimTypeStringToStringMap) } return ValidateStringMapProvided(casted, v) } func StringMapFromInterfaceMap(key string, iMap map[string]interface{}, v *StringMapValidation) (map[string]string, error) { inter, ok := ReadInterfaceMapValue(key, iMap) if !ok { val, err := ValidateStringMapMissing(v) if err != nil { return nil, errors.Wrap(err, key) } return val, nil } val, err := StringMap(inter, v) if err != nil { return nil, errors.Wrap(err, key) } return val, nil } func ValidateStringMapMissing(v *StringMapValidation) (map[string]string, error) { if v.Required { return nil, ErrorMustBeDefined() } return validateStringMap(v.Default, v) } func ValidateStringMapProvided(val map[string]string, v *StringMapValidation) (map[string]string, error) { if v.CantBeSpecifiedErrStr != nil { return nil, ErrorFieldCantBeSpecified(*v.CantBeSpecifiedErrStr) } if !v.AllowExplicitNull && val == nil { return nil, ErrorCannotBeNull(v.Required) } return validateStringMap(val, v) } func validateStringMap(val map[string]string, v *StringMapValidation) (map[string]string, error) { if !v.AllowEmpty { if val != nil && len(val) == 0 { return nil, ErrorCannotBeEmpty() } } if v.KeyStringValidator != nil { for mapKey := range val { err := ValidateStringVal(mapKey, v.KeyStringValidator) if err != nil { return nil, err } } } if v.ValueStringValidator != nil { for mapKey, mapVal := range val { err := ValidateStringVal(mapVal, v.ValueStringValidator) if err != nil { return nil, errors.Wrap(err, mapKey) } } } if v.Validator != nil { return v.Validator(val) } if val == nil && v.ConvertNullToEmpty { val = make(map[string]string) } return val, nil } ================================================ FILE: pkg/lib/configreader/string_ptr.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package configreader import ( "github.com/cortexlabs/cortex/pkg/lib/cast" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/files" "github.com/cortexlabs/cortex/pkg/lib/prompt" s "github.com/cortexlabs/cortex/pkg/lib/strings" ) type StringPtrValidation struct { Required bool Default *string AllowExplicitNull bool AllowEmpty bool AllowedValues []string HiddenAllowedValues []string // allowed, but will not be listed as valid values (must be used in conjunction with AllowedValues) DisallowedValues []string CantBeSpecifiedErrStr *string Prefix string Suffix string InvalidPrefixes []string InvalidSuffixes []string AllowedPrefixes []string AllowedSuffixes []string MaxLength int MinLength int DisallowLeadingWhitespace bool DisallowTrailingWhitespace bool AlphaNumericDashDotUnderscoreOrEmpty bool AlphaNumericDashDotUnderscore bool AlphaNumericDashUnderscore bool AlphaNumericDotUnderscore bool AWSTag bool DNS1035 bool DNS1123 bool CastInt bool CastNumeric bool CastScalar bool AllowCortexResources bool RequireCortexResources bool DockerImage bool DockerImageOrEmpty bool Validator func(string) (string, error) } func makeStringValValidation(v *StringPtrValidation) *StringValidation { return &StringValidation{ AllowEmpty: v.AllowEmpty, AllowedValues: v.AllowedValues, HiddenAllowedValues: v.HiddenAllowedValues, DisallowedValues: v.DisallowedValues, Prefix: v.Prefix, Suffix: v.Suffix, InvalidPrefixes: v.InvalidPrefixes, InvalidSuffixes: v.InvalidSuffixes, AllowedPrefixes: v.AllowedPrefixes, AllowedSuffixes: v.AllowedSuffixes, MaxLength: v.MaxLength, MinLength: v.MinLength, DisallowLeadingWhitespace: v.DisallowLeadingWhitespace, DisallowTrailingWhitespace: v.DisallowTrailingWhitespace, AlphaNumericDashDotUnderscoreOrEmpty: v.AlphaNumericDashDotUnderscoreOrEmpty, AlphaNumericDashDotUnderscore: v.AlphaNumericDashDotUnderscore, AlphaNumericDashUnderscore: v.AlphaNumericDashUnderscore, AlphaNumericDotUnderscore: v.AlphaNumericDotUnderscore, AWSTag: v.AWSTag, DNS1035: v.DNS1035, DNS1123: v.DNS1123, CastInt: v.CastInt, CastNumeric: v.CastNumeric, CastScalar: v.CastScalar, AllowCortexResources: v.AllowCortexResources, RequireCortexResources: v.RequireCortexResources, DockerImage: v.DockerImage, DockerImageOrEmpty: v.DockerImageOrEmpty, } } func StringPtr(inter interface{}, v *StringPtrValidation) (*string, error) { if inter == nil { return ValidateStringPtrProvided(nil, v) } casted, castOk := inter.(string) if !castOk { if v.CastScalar { if !cast.IsScalarType(inter) { return nil, ErrorInvalidPrimitiveType(inter, PrimTypeString, PrimTypeInt, PrimTypeFloat, PrimTypeBool) } casted = s.ObjFlatNoQuotes(inter) } else if v.CastNumeric { if !cast.IsNumericType(inter) { return nil, ErrorInvalidPrimitiveType(inter, PrimTypeString, PrimTypeInt, PrimTypeFloat) } casted = s.ObjFlatNoQuotes(inter) } else if v.CastInt { if !cast.IsIntType(inter) { return nil, ErrorInvalidPrimitiveType(inter, PrimTypeString, PrimTypeInt) } casted = s.ObjFlatNoQuotes(inter) } else { return nil, ErrorInvalidPrimitiveType(inter, PrimTypeString) } } return ValidateStringPtrProvided(&casted, v) } func StringPtrFromInterfaceMap(key string, iMap map[string]interface{}, v *StringPtrValidation) (*string, error) { inter, ok := ReadInterfaceMapValue(key, iMap) if !ok { val, err := ValidateStringPtrMissing(v) if err != nil { return nil, errors.Wrap(err, key) } return val, nil } val, err := StringPtr(inter, v) if err != nil { return nil, errors.Wrap(err, key) } return val, nil } func StringPtrFromStrMap(key string, sMap map[string]string, v *StringPtrValidation) (*string, error) { valStr, ok := sMap[key] if !ok { val, err := ValidateStringPtrMissing(v) if err != nil { return nil, errors.Wrap(err, key) } return val, nil } val, err := StringPtrFromStr(valStr, v) if err != nil { return nil, errors.Wrap(err, key) } return val, nil } func StringPtrFromStr(str string, v *StringPtrValidation) (*string, error) { return ValidateStringPtrProvided(&str, v) } func StringPtrFromEnv(envVarName string, v *StringPtrValidation) (*string, error) { valStr := ReadEnvVar(envVarName) if valStr == nil { val, err := ValidateStringPtrMissing(v) if err != nil { return nil, errors.Wrap(err, EnvVar(envVarName)) } return val, nil } val, err := StringPtrFromStr(*valStr, v) if err != nil { return nil, errors.Wrap(err, EnvVar(envVarName)) } return val, nil } func StringPtrFromFile(filePath string, v *StringPtrValidation) (*string, error) { if !files.IsFile(filePath) { val, err := ValidateStringPtrMissing(v) if err != nil { return nil, errors.Wrap(err, filePath) } return val, nil } valStr, err := files.ReadFile(filePath) if err != nil { return nil, err } if len(valStr) == 0 { val, err := ValidateStringPtrMissing(v) if err != nil { return nil, errors.Wrap(err, filePath) } return val, nil } val, err := StringPtrFromStr(valStr, v) if err != nil { return nil, errors.Wrap(err, filePath) } return val, nil } func StringPtrFromEnvOrFile(envVarName string, filePath string, v *StringPtrValidation) (*string, error) { valStr := ReadEnvVar(envVarName) if valStr != nil { return StringPtrFromEnv(envVarName, v) } return StringPtrFromFile(filePath, v) } func StringPtrFromPrompt(promptOpts *prompt.Options, v *StringPtrValidation) (*string, error) { if v.Default != nil && promptOpts.DefaultStr == "" { promptOpts.DefaultStr = *v.Default } valStr := prompt.Prompt(promptOpts) if valStr == "" { // Treat empty prompt value as missing return ValidateStringPtrMissing(v) } return StringPtrFromStr(valStr, v) } func ValidateStringPtrMissing(v *StringPtrValidation) (*string, error) { if v.Required { return nil, ErrorMustBeDefined(v.AllowedValues) } return validateStringPtr(v.Default, v) } func ValidateStringPtrProvided(val *string, v *StringPtrValidation) (*string, error) { if v.CantBeSpecifiedErrStr != nil { return nil, ErrorFieldCantBeSpecified(*v.CantBeSpecifiedErrStr) } if !v.AllowExplicitNull && val == nil { return nil, ErrorCannotBeNull(v.Required) } return validateStringPtr(val, v) } func validateStringPtr(val *string, v *StringPtrValidation) (*string, error) { if val != nil { err := ValidateStringVal(*val, makeStringValValidation(v)) if err != nil { return nil, err } } if val == nil { return val, nil } if v.Validator != nil { validated, err := v.Validator(*val) if err != nil { return nil, err } return &validated, nil } return val, nil } ================================================ FILE: pkg/lib/configreader/types.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package configreader type TypePlaceholder struct { Type string `json:"type"` } type PrimitiveType string type PrimitiveTypes []PrimitiveType var ( PrimTypeInt PrimitiveType = "integer" PrimTypeIntList PrimitiveType = "integer list" PrimTypeFloat PrimitiveType = "float" PrimTypeFloatList PrimitiveType = "float list" PrimTypeString PrimitiveType = "string" PrimTypeStringList PrimitiveType = "string list" PrimTypeBool PrimitiveType = "boolean" PrimTypeBoolList PrimitiveType = "boolean list" PrimTypeMap PrimitiveType = "map" PrimTypeMapList PrimitiveType = "list of maps" PrimTypeList PrimitiveType = "list" PrimTypeStringToStringMap PrimitiveType = "map of strings to strings" ) var PrimTypeScalars = []PrimitiveType{PrimTypeInt, PrimTypeFloat, PrimTypeString, PrimTypeBool} func (ts PrimitiveTypes) StringList() []string { strs := make([]string, len(ts)) for i, t := range ts { strs[i] = string(t) } return strs } ================================================ FILE: pkg/lib/configreader/validators.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package configreader import ( "regexp" "strings" "time" "github.com/cortexlabs/cortex/pkg/lib/aws" "github.com/cortexlabs/cortex/pkg/lib/docker" "github.com/cortexlabs/cortex/pkg/lib/files" ) var _emailRegex *regexp.Regexp func init() { _emailRegex = regexp.MustCompile("^[a-zA-Z0-9.!#$%&'*+/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$") } func GetFilePathValidator(baseDir string) func(string) (string, error) { return func(val string) (string, error) { if err := files.CheckFile(val); err != nil { return "", err } return val, nil } } func S3aPathValidator(val string) (string, error) { if !aws.IsValidS3aPath(val) { return "", aws.ErrorInvalidS3aPath(val) } return val, nil } func S3PathValidator(val string) (string, error) { if !aws.IsValidS3Path(val) { return "", aws.ErrorInvalidS3Path(val) } return val, nil } func EmailValidator(val string) (string, error) { if len(val) > 320 { return "", ErrorEmailTooLong() } if !_emailRegex.MatchString(val) { return "", ErrorEmailInvalid() } return val, nil } type DurationValidation struct { GreaterThan *time.Duration GreaterThanOrEqualTo *time.Duration LessThan *time.Duration LessThanOrEqualTo *time.Duration MultipleOf *time.Duration } func DurationParser(v *DurationValidation) func(string) (interface{}, error) { return func(str string) (interface{}, error) { d, err := time.ParseDuration(str) if err != nil { return nil, err } if v == nil { return d, nil } if v.GreaterThan != nil { if d <= *v.GreaterThan { return nil, ErrorMustBeGreaterThan(str, *v.GreaterThan) } } if v.GreaterThanOrEqualTo != nil { if d < *v.GreaterThanOrEqualTo { return nil, ErrorMustBeGreaterThanOrEqualTo(str, *v.GreaterThanOrEqualTo) } } if v.LessThan != nil { if d >= *v.LessThan { return nil, ErrorMustBeLessThan(str, *v.LessThan) } } if v.LessThanOrEqualTo != nil { if d > *v.LessThanOrEqualTo { return nil, ErrorMustBeLessThanOrEqualTo(str, *v.LessThanOrEqualTo) } } if v.MultipleOf != nil { if d.Nanoseconds()%(*v.MultipleOf).Nanoseconds() != 0 { return nil, ErrorIsNotMultiple(d, *v.MultipleOf) } } return d, nil } } func ValidateImageVersion(image, cortexVersion string) (string, error) { if !strings.HasPrefix(image, "quay.io/cortexlabs/") && !strings.HasPrefix(image, "quay.io/cortexlabsdev/") && !strings.HasPrefix(image, "docker.io/cortexlabs/") && !strings.HasPrefix(image, "docker.io/cortexlabsdev/") && !strings.HasPrefix(image, "cortexlabs/") && !strings.HasPrefix(image, "cortexlabsdev/") { return image, nil } tag := docker.ExtractImageTag(image) // in docker, missing tag implies "latest" if tag == "" { tag = "latest" } if !strings.HasPrefix(tag, cortexVersion) { return "", ErrorImageVersionMismatch(image, tag, cortexVersion) } return image, nil } ================================================ FILE: pkg/lib/console/format.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package console import ( "github.com/fatih/color" ) var _bold = color.New(color.Bold).SprintFunc() // Bold returns a string formatted in bold func Bold(a ...interface{}) string { return _bold(a...) } // BoolColor converts a boolean into a colored string (green: true, red: false) func BoolColor(b bool) string { if b { return color.GreenString("%t", b) } return color.RedString("%t", b) } ================================================ FILE: pkg/lib/cron/cron.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package cron import ( "time" "github.com/cortexlabs/cortex/pkg/lib/errors" ) type Cron struct { cronRun chan struct{} cronCancel chan struct{} } func Run(f func() error, errHandler func(error), delay time.Duration) Cron { cronRun := make(chan struct{}, 1) cronCancel := make(chan struct{}, 1) runCron := func() { defer Recoverer(errHandler) err := f() if err != nil && errHandler != nil { errHandler(err) } } go func() { timer := time.NewTimer(0) defer timer.Stop() for { select { case <-cronCancel: return case <-cronRun: runCron() case <-timer.C: runCron() } timer.Reset(delay) } }() return Cron{ cronRun: cronRun, cronCancel: cronCancel, } } func (c *Cron) RunNow() { c.cronRun <- struct{}{} } func (c *Cron) Cancel() { c.cronCancel <- struct{}{} } func Recoverer(errHandler func(error)) { if errInterface := recover(); errInterface != nil { err := errors.CastRecoverError(errInterface) errors.PrintStacktrace(err) if errHandler != nil { errHandler(err) } } } ================================================ FILE: pkg/lib/debug/debug.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package debug import ( "encoding/json" "fmt" "github.com/cortexlabs/cortex/pkg/lib/errors" s "github.com/cortexlabs/cortex/pkg/lib/strings" "github.com/cortexlabs/yaml" "github.com/davecgh/go-spew/spew" ) func Pp(obj interface{}) { fmt.Println(s.Obj(obj)) } func Ppg(obj interface{}) { fmt.Print(Sppg(obj)) } func Sppg(obj interface{}) string { spew.Config.SortKeys = true spew.Config.SpewKeys = true spew.Config.Indent = " " spew.Config.ContinueOnMethod = true spew.Config.DisablePointerAddresses = true spew.Config.DisableCapacities = true return spew.Sdump(obj) } func Ppj(obj interface{}) { b, err := json.MarshalIndent(obj, "", " ") if err != nil { errors.PrintError(err) } fmt.Println(string(b)) } func Ppy(obj interface{}) { b, err := yaml.Marshal(obj) if err != nil { errors.PrintError(err) } fmt.Println(string(b)) } ================================================ FILE: pkg/lib/docker/docker.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package docker import ( "bytes" "context" "encoding/base64" "encoding/json" "fmt" "io" "io/ioutil" "os" "path/filepath" "strings" "time" "github.com/cortexlabs/cortex/pkg/lib/archive" "github.com/cortexlabs/cortex/pkg/lib/aws" "github.com/cortexlabs/cortex/pkg/lib/cron" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/exit" "github.com/cortexlabs/cortex/pkg/lib/parallel" "github.com/cortexlabs/cortex/pkg/lib/print" "github.com/cortexlabs/cortex/pkg/lib/slices" dockertypes "github.com/docker/docker/api/types" dockerclient "github.com/docker/docker/client" "github.com/docker/docker/pkg/jsonmessage" "github.com/docker/docker/pkg/term" ) var NoAuth string var _cachedClient *Client func init() { NoAuth, _ = EncodeAuthConfig(dockertypes.AuthConfig{}) } type Client struct { *dockerclient.Client Info dockertypes.Info } func GetDockerClient() (*Client, error) { if _cachedClient != nil { return _cachedClient, nil } baseClient, err := dockerclient.NewClientWithOpts(dockerclient.FromEnv) if err != nil { return nil, WrapDockerError(err) } baseClient.NegotiateAPIVersion(context.Background()) info, err := baseClient.Info(context.Background()) if err != nil { return nil, WrapDockerError(err) } _cachedClient = &Client{ Client: baseClient, Info: info, } return _cachedClient, nil } func MustDockerClient() *Client { dockerClient, err := GetDockerClient() if err != nil { exit.Error(err) } return dockerClient } func AWSAuthConfig(awsClient *aws.Client) (string, error) { dockerClient, err := GetDockerClient() if err != nil { return "", err } ecrAuthConfig, err := awsClient.GetECRAuthConfig() if err != nil { return "", err } auth := dockertypes.AuthConfig{ Username: ecrAuthConfig.Username, Password: ecrAuthConfig.AccessToken, ServerAddress: ecrAuthConfig.ProxyEndpoint, } _, err = dockerClient.RegistryLogin(context.Background(), auth) if err != nil { return "", err } authConfig, err := EncodeAuthConfig(auth) if err != nil { return "", err } return authConfig, nil } func WrapDockerError(err error) error { if dockerclient.IsErrConnectionFailed(err) { return ErrorConnectToDockerDaemon() } if strings.Contains(strings.ToLower(err.Error()), "permission denied") { return ErrorDockerPermissions(err) } return errors.WithStack(err) } type PullVerbosity int const ( NoPrint PullVerbosity = iota PrintDots PrintProgressBars ) func PullImage(image string, encodedAuthConfig string, pullVerbosity PullVerbosity) (bool, error) { dockerClient, err := GetDockerClient() if err != nil { return false, err } if err := CheckImageExistsLocally(dockerClient, image); err == nil { return false, nil } pullOutput, err := dockerClient.ImagePull(context.Background(), image, dockertypes.ImagePullOptions{ RegistryAuth: encodedAuthConfig, }) if err != nil { return false, WrapDockerError(err) } defer pullOutput.Close() switch pullVerbosity { case PrintProgressBars: termFd, isTerm := term.GetFdInfo(os.Stderr) jsonmessage.DisplayJSONMessagesStream(pullOutput, os.Stderr, termFd, isTerm, nil) fmt.Println() case PrintDots: var err error fmt.Printf("￮ downloading docker image %s ", image) defer func() { if err == nil { fmt.Print(" ✓\n") } else { fmt.Print(" x\n") } }() d := json.NewDecoder(pullOutput) var result jsonmessage.JSONMessage dotCron := cron.Run(print.Dot, nil, 2*time.Second) defer dotCron.Cancel() for { if e := d.Decode(&result); e != nil { if e == io.EOF { return true, nil } err = e return false, err } if result.Error != nil { err = result.Error break } } if err != nil { return false, err } default: // wait until the pull has completed if _, err := ioutil.ReadAll(pullOutput); err != nil { return false, err } } return true, nil } func StreamDockerLogs(containerID string, containerIDs ...string) error { containerIDs = append([]string{containerID}, containerIDs...) dockerClient, err := GetDockerClient() if err != nil { return err } fns := make([]func() error, len(containerIDs)) for i, containerID := range containerIDs { fns[i] = StreamDockerLogsFn(containerID, dockerClient) } err = parallel.RunFirstErr(fns[0], fns[1:]...) if err != nil { return WrapDockerError(err) } return nil } func StreamDockerLogsFn(containerID string, dockerClient *Client) func() error { return func() error { // Use ContainerLogs() so lines are only printed once they end in \n logsOutput, err := dockerClient.ContainerLogs(context.Background(), containerID, dockertypes.ContainerLogsOptions{ ShowStdout: true, ShowStderr: true, Follow: true, }) if err != nil { return WrapDockerError(err) } _, err = io.Copy(os.Stdout, logsOutput) if err != nil && err != io.EOF { return errors.WithStack(err) } return nil } } // The provided input will be extracted into the container's containerPath directory func CopyToContainer(containerID string, input *archive.Input, containerPath string) error { if !strings.HasPrefix(containerPath, "/") { return errors.ErrorUnexpected("containerPath must start with /") } dockerClient, err := GetDockerClient() if err != nil { return err } // this is necessary to ensure that missing parent directories are created in the container input.AddPrefix = filepath.Join(containerPath, input.AddPrefix) buf := new(bytes.Buffer) _, err = archive.TarToWriter(input, buf) if err != nil { return err } err = dockerClient.CopyToContainer(context.Background(), containerID, "/", buf, dockertypes.CopyToContainerOptions{ AllowOverwriteDirWithFile: true, }) if err != nil { return WrapDockerError(err) } return nil } // The file or directory name of containerPath will be preserved in localDir // For example, if the container has /aaa/zzz.txt, // - CopyFromContainer(_, "/aaa", "~/test") will create "~/test/aaa/zzz.txt" // - CopyFromContainer(_, "/aaa/zzz.txt", "~/test") will create "~/test/zzz.txt" func CopyFromContainer(containerID string, containerPath string, localDir string) error { if !strings.HasPrefix(containerPath, "/") { return errors.ErrorUnexpected("containerPath must start with /") } dockerClient, err := GetDockerClient() if err != nil { return err } reader, _, err := dockerClient.CopyFromContainer(context.Background(), containerID, containerPath) if err != nil { return WrapDockerError(err) } defer reader.Close() _, err = archive.UntarReaderToDir(reader, localDir) if err != nil { return err } return nil } func EncodeAuthConfig(authConfig dockertypes.AuthConfig) (string, error) { encoded, err := json.Marshal(authConfig) if err != nil { return "", errors.Wrap(err, "failed to encode docker login credentials") } registryAuth := base64.URLEncoding.EncodeToString(encoded) return registryAuth, nil } func CheckImageAccessible(dockerClient *Client, dockerImage, authConfig string) error { if _, err := dockerClient.DistributionInspect(context.Background(), dockerImage, authConfig); err != nil { return ErrorImageInaccessible(dockerImage, err) } return nil } func CheckImageExistsLocally(dockerClient *Client, dockerImage string) error { images, err := dockerClient.ImageList(context.Background(), dockertypes.ImageListOptions{}) if err != nil { return WrapDockerError(err) } // in docker, missing tag implies "latest" if ExtractImageTag(dockerImage) == "" { dockerImage = fmt.Sprintf("%s:latest", dockerImage) } for _, image := range images { if slices.HasString(image.RepoTags, dockerImage) { return nil } } return ErrorImageDoesntExistLocally(dockerImage) } func ExtractImageTag(dockerImage string) string { if colonIndex := strings.LastIndex(dockerImage, ":"); colonIndex != -1 { return dockerImage[colonIndex+1:] } return "" } ================================================ FILE: pkg/lib/docker/errors.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package docker import ( "fmt" "runtime" "strings" "github.com/cortexlabs/cortex/pkg/consts" "github.com/cortexlabs/cortex/pkg/lib/errors" ) const ( ErrConnectToDockerDaemon = "docker.connect_to_docker_daemon" ErrDockerPermissions = "docker.docker_permissions" ErrImageDoesntExistLocally = "docker.image_doesnt_exist_locally" ErrImageInaccessible = "docker.image_inaccessible" ) func ErrorConnectToDockerDaemon() error { installMsg := "install it by following the instructions for your operating system: https://docs.docker.com/install" if strings.HasPrefix(runtime.GOOS, "darwin") { installMsg = "install it here: https://docs.docker.com/docker-for-mac/install" } return errors.WithStack(&errors.Error{ Kind: ErrConnectToDockerDaemon, Message: fmt.Sprintf("unable to connect to the Docker daemon\n\nplease confirm Docker is running, or if Docker is not installed, %s", installMsg), }) } func ErrorDockerPermissions(err error) error { errStr := errors.Message(err) var groupAddStr string if strings.HasPrefix(runtime.GOOS, "linux") { groupAddStr = " (e.g. by running `sudo groupadd docker; sudo gpasswd -a $USER docker` and then restarting your terminal)" } return errors.WithStack(&errors.Error{ Kind: ErrDockerPermissions, Message: errStr + "\n\nyou can re-run this command with `sudo`, or grant your current user access to docker" + groupAddStr, }) } func ErrorImageDoesntExistLocally(image string) error { return errors.WithStack(&errors.Error{ Kind: ErrImageDoesntExistLocally, Message: fmt.Sprintf("%s does not exist locally; download it with `docker pull %s` (if your registry is private, run `docker login` first)", image, image), }) } func ErrorImageInaccessible(image string, cause error) error { message := fmt.Sprintf("%s is not accessible", image) if cause != nil { message += "\n" + errors.Message(cause) // add \n because docker client errors are verbose but useful } if strings.Contains(cause.Error(), "auth") { message += fmt.Sprintf("\n\nif you would like to use a private docker registry, see https://docs.cortexlabs.com/v/%s/", consts.CortexVersionMinor) } return errors.WithStack(&errors.Error{ Kind: ErrImageInaccessible, Message: message, Cause: cause, }) } ================================================ FILE: pkg/lib/errors/error.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package errors import ( "fmt" "io" "strings" pkgerrors "github.com/pkg/errors" ) const ErrNotCortexError = "error" type Error struct { Kind string Message string Metadata interface{} // won't be printed NoTelemetry bool NoPrint bool Cause error stack *stack } func (cortexError *Error) Error() string { return cortexError.Message } func (cortexError *Error) StackTrace() pkgerrors.StackTrace { stackTrace := make([]pkgerrors.Frame, len(*cortexError.stack)) for i := 0; i < len(stackTrace); i++ { stackTrace[i] = pkgerrors.Frame((*cortexError.stack)[i]) } return stackTrace } func WithStack(err error) error { if err == nil { return nil } cortexError := getCortexError(err) if cortexError == nil { cortexError = &Error{ Kind: ErrNotCortexError, Message: strings.TrimSpace(err.Error()), Cause: err, } } if cortexError.stack == nil { cortexError.stack = callers() } return cortexError } func Wrap(err error, strs ...string) error { if err == nil { return nil } cortexError := WithStack(err).(*Error) strs = removeEmptyStrs(strs) strs = append(strs, cortexError.Message) cortexError.Message = strings.Join(strs, ": ") return cortexError } func Wrapf(err error, template string, params ...string) error { return Wrap(err, fmt.Sprintf(template, params)) } // adds to the end of the error message (without adding any whitespace or punctuation) func Append(err error, str string) error { if err == nil { return nil } cortexError := WithStack(err).(*Error) cortexError.Message = cortexError.Message + str return cortexError } func getCortexError(err error) *Error { if cortexError, ok := err.(*Error); ok { return cortexError } return nil } func GetKind(err error) string { if cortexError, ok := err.(*Error); ok { return cortexError.Kind } return ErrNotCortexError } func GetMetadata(err error) interface{} { if cortexError, ok := err.(*Error); ok { return cortexError.Metadata } return nil } func IsNoTelemetry(err error) bool { if cortexError, ok := err.(*Error); ok { return cortexError.NoTelemetry } return false } func SetNoTelemetry(err error) error { cortexError := WithStack(err).(*Error) cortexError.NoTelemetry = true return cortexError } func IsNoPrint(err error) bool { if cortexError, ok := err.(*Error); ok { return cortexError.NoPrint } return false } func SetNoPrint(err error) error { cortexError := WithStack(err).(*Error) cortexError.NoPrint = true return cortexError } // Returns nil if no cause func Cause(err error) error { if cortexError, ok := err.(*Error); ok { return cortexError.Cause } return nil } func CauseOrSelf(err error) error { if cortexError, ok := err.(*Error); ok { cause := cortexError.Cause if cause != nil { return cause } } return err } func PrintStacktrace(err error) { fmt.Printf("%+v\n", err) } func (cortexError *Error) Format(s fmt.State, verb rune) { switch verb { case 'v': if s.Flag('+') { io.WriteString(s, cortexError.Message) cortexError.stack.Format(s, verb) return } fallthrough case 's': io.WriteString(s, cortexError.Message) case 'q': fmt.Fprintf(s, "%q", cortexError.Message) } } func CastRecoverError(errInterface interface{}, strs ...string) error { var err error var ok bool err, ok = errInterface.(error) if !ok { err = &Error{ Kind: ErrNotCortexError, Message: fmt.Sprint(errInterface), } } return Wrap(err, strs...) } func removeEmptyStrs(strs []string) []string { var cleanStrs []string for _, str := range strs { if str != "" { cleanStrs = append(cleanStrs, str) } } return cleanStrs } ================================================ FILE: pkg/lib/errors/errors.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package errors import ( "fmt" "strings" s "github.com/cortexlabs/cortex/pkg/lib/strings" ) const ( ErrUnexpected = "errors.unexpected" ) func ErrorUnexpected(msgs ...interface{}) error { strs := make([]string, len(msgs)) for i, msg := range msgs { strs[i] = s.ObjFlatNoQuotes(msg) } return WithStack(&Error{ Kind: ErrUnexpected, Message: strings.Join(strs, ": "), }) } func ListOfErrors(errKind string, shouldPrint bool, errors ...error) error { var errorsContents string for i, err := range errors { if err != nil { errorsContents += fmt.Sprintf("error #%d: %s\n", i+1, err.Error()) } } if errorsContents == "" { return nil } return WithStack(&Error{ Kind: errKind, Message: errorsContents, NoPrint: !shouldPrint, }) } ================================================ FILE: pkg/lib/errors/message.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package errors import ( "strings" "github.com/aws/aws-sdk-go/aws/awserr" "github.com/cortexlabs/cortex/pkg/lib/print" ) func PrintError(err error, strs ...string) { print.StderrPrintln(ErrorStr(err, strs...)) // PrintStacktrace(err) } func PrintErrorForUser(err error, strs ...string) { print.StderrBoldFirstLine(ErrorStr(err, strs...)) // PrintStacktrace(err) } func ErrorStr(err error, strs ...string) string { wrappedErr := Wrap(err, strs...) return "error: " + strings.TrimSpace(Message(wrappedErr)) } func Message(err error, strs ...string) string { wrappedErr := Wrap(err, strs...) errStr := wrappedErr.Error() return strings.TrimSpace(errStr) } func MessageFirstLine(err error, strs ...string) string { wrappedErr := Wrap(err, strs...) var errStr string if _, ok := CauseOrSelf(wrappedErr).(awserr.Error); ok { errStr = strings.Split(strings.TrimSpace(wrappedErr.Error()), "\n")[0] } else { errStr = wrappedErr.Error() } return strings.TrimSpace(errStr) } ================================================ FILE: pkg/lib/errors/multi.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package errors func AddError(errs []error, err error, strs ...string) ([]error, bool) { ok := false if err != nil { errs = append(errs, Wrap(err, strs...)) ok = true } return errs, ok } func AddErrors(errs []error, newErrs []error, strs ...string) ([]error, bool) { ok := false for _, err := range newErrs { if err != nil { errs = append(errs, Wrap(err, strs...)) ok = true } } return errs, ok } func WrapAll(errs []error, strs ...string) []error { if !HasError(errs) { return nil } wrappedErrs := make([]error, len(errs)) for i, err := range errs { wrappedErrs[i] = Wrap(err, strs...) } return wrappedErrs } func HasError(errs []error) bool { for _, err := range errs { if err != nil { return true } } return false } func AreAllErrors(errs []error) bool { for _, err := range errs { if err == nil { return false } } return true } func FirstError(errs ...error) error { for _, err := range errs { if err != nil { return err } } return nil } func MapHasError(errs map[string]error) bool { for _, err := range errs { if err != nil { return true } } return false } func FirstErrorInMap(errs map[string]error) error { for _, err := range errs { if err != nil { return err } } return nil } func FirstKeyInErrorMap(errs map[string]error) string { for k, err := range errs { if err != nil { return k } } return "" } func NonNilErrorMapKeys(errs map[string]error) []string { var keys []string for k, err := range errs { if err != nil { keys = append(keys, k) } } return keys } ================================================ FILE: pkg/lib/errors/stack.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package errors import ( "fmt" "runtime" pkgerrors "github.com/pkg/errors" ) type stack []uintptr func callers() *stack { const depth = 32 var pcs [depth]uintptr n := runtime.Callers(3, pcs[:]) var st stack = pcs[0:n] return &st } func (s *stack) Format(st fmt.State, verb rune) { switch verb { case 'v': switch { case st.Flag('+'): for _, pc := range *s { f := pkgerrors.Frame(pc) fmt.Fprintf(st, "\n%+v", f) } } } } ================================================ FILE: pkg/lib/exit/exit.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package exit import ( "os" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/telemetry" ) func Ok() { telemetry.Close() os.Exit(0) } func Error(err error, wrapStrs ...string) { for _, str := range wrapStrs { err = errors.Wrap(err, str) } if err != nil && !errors.IsNoTelemetry(err) { telemetry.Error(err) } if err != nil && !errors.IsNoPrint(err) { errors.PrintErrorForUser(err) } telemetry.Close() os.Exit(1) } func Panic(err error, wrapStrs ...string) { for _, str := range wrapStrs { err = errors.Wrap(err, str) } if err != nil && !errors.IsNoTelemetry(err) { telemetry.Error(err) } telemetry.Close() panic(err) } func RecoverAndExit(strs ...string) { if errInterface := recover(); errInterface != nil { err := errors.CastRecoverError(errInterface, strs...) Panic(err) } } ================================================ FILE: pkg/lib/files/errors.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package files import ( "fmt" "github.com/cortexlabs/cortex/pkg/lib/errors" s "github.com/cortexlabs/cortex/pkg/lib/strings" ) const ( ErrCreateDir = "files.create_dir" ErrDeleteDir = "files.delete_dir" ErrReadFormFile = "files.read_form_file" ErrCreateFile = "files.create_file" ErrReadDir = "files.read_dir" ErrReadFile = "files.read_file" ErrFileAlreadyExists = "files.file_already_exists" ErrInsufficientMemoryToReadFile = "files.insufficient_memory_to_read_file" ErrFileSizeLimit = "files.file_size_limit" ErrProjectSizeLimit = "files.project_size_limit" ErrUnexpected = "files.unexpected" ErrFileDoesNotExist = "files.file_does_not_exist" ErrDirDoesNotExist = "files.dir_does_not_exist" ErrNotAFile = "files.not_a_file" ErrNotADir = "files.not_a_dir" ) func ErrorCreateDir(path string) error { return errors.WithStack(&errors.Error{ Kind: ErrCreateDir, Message: fmt.Sprintf("%s: unable to create directory", path), }) } func ErrorDeleteDir(path string) error { return errors.WithStack(&errors.Error{ Kind: ErrDeleteDir, Message: fmt.Sprintf("%s: unable to delete directory", path), }) } func ErrorReadFormFile(fileName string) error { return errors.WithStack(&errors.Error{ Kind: ErrReadFormFile, Message: fmt.Sprintf("unable to read request form file %s", s.UserStr(fileName)), }) } func ErrorCreateFile(path string) error { return errors.WithStack(&errors.Error{ Kind: ErrCreateFile, Message: fmt.Sprintf("%s: unable to create file", path), }) } func ErrorReadDir(path string) error { return errors.WithStack(&errors.Error{ Kind: ErrReadDir, Message: fmt.Sprintf("%s: unable to read directory", path), }) } func ErrorReadFile(path string) error { return errors.WithStack(&errors.Error{ Kind: ErrReadFile, Message: fmt.Sprintf("%s: unable to read file", path), }) } func ErrorFileAlreadyExists(path string) error { return errors.WithStack(&errors.Error{ Kind: ErrFileAlreadyExists, Message: fmt.Sprintf("%s: file already exists", path), }) } func ErrorInsufficientMemoryToReadFile(fileSizeBytes, availableMemBytes int64) error { return errors.WithStack(&errors.Error{ Kind: ErrInsufficientMemoryToReadFile, Message: fmt.Sprintf("unable to read file due to insufficient system memory; needs %s but is only allowed to use %s", s.Int64ToBase2Byte(fileSizeBytes), s.Int64ToBase2Byte(availableMemBytes)), }) } func ErrorFileSizeLimit(maxFileSizeBytes int64) error { return errors.WithStack(&errors.Error{ Kind: ErrFileSizeLimit, Message: fmt.Sprintf("file size cannot be greater than %s", s.Int64ToBase2Byte(maxFileSizeBytes)), }) } func ErrorProjectSizeLimit(maxProjectSizeBytes int64) error { return errors.WithStack(&errors.Error{ Kind: ErrProjectSizeLimit, Message: fmt.Sprintf("project size cannot exceed %s", s.Int64ToBase2Byte(maxProjectSizeBytes)), }) } func ErrorUnexpected() error { return errors.WithStack(&errors.Error{ Kind: ErrUnexpected, Message: "an unexpected error occurred", }) } func ErrorFileDoesNotExist(path string) error { return errors.WithStack(&errors.Error{ Kind: ErrFileDoesNotExist, Message: fmt.Sprintf("%s: file does not exist", path), }) } func ErrorDirDoesNotExist(path string) error { return errors.WithStack(&errors.Error{ Kind: ErrDirDoesNotExist, Message: fmt.Sprintf("%s: directory does not exist", path), }) } func ErrorNotAFile(path string) error { return errors.WithStack(&errors.Error{ Kind: ErrNotAFile, Message: fmt.Sprintf("%s: no such file", path), }) } func ErrorNotADir(path string) error { return errors.WithStack(&errors.Error{ Kind: ErrNotADir, Message: fmt.Sprintf("%s: not a directory path", path), }) } ================================================ FILE: pkg/lib/files/files.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package files import ( "bytes" "crypto/md5" "encoding/hex" "fmt" "io" "io/ioutil" "net/http" "os" "os/exec" "path" "path/filepath" "sort" "strings" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/prompt" "github.com/cortexlabs/cortex/pkg/lib/sets/strset" "github.com/cortexlabs/cortex/pkg/lib/slices" s "github.com/cortexlabs/cortex/pkg/lib/strings" "github.com/denormal/go-gitignore" "github.com/mitchellh/go-homedir" "github.com/shirou/gopsutil/mem" "github.com/xlab/treeprint" ) var ( _homeDir string ) // the returned file should be closed by the caller func Open(path string) (*os.File, error) { cleanPath, err := EscapeTilde(path) if err != nil { return nil, err } file, err := os.Open(cleanPath) if err != nil { return nil, errors.Wrap(err, errors.Message(ErrorReadFile(path))) } return file, nil } // the returned file should be closed by the caller func OpenFile(path string, flag int, perm os.FileMode) (*os.File, error) { cleanPath, err := EscapeTilde(path) if err != nil { return nil, err } file, err := os.OpenFile(cleanPath, flag, perm) if err != nil { return nil, errors.Wrap(err, errors.Message(ErrorCreateFile(path))) } return file, err } // the returned file should be closed by the caller func Create(path string) (*os.File, error) { cleanPath, err := EscapeTilde(path) if err != nil { return nil, err } file, err := os.Create(cleanPath) if err != nil { return nil, errors.Wrap(err, errors.Message(ErrorCreateFile(path))) } return file, nil } func ReadFile(path string) (string, error) { fileBytes, err := ReadFileBytes(path) if err != nil { return "", err } return string(fileBytes), nil } func ReadFileBytes(path string) ([]byte, error) { return ReadFileBytesErrPath(path, path) } func ReadFileBytesErrPath(path string, errMsgPath string) ([]byte, error) { path, err := EscapeTilde(path) if err != nil { return nil, err } if err := CheckFileErrPath(path, errMsgPath); err != nil { return nil, err } fileBytes, err := ioutil.ReadFile(path) if err != nil { return nil, errors.Wrap(err, errors.Message(ErrorReadFile(errMsgPath))) } return fileBytes, nil } func CreateFile(path string) error { cleanPath, err := EscapeTilde(path) if err != nil { return err } file, err := os.Create(cleanPath) if err != nil { return errors.Wrap(err, errors.Message(ErrorCreateFile(path))) } defer file.Close() return nil } func WriteFileFromReader(reader io.Reader, path string) error { cleanPath, err := EscapeTilde(path) if err != nil { return err } file, err := os.Create(cleanPath) if err != nil { return errors.Wrap(err, errors.Message(ErrorCreateFile(path))) } defer file.Close() _, err = io.Copy(file, reader) if err != nil { return errors.Wrap(err, errors.Message(ErrorCreateFile(path))) } return nil } func WriteFile(data []byte, path string) error { cleanPath, err := EscapeTilde(path) if err != nil { return err } if err := ioutil.WriteFile(cleanPath, data, 0664); err != nil { return errors.Wrap(err, errors.Message(ErrorCreateFile(path))) } return nil } func IsAbsOrTildePrefixed(path string) bool { return strings.HasPrefix(path, "/") || strings.HasPrefix(path, "~/") } // e.g. ~/path -> /home/ubuntu/path // returns original path if there was an error func EscapeTilde(path string) (string, error) { if !(path == "~" || strings.HasPrefix(path, "~/")) { return path, nil } if _homeDir == "" { homeDir, err := homedir.Dir() if err != nil { return path, err } if homeDir == "" || homeDir == "/" { return path, nil } _homeDir = homeDir } if path == "~" { return _homeDir, nil } // path starts with "~/" return filepath.Join(_homeDir, path[2:]), nil } // e.g. ~/path/../path2 -> /home/ubuntu/path2 // returns without escaping tilde if there was an error func Clean(path string) (string, error) { path, err := EscapeTilde(path) path = filepath.Clean(path) if err != nil { return path, err } return path, nil } // e.g. /home/ubuntu/path -> ~/path func ReplacePathWithTilde(absPath string) string { if !strings.HasPrefix(absPath, "/") { return absPath } if _homeDir == "" { homeDir, err := homedir.Dir() if err != nil || homeDir == "" || homeDir == "/" { return absPath } _homeDir = homeDir } trimmedHomeDir := strings.TrimSuffix(s.EnsurePrefix(_homeDir, "/"), "/") if strings.Index(absPath, trimmedHomeDir) == 0 { return "~" + absPath[len(trimmedHomeDir):] } return absPath } // e.g. /home/ubuntu/path -> / // or e.g. home/ubuntu/path -> home func GetTopLevelDirectory(path string) string { if strings.HasPrefix(path, "/") { return "/" } if path == "" { return "." } splitList := strings.Split(path, "/") if len(splitList) == 1 { return "." } return splitList[0] } func TrimDirPrefix(fullPath string, dirPath string) string { if !strings.HasSuffix(dirPath, "/") { dirPath = dirPath + "/" } return strings.TrimPrefix(fullPath, dirPath) } func RelToAbsPath(relativePath string, baseDir string) string { if !IsAbsOrTildePrefixed(relativePath) { relativePath = filepath.Join(baseDir, relativePath) } cleanPath, _ := Clean(relativePath) return cleanPath } func UserRelToAbsPath(relativePath string) string { cwd, err := os.Getwd() if err != nil { return relativePath } return RelToAbsPath(relativePath, cwd) } func PathRelativeToCWD(absPath string) string { cwd, err := os.Getwd() if err != nil { return absPath } return PathRelativeToDir(absPath, cwd) } func PathRelativeToDir(absPath string, dir string) string { if !IsAbsOrTildePrefixed(absPath) { return absPath } absPath, _ = EscapeTilde(absPath) dir, _ = EscapeTilde(dir) dir = s.EnsureSuffix(dir, "/") return strings.TrimPrefix(absPath, dir) } func DirPathRelativeToCWD(absPath string) string { return s.EnsureSuffix(PathRelativeToCWD(absPath), "/") } func DirPathRelativeToDir(absPath string, dir string) string { return s.EnsureSuffix(PathRelativeToDir(absPath, dir), "/") } func IsFileOrDir(path string) bool { path, _ = EscapeTilde(path) _, err := os.Stat(path) return err == nil } func IsDir(path string) bool { if err := CheckDir(path); err != nil { return false } return true } // CheckDir returns nil if the path is a directory func CheckDir(dirPath string) error { return CheckDirErrPath(dirPath, dirPath) } // CheckDir returns nil if the path is a directory func CheckDirErrPath(dirPath string, errMsgPath string) error { dirPath, err := EscapeTilde(dirPath) if err != nil { return err } fileInfo, err := os.Stat(dirPath) if err != nil { return errors.Wrap(err, errors.Message(ErrorDirDoesNotExist(errMsgPath))) } if !fileInfo.IsDir() { return ErrorNotADir(errMsgPath) } return nil } func IsFile(path string) bool { if err := CheckFile(path); err != nil { return false } return true } // CheckFile returns nil if the path is a file func CheckFile(path string) error { return CheckFileErrPath(path, path) } // CheckFile returns nil if the path is a file func CheckFileErrPath(path string, errMsgPath string) error { path, err := EscapeTilde(path) if err != nil { return err } fileInfo, err := os.Stat(path) if err != nil { return ErrorFileDoesNotExist(errMsgPath) } if fileInfo.IsDir() { return ErrorNotAFile(errMsgPath) } return nil } func CreateDir(path string) error { cleanPath, err := EscapeTilde(path) if err != nil { return err } if err := os.MkdirAll(cleanPath, os.ModePerm); err != nil { return errors.Wrap(err, errors.Message(ErrorCreateDir(path))) } return nil } func CreateDirIfMissing(path string) (bool, error) { if IsDir(path) { return false, nil } if IsFile(path) { return false, ErrorFileAlreadyExists(path) } err := CreateDir(path) if err != nil { return false, err } return true, nil } func DeleteDir(path string) error { cleanPath, err := EscapeTilde(path) if err != nil { return err } if err := os.RemoveAll(cleanPath); err != nil { return errors.Wrap(err, errors.Message(ErrorDeleteDir(path))) } return nil } func DeleteDirIfPresent(path string) (bool, error) { if IsFile(path) { return false, ErrorNotADir(path) } if !IsDir(path) { return false, nil } err := DeleteDir(path) if err != nil { return false, err } return true, nil } func TmpDir() (string, error) { filePath, err := ioutil.TempDir("", "") if err != nil { return "", errors.Wrap(err) } return filePath, nil } func ParentDir(dir string) string { return filepath.Clean(filepath.Join(dir, "..")) } func SearchForFile(filename string, dir string) (string, error) { dir, err := Clean(dir) if err != nil { return "", err } for true { files, err := ioutil.ReadDir(dir) if err != nil { return "", errors.Wrap(err, errors.Message(ErrorReadDir(dir))) } for _, file := range files { if file.Name() == filename { return filepath.Join(dir, filename), nil } } if dir == "/" { return "", nil } dir = ParentDir(dir) } return "", ErrorUnexpected() } func MakeEmptyFile(path string) error { cleanPath, err := Clean(path) if err != nil { return err } err = os.MkdirAll(filepath.Dir(cleanPath), os.ModePerm) if err != nil { return errors.Wrap(err, errors.Message(ErrorCreateDir(filepath.Dir(path)))) } f, err := os.OpenFile(cleanPath, os.O_RDONLY|os.O_CREATE, 0666) if err != nil { return errors.Wrap(err, errors.Message(ErrorCreateFile(path))) } defer f.Close() return nil } func MakeEmptyFiles(path string, paths ...string) error { allPaths := append(paths, path) for _, path := range allPaths { if err := MakeEmptyFile(path); err != nil { return err } } return nil } func MakeEmptyFilesInDir(dir string, path string, paths ...string) error { allPaths := append(paths, path) for _, path := range allPaths { fullPath := filepath.Join(dir, path) if err := MakeEmptyFile(fullPath); err != nil { return err } } return nil } func IsFilePathYAML(path string) bool { ext := filepath.Ext(path) return ext == ".yaml" || ext == ".yml" } func IsFilePathPython(path string) bool { ext := filepath.Ext(path) return ext == ".py" } // IgnoreFn if passed a dir, returning true will ignore all subdirs of dir type IgnoreFn func(string, os.FileInfo) (bool, error) func IgnoreHiddenFiles(path string, fi os.FileInfo) (bool, error) { if !fi.IsDir() && strings.HasPrefix(fi.Name(), ".") { return true, nil } return false, nil } func IgnoreCortexYAML(path string, fi os.FileInfo) (bool, error) { if !fi.IsDir() && fi.Name() == "cortex.yaml" { return true, nil } return false, nil } func IgnoreCortexDebug(path string, fi os.FileInfo) (bool, error) { if strings.HasPrefix(fi.Name(), "cortex-debug-") { return true, nil } return false, nil } func IgnoreHiddenFolders(path string, fi os.FileInfo) (bool, error) { if fi.IsDir() && strings.HasPrefix(fi.Name(), ".") { return true, nil } return false, nil } func IgnorePythonGeneratedFiles(path string, fi os.FileInfo) (bool, error) { if fi.IsDir() && fi.Name() == "__pycache__" { return true, nil } if !fi.IsDir() { ext := filepath.Ext(path) return ext == ".pyc" || ext == ".pyo" || ext == ".pyd", nil } return false, nil } func IgnoreNonPython(path string, fi os.FileInfo) (bool, error) { if !fi.IsDir() && !IsFilePathPython(path) { return true, nil } return false, nil } func IgnoreNonYAML(path string, fi os.FileInfo) (bool, error) { if !fi.IsDir() && !IsFilePathYAML(path) { return true, nil } return false, nil } func IgnoreSpecificFiles(absPaths ...string) IgnoreFn { absPathsSet := strset.New(absPaths...) return func(path string, fi os.FileInfo) (bool, error) { return absPathsSet.Has(path), nil } } func GitIgnoreFn(gitIgnorePath string) (IgnoreFn, error) { gitIgnoreDir := filepath.Dir(gitIgnorePath) ignore, err := gitignore.NewFromFile(gitIgnorePath) if err != nil { return nil, errors.Wrap(err, gitIgnorePath) } return func(path string, fi os.FileInfo) (bool, error) { if path == gitIgnoreDir { // This is to avoid a bug in ignore.Ignore() return false, nil } return ignore.Ignore(path), nil }, nil } // promptMsgTemplate should have two placeholders: the first is for the file path and the second is for the file size func PromptForFilesAboveSize(size int, promptMsgTemplate string) IgnoreFn { if promptMsgTemplate == "" { promptMsgTemplate = "do you want to zip %s (%s)?" } return func(path string, fi os.FileInfo) (bool, error) { if !fi.IsDir() && fi.Size() > int64(size) { promptMsg := fmt.Sprintf(promptMsgTemplate, PathRelativeToCWD(path), s.Int64ToBase2Byte(fi.Size())) return !prompt.YesOrNo(promptMsg, "", ""), nil } return false, nil } } func ErrorOnBigFilesFn(maxFileSizeBytes int64) IgnoreFn { return func(path string, fi os.FileInfo) (bool, error) { if !fi.IsDir() { fileSizeBytes := fi.Size() virtual, _ := mem.VirtualMemory() if fileSizeBytes > int64(virtual.Available) { return false, errors.Wrap( ErrorInsufficientMemoryToReadFile(fileSizeBytes, int64(virtual.Available)), path, ) } if fileSizeBytes > maxFileSizeBytes { return false, errors.Wrap(ErrorFileSizeLimit(maxFileSizeBytes), path) } } return false, nil } } func ErrorOnProjectSizeLimit(maxProjectSizeBytes int64) IgnoreFn { filesSizeSum := int64(0) return func(path string, fi os.FileInfo) (bool, error) { if !fi.IsDir() { filesSizeSum += fi.Size() if filesSizeSum > maxProjectSizeBytes { return false, errors.Wrap(ErrorProjectSizeLimit(maxProjectSizeBytes), path) } } return false, nil } } // Retrieves the longest common path given a list of paths. func LongestCommonPath(paths ...string) string { // Handle special cases. switch len(paths) { case 0: return "" case 1: return path.Clean(paths[0]) } startsWithSlash := false allStartWithSlash := true var splitPaths [][]string shortestPathLength := -1 for _, path := range paths { if strings.HasPrefix(path, "/") { startsWithSlash = true } else { allStartWithSlash = false } splitPath := slices.RemoveEmpties(strings.Split(path, "/")) splitPaths = append(splitPaths, splitPath) if len(splitPath) < shortestPathLength || shortestPathLength == -1 { shortestPathLength = len(splitPath) } } commonPath := "" numPaths := len(splitPaths) for level := 0; level < shortestPathLength; level++ { element := splitPaths[0][level] counter := 1 for _, splitPath := range splitPaths[1:] { if splitPath[level] != element { break } counter++ } if counter != numPaths { break } commonPath = filepath.Join(commonPath, element) } if commonPath != "" && startsWithSlash { commonPath = s.EnsurePrefix(commonPath, "/") commonPath = s.EnsureSuffix(commonPath, "/") } if commonPath == "" && allStartWithSlash { return "/" } return commonPath } func FilterPathsWithDirPrefix(paths []string, prefix string) []string { prefix = s.EnsureSuffix(prefix, "/") var filteredPaths []string for _, path := range paths { if strings.HasPrefix(path, prefix) { filteredPaths = append(filteredPaths, path) } } return filteredPaths } type DirsOrder string var DirsSorted DirsOrder = "sorted" var DirsOnTop DirsOrder = "top" var DirsOnBottom DirsOrder = "bottom" func SortFilePaths(paths []string, dirsOrder DirsOrder) []string { if dirsOrder == DirsSorted { sort.Strings(paths) return paths } dirsSortChar := "" if dirsOrder == DirsOnTop { dirsSortChar = " " } if dirsOrder == DirsOnBottom { dirsSortChar = "|" } for i, path := range paths { dirPath := filepath.Dir(path) if dirPath == "." || dirPath == "/" { continue } replacedDir := strings.Replace(dirPath, "/", "/"+dirsSortChar, -1) paths[i] = dirsSortChar + replacedDir + "/" + filepath.Base(path) } sort.Strings(paths) for i, path := range paths { paths[i] = strings.Replace(path, dirsSortChar, "", -1) } return paths } func FileTree(paths []string, cwd string, dirsOrder DirsOrder) string { if len(paths) == 0 { return "." } paths = SortFilePaths(paths, dirsOrder) dirPaths := DirPaths(paths, true) didTrimCwd := false if cwd != "" { cwd = s.EnsureSuffix(cwd, "/") paths, didTrimCwd = s.TrimPrefixIfPresentInAll(paths, cwd) dirPaths = DirPaths(paths, true) } commonPrefix := LongestCommonPath(dirPaths...) paths, _ = s.TrimPrefixIfPresentInAll(paths, commonPrefix) var header string if didTrimCwd && commonPrefix == "" { header = ".\n" } else if !didTrimCwd && commonPrefix == "" { header = "" } else if didTrimCwd && commonPrefix != "" { header = "./" + commonPrefix header = s.EnsureSingleOccurrenceCharSuffix(header, "/") + "\n" } else if !didTrimCwd && commonPrefix != "" { header = commonPrefix + "/" header = s.EnsureSingleOccurrenceCharSuffix(header, "/") + "\n" } tree := treeprint.New() cachedTrees := make(map[string]treeprint.Tree) cachedTrees["."] = tree cachedTrees["/"] = tree cachedTrees[""] = tree for _, path := range paths { dir := filepath.Dir(path) branch := getTreeBranch(dir, cachedTrees) branch.AddNode(filepath.Base(path)) } treeStr := tree.String() return header + treeStr[2:] } func getTreeBranch(dir string, cachedTrees map[string]treeprint.Tree) treeprint.Tree { dir = s.TrimPrefixAndSuffix(dir, "/") if cachedTree, ok := cachedTrees[dir]; ok { return cachedTree } var parentDir, lastDir string lastIndex := strings.LastIndex(dir, "/") if lastIndex == -1 { parentDir = "." lastDir = dir } else { parentDir = s.TrimPrefixAndSuffix(dir[:lastIndex], "/") lastDir = s.TrimPrefixAndSuffix(dir[lastIndex:], "/") } parentBranch := getTreeBranch(parentDir, cachedTrees) branch := parentBranch.AddBranch(lastDir) cachedTrees[dir] = branch return branch } // Return the path to the directory containing the provided path (with a trailing slash) func Dir(path string) string { return s.EnsureSuffix(filepath.Dir(path), "/") } func DirPaths(paths []string, addTrailingSlash bool) []string { suffix := "" if addTrailingSlash { suffix = "/" } dirPaths := make([]string, len(paths)) for i, path := range paths { dirPaths[i] = s.EnsureSuffix(filepath.Dir(path), suffix) } return dirPaths } func ListDirRecursive(dir string, relative bool, ignoreFns ...IgnoreFn) ([]string, error) { cleanDir, err := Clean(dir) if err != nil { return nil, err } cleanDir = strings.TrimSuffix(cleanDir, "/") if err := CheckDir(cleanDir); err != nil { return nil, err } var fileList []string walkErr := filepath.Walk(cleanDir, func(path string, fi os.FileInfo, err error) error { if err != nil { return errors.Wrap(err, path) } for _, ignoreFn := range ignoreFns { ignore, err := ignoreFn(path, fi) if err != nil { return err } if ignore { if fi.IsDir() { return filepath.SkipDir } return nil } } if !fi.IsDir() { if relative && dir != "." { path = path[len(cleanDir)+1:] } fileList = append(fileList, path) } return nil }) if walkErr != nil { return nil, walkErr } return fileList, nil } func ListDir(dir string, relative bool) ([]string, error) { cleanDir, err := Clean(dir) if err != nil { return nil, err } cleanDir = strings.TrimSuffix(cleanDir, "/") if err := CheckDir(cleanDir); err != nil { return nil, err } var filenames []string fileInfo, err := ioutil.ReadDir(cleanDir) if err != nil { return nil, errors.Wrap(err, errors.Message(ErrorReadDir(dir))) } for _, file := range fileInfo { filename := file.Name() if !relative { filename = filepath.Join(cleanDir, filename) } filenames = append(filenames, filename) } return filenames, nil } func CopyFileOverwrite(src string, dest string) error { srcFile, err := Open(src) if err != nil { return err } defer srcFile.Close() destFile, err := Create(dest) if err != nil { return err } defer destFile.Close() if _, err = io.Copy(destFile, srcFile); err != nil { return err } return nil } func CopyDirOverwrite(src string, dest string, ignoreFns ...IgnoreFn) error { srcRelFilePaths, err := ListDirRecursive(src, true, ignoreFns...) if err != nil { return err } if _, err := CreateDirIfMissing(dest); err != nil { return err } createdDirs := strset.New(dest) for _, srcRelFilePath := range srcRelFilePaths { srcFilePath := filepath.Join(src, srcRelFilePath) destFilePath := filepath.Join(dest, srcRelFilePath) destFileDir := filepath.Dir(destFilePath) if !createdDirs.Has(destFileDir) { if _, err := CreateDirIfMissing(destFileDir); err != nil { return err } createdDirs.Add(destFileDir) } if err := CopyFileOverwrite(srcFilePath, destFilePath); err != nil { return err } } return nil } func CopyRecursiveShell(src string, dest string) error { cleanSrc, err := EscapeTilde(src) if err != nil { return err } cleanDest, err := EscapeTilde(dest) if err != nil { return err } cmd := exec.Command("cp", "-r", cleanSrc, cleanDest) var out bytes.Buffer cmd.Stdout = &out cmd.Stderr = &out if err := cmd.Run(); err != nil { return errors.Wrap(err, strings.TrimSpace(out.String())) } return nil } func HashFile(path string, paths ...string) (string, error) { md5Hash := md5.New() allPaths := append(paths, path) for _, path := range allPaths { f, err := Open(path) if err != nil { return "", err } // Skip directories fileInfo, err := f.Stat() if err != nil { f.Close() return "", errors.WithStack(err) } if fileInfo.IsDir() { f.Close() continue } if _, err := io.Copy(md5Hash, f); err != nil { f.Close() return "", errors.Wrap(err, path) } io.WriteString(md5Hash, path) f.Close() } return hex.EncodeToString(md5Hash.Sum(nil)), nil } func HashDirectory(dir string, ignoreFns ...IgnoreFn) (string, error) { md5Hash := md5.New() walkErr := filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error { if err != nil { return errors.Wrap(err, path) } for _, ignoreFn := range ignoreFns { ignore, err := ignoreFn(path, fi) if err != nil { return errors.Wrap(err, path) } if ignore { if fi.IsDir() { return filepath.SkipDir } return nil } } if fi.IsDir() { return nil } f, err := os.Open(path) if err != nil { return errors.Wrap(err, path) } defer f.Close() if _, err := io.Copy(md5Hash, f); err != nil { return errors.Wrap(err, path) } io.WriteString(md5Hash, path) return nil }) if walkErr != nil { return "", walkErr } return hex.EncodeToString(md5Hash.Sum(nil)), nil } func CloseSilent(closer io.Closer, closers ...io.Closer) { allClosers := append(closers, closer) for _, closer := range allClosers { closer.Close() } } // ReadReqFile returns nil if no file func ReadReqFile(r *http.Request, fileName string) ([]byte, error) { mpFile, _, err := r.FormFile(fileName) if err != nil { if strings.Contains(errors.Message(err), "no such file") { return nil, nil } return nil, errors.Wrap(err, errors.Message(ErrorReadFormFile(fileName))) } defer mpFile.Close() fileBytes, err := ioutil.ReadAll(mpFile) if err != nil { return nil, errors.Wrap(err, errors.Message(ErrorReadFormFile(fileName))) } return fileBytes, nil } ================================================ FILE: pkg/lib/files/files_test.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package files import ( "os" "path/filepath" "testing" "github.com/stretchr/testify/require" ) func IgnoreDir3(path string, fi os.FileInfo) (bool, error) { if fi.IsDir() && fi.Name() == "3" { return true, nil } return false, nil } func TestPrintFileTree(t *testing.T) { var filesList []string var cwd string var expectedTree string var expectedHeader string filesList = []string{ "/1/2/3.txt", "/1/2/3/5.txt", "/1/2/4/5/6.txt", "/1/2/3/4.txt", } expectedTree = ` ├── 3.txt ├── 3 │   ├── 4.txt │   └── 5.txt └── 4 └── 5 └── 6.txt ` cwd = "" expectedHeader = "/1/2/" require.Equal(t, expectedHeader+expectedTree, FileTree(filesList, cwd, DirsSorted)) cwd = "/missing" expectedHeader = "/1/2/" require.Equal(t, expectedHeader+expectedTree, FileTree(filesList, cwd, DirsSorted)) cwd = "/1/2" expectedHeader = "." require.Equal(t, expectedHeader+expectedTree, FileTree(filesList, cwd, DirsSorted)) cwd = "/1/2/" expectedHeader = "." require.Equal(t, expectedHeader+expectedTree, FileTree(filesList, cwd, DirsSorted)) cwd = "/1" expectedHeader = "./2/" require.Equal(t, expectedHeader+expectedTree, FileTree(filesList, cwd, DirsSorted)) filesList = []string{ "/1", "/2", "/1/1", "/1/2", "/2/1", "/2/2", "/1/1/1", "/1/1/2", "/1/2/1", "/1/2/2", "/2/1/1", "/2/1/2", "/2/2/1", "/2/2/2", } expectedTree = ` ├── 1 ├── 1 │   ├── 1 │   ├── 1 │   │   ├── 1 │   │   └── 2 │   ├── 2 │   └── 2 │   ├── 1 │   └── 2 ├── 2 └── 2 ├── 1 ├── 1 │   ├── 1 │   └── 2 ├── 2 └── 2 ├── 1 └── 2 ` require.Equal(t, "/"+expectedTree, FileTree(filesList, cwd, DirsSorted)) expectedTree = ` ├── 1 ├── 2 ├── 1 │   ├── 1 │   ├── 2 │   ├── 1 │   │   ├── 1 │   │   └── 2 │   └── 2 │   ├── 1 │   └── 2 └── 2 ├── 1 ├── 2 ├── 1 │   ├── 1 │   └── 2 └── 2 ├── 1 └── 2 ` require.Equal(t, "/"+expectedTree, FileTree(filesList, cwd, DirsOnBottom)) expectedTree = ` ├── 1 │   ├── 1 │   │   ├── 1 │   │   └── 2 │   ├── 2 │   │   ├── 1 │   │   └── 2 │   ├── 1 │   └── 2 ├── 2 │   ├── 1 │   │   ├── 1 │   │   └── 2 │   ├── 2 │   │   ├── 1 │   │   └── 2 │   ├── 1 │   └── 2 ├── 1 └── 2 ` require.Equal(t, "/"+expectedTree, FileTree(filesList, cwd, DirsOnTop)) } func TestListDirRecursive(t *testing.T) { _, err := ListDirRecursive("/home/path/to/fake/dir", false) require.Error(t, err) tmpDir, err := TmpDir() defer os.RemoveAll(tmpDir) require.NoError(t, err) filesList := []string{ filepath.Join(tmpDir, "1.txt"), filepath.Join(tmpDir, "2.py"), filepath.Join(tmpDir, "3/1.py"), filepath.Join(tmpDir, "3/2/1.py"), filepath.Join(tmpDir, "3/2/2.txt"), filepath.Join(tmpDir, "3/2/4.md"), filepath.Join(tmpDir, "3/2/3/.tmp"), filepath.Join(tmpDir, "4/1.yaml"), filepath.Join(tmpDir, "4/2.pyc"), filepath.Join(tmpDir, "4/3.md"), filepath.Join(tmpDir, "4/.git/HEAD"), filepath.Join(tmpDir, "README.md"), filepath.Join(tmpDir, ".ignore"), } ignoreContents := ` *.md *.txt 4/.git 3/2/1.py !README.md ` WriteFile([]byte(ignoreContents), filepath.Join(tmpDir, ".ignore")) ignoreFn, err := GitIgnoreFn(filepath.Join(tmpDir, ".ignore")) require.NoError(t, err) err = MakeEmptyFiles(filesList[0], filesList[1:]...) require.NoError(t, err) var filesListRecursive []string var expected []string filesListRecursive, err = ListDirRecursive(tmpDir, false) require.NoError(t, err) require.ElementsMatch(t, filesList, filesListRecursive) filesListRecursive, err = ListDirRecursive(tmpDir, false, IgnoreHiddenFiles) expected = []string{ filepath.Join(tmpDir, "1.txt"), filepath.Join(tmpDir, "2.py"), filepath.Join(tmpDir, "3/1.py"), filepath.Join(tmpDir, "3/2/1.py"), filepath.Join(tmpDir, "3/2/2.txt"), filepath.Join(tmpDir, "3/2/4.md"), filepath.Join(tmpDir, "4/1.yaml"), filepath.Join(tmpDir, "4/2.pyc"), filepath.Join(tmpDir, "4/3.md"), filepath.Join(tmpDir, "4/.git/HEAD"), filepath.Join(tmpDir, "README.md"), } require.NoError(t, err) require.ElementsMatch(t, expected, filesListRecursive) filesListRecursive, err = ListDirRecursive(tmpDir, false, IgnoreHiddenFiles, IgnoreHiddenFolders) expected = []string{ filepath.Join(tmpDir, "1.txt"), filepath.Join(tmpDir, "2.py"), filepath.Join(tmpDir, "3/1.py"), filepath.Join(tmpDir, "3/2/1.py"), filepath.Join(tmpDir, "3/2/2.txt"), filepath.Join(tmpDir, "3/2/4.md"), filepath.Join(tmpDir, "4/1.yaml"), filepath.Join(tmpDir, "4/2.pyc"), filepath.Join(tmpDir, "4/3.md"), filepath.Join(tmpDir, "README.md"), } require.NoError(t, err) require.ElementsMatch(t, expected, filesListRecursive) filesListRecursive, err = ListDirRecursive(tmpDir, false, IgnoreHiddenFiles, IgnoreDir3, IgnorePythonGeneratedFiles) expected = []string{ filepath.Join(tmpDir, "1.txt"), filepath.Join(tmpDir, "2.py"), filepath.Join(tmpDir, "4/1.yaml"), filepath.Join(tmpDir, "4/3.md"), filepath.Join(tmpDir, "4/.git/HEAD"), filepath.Join(tmpDir, "README.md"), } require.NoError(t, err) require.ElementsMatch(t, expected, filesListRecursive) filesListRecursive, err = ListDirRecursive(tmpDir, false, IgnoreNonPython) expected = []string{ filepath.Join(tmpDir, "2.py"), filepath.Join(tmpDir, "3/1.py"), filepath.Join(tmpDir, "3/2/1.py"), } require.NoError(t, err) require.ElementsMatch(t, expected, filesListRecursive) filesListRecursive, err = ListDirRecursive(tmpDir, true, IgnoreNonPython) expected = []string{ filepath.Join("2.py"), filepath.Join("3/1.py"), filepath.Join("3/2/1.py"), } require.NoError(t, err) require.ElementsMatch(t, expected, filesListRecursive) filesListRecursive, err = ListDirRecursive(tmpDir, true, ignoreFn) expected = []string{ "2.py", "3/1.py", "3/2/3/.tmp", "4/1.yaml", "4/2.pyc", "README.md", ".ignore", } require.NoError(t, err) require.ElementsMatch(t, expected, filesListRecursive) } ================================================ FILE: pkg/lib/hash/hash.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package hash import ( "crypto/sha256" "encoding/hex" "strings" "github.com/cortexlabs/cortex/pkg/lib/files" s "github.com/cortexlabs/cortex/pkg/lib/strings" ) // Bytes will trim to 63 characters because e.g. K8s labels must be < 64 func Bytes(bytes []byte) string { hash := sha256.New() hash.Write(bytes) str := hex.EncodeToString(hash.Sum(nil)) return str[:63] } func String(str string) string { return Bytes([]byte(str)) } func Strings(strs ...string) string { return String(strings.Join(strs, ",")) } func Any(obj interface{}) string { return String(s.Obj(obj)) } func File(path string) (string, error) { fileBytes, err := files.ReadFileBytes(path) if err != nil { return "", err } return Bytes(fileBytes), nil } ================================================ FILE: pkg/lib/json/errors.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package json const ( errStrMarshalJSON = "invalid json" errStrUnmarshalJSON = "invalid json cannot be serialized" ) ================================================ FILE: pkg/lib/json/json.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package json import ( "bytes" "encoding/json" "path/filepath" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/files" ) func MarshalIndent(obj interface{}) ([]byte, error) { jsonBytes, err := json.MarshalIndent(obj, "", " ") if err != nil { return nil, errors.Wrap(err, errStrMarshalJSON) } return jsonBytes, nil } func Marshal(obj interface{}) ([]byte, error) { jsonBytes, err := json.Marshal(obj) if err != nil { return nil, errors.Wrap(err, errStrMarshalJSON) } return jsonBytes, nil } func Unmarshal(jsonBytes []byte, dst interface{}) error { if err := json.Unmarshal(jsonBytes, dst); err != nil { return errors.Wrap(err, errStrUnmarshalJSON) } return nil } func DecodeWithNumber(jsonBytes []byte, dst interface{}) error { d := json.NewDecoder(bytes.NewReader(jsonBytes)) d.UseNumber() if err := d.Decode(&dst); err != nil { return errors.Wrap(err, errStrUnmarshalJSON) } return nil } func MarshalJSONStr(obj interface{}) (string, error) { jsonBytes, err := Marshal(obj) if err != nil { return "", err } return string(jsonBytes), nil } func WriteJSON(obj interface{}, outPath string) error { jsonBytes, err := Marshal(obj) if err != nil { return err } if err := files.CreateDir(filepath.Dir(outPath)); err != nil { return err } if err := files.WriteFile(jsonBytes, outPath); err != nil { return err } return nil } func Pretty(obj interface{}) (string, error) { b, err := json.MarshalIndent(obj, "", " ") if err != nil { return "", errors.Wrap(err, errStrMarshalJSON) } return string(b), nil } ================================================ FILE: pkg/lib/k8s/configmap.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package k8s import ( "context" "github.com/cortexlabs/cortex/pkg/lib/errors" kcore "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" kmeta "k8s.io/apimachinery/pkg/apis/meta/v1" klabels "k8s.io/apimachinery/pkg/labels" ) var _configMapTypeMeta = kmeta.TypeMeta{ APIVersion: "v1", Kind: "ConfigMap", } type ConfigMapSpec struct { Name string Data map[string]string // Data and BinaryData must not have overlapping keys BinaryData map[string][]byte // Data and BinaryData must not have overlapping keys Labels map[string]string Annotations map[string]string } func ConfigMap(spec *ConfigMapSpec) *kcore.ConfigMap { configMap := &kcore.ConfigMap{ TypeMeta: _configMapTypeMeta, ObjectMeta: kmeta.ObjectMeta{ Name: spec.Name, Labels: spec.Labels, Annotations: spec.Annotations, }, Data: spec.Data, BinaryData: spec.BinaryData, } return configMap } func (c *Client) CreateConfigMap(configMap *kcore.ConfigMap) (*kcore.ConfigMap, error) { configMap.TypeMeta = _configMapTypeMeta configMap, err := c.configMapClient.Create(context.Background(), configMap, kmeta.CreateOptions{}) if err != nil { return nil, errors.WithStack(err) } return configMap, nil } func (c *Client) UpdateConfigMap(configMap *kcore.ConfigMap) (*kcore.ConfigMap, error) { configMap.TypeMeta = _configMapTypeMeta configMap, err := c.configMapClient.Update(context.Background(), configMap, kmeta.UpdateOptions{}) if err != nil { return nil, errors.WithStack(err) } return configMap, nil } func (c *Client) ApplyConfigMap(configMap *kcore.ConfigMap) (*kcore.ConfigMap, error) { existing, err := c.GetConfigMap(configMap.Name) if err != nil { return nil, err } if existing == nil { return c.CreateConfigMap(configMap) } return c.UpdateConfigMap(configMap) } func (c *Client) GetConfigMap(name string) (*kcore.ConfigMap, error) { configMap, err := c.configMapClient.Get(context.Background(), name, kmeta.GetOptions{}) if err != nil { if kerrors.IsNotFound(err) { return nil, nil } return nil, errors.WithStack(err) } configMap.TypeMeta = _configMapTypeMeta return configMap, nil } func (c *Client) GetConfigMapData(name string) (map[string]string, map[string][]byte, error) { configMap, err := c.GetConfigMap(name) if err != nil { return nil, nil, err } if configMap == nil { return nil, nil, nil } return configMap.Data, configMap.BinaryData, nil } func (c *Client) DeleteConfigMap(name string) (bool, error) { err := c.configMapClient.Delete(context.Background(), name, _deleteOpts) if err != nil { if kerrors.IsNotFound(err) { return false, nil } return false, errors.WithStack(err) } return true, nil } func (c *Client) ListConfigMaps(opts *kmeta.ListOptions) ([]kcore.ConfigMap, error) { if opts == nil { opts = &kmeta.ListOptions{} } configMapList, err := c.configMapClient.List(context.Background(), *opts) if err != nil { return nil, errors.WithStack(err) } for i := range configMapList.Items { configMapList.Items[i].TypeMeta = _configMapTypeMeta } return configMapList.Items, nil } func (c *Client) ListConfigMapsByLabels(labels map[string]string) ([]kcore.ConfigMap, error) { opts := &kmeta.ListOptions{ LabelSelector: klabels.SelectorFromSet(labels).String(), } return c.ListConfigMaps(opts) } func (c *Client) ListConfigMapsByLabel(labelKey string, labelValue string) ([]kcore.ConfigMap, error) { return c.ListConfigMapsByLabels(map[string]string{labelKey: labelValue}) } func (c *Client) ListConfigMapsWithLabelKeys(labelKeys ...string) ([]kcore.ConfigMap, error) { opts := &kmeta.ListOptions{ LabelSelector: LabelExistsSelector(labelKeys...), } return c.ListConfigMaps(opts) } func ConfigMapMap(configMaps []kcore.ConfigMap) map[string]kcore.ConfigMap { configMapMap := map[string]kcore.ConfigMap{} for _, configMap := range configMaps { configMapMap[configMap.Name] = configMap } return configMapMap } ================================================ FILE: pkg/lib/k8s/deployment.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package k8s import ( "context" "time" "github.com/cortexlabs/cortex/pkg/lib/errors" kapps "k8s.io/api/apps/v1" kcore "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" kmeta "k8s.io/apimachinery/pkg/apis/meta/v1" klabels "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/intstr" ) var _deploymentTypeMeta = kmeta.TypeMeta{ APIVersion: "apps/v1", Kind: "Deployment", } type DeploymentSpec struct { Name string Replicas int32 PodSpec PodSpec MaxSurge *string // Can be a percentage (e.g. 10%) or an absolute number (e.g. 2) MaxUnavailable *string // Can be a percentage (e.g. 10%) or an absolute number (e.g. 2) Selector map[string]string Labels map[string]string Annotations map[string]string } func Deployment(spec *DeploymentSpec) *kapps.Deployment { if spec.PodSpec.Name == "" { spec.PodSpec.Name = spec.Name } if spec.Selector == nil { spec.Selector = spec.PodSpec.Labels } var maxSurge *intstr.IntOrString if spec.MaxSurge != nil { intStr := intstr.Parse(*spec.MaxSurge) maxSurge = &intStr } var maxUnavailable *intstr.IntOrString if spec.MaxUnavailable != nil { intStr := intstr.Parse(*spec.MaxUnavailable) maxUnavailable = &intStr } deployment := &kapps.Deployment{ TypeMeta: _deploymentTypeMeta, ObjectMeta: kmeta.ObjectMeta{ Name: spec.Name, Labels: spec.Labels, Annotations: spec.Annotations, }, Spec: kapps.DeploymentSpec{ Replicas: &spec.Replicas, Strategy: kapps.DeploymentStrategy{ Type: kapps.RollingUpdateDeploymentStrategyType, RollingUpdate: &kapps.RollingUpdateDeployment{ MaxSurge: maxSurge, MaxUnavailable: maxUnavailable, }, }, Template: kcore.PodTemplateSpec{ ObjectMeta: kmeta.ObjectMeta{ Name: spec.PodSpec.Name, Labels: spec.PodSpec.Labels, Annotations: spec.PodSpec.Annotations, }, Spec: spec.PodSpec.K8sPodSpec, }, Selector: &kmeta.LabelSelector{ MatchLabels: spec.Selector, }, }, } return deployment } func (c *Client) CreateDeployment(deployment *kapps.Deployment) (*kapps.Deployment, error) { deployment.TypeMeta = _deploymentTypeMeta deployment, err := c.deploymentClient.Create(context.Background(), deployment, kmeta.CreateOptions{}) if err != nil { return nil, errors.WithStack(err) } return deployment, nil } func (c *Client) UpdateDeployment(deployment *kapps.Deployment) (*kapps.Deployment, error) { deployment.TypeMeta = _deploymentTypeMeta deployment, err := c.deploymentClient.Update(context.Background(), deployment, kmeta.UpdateOptions{}) if err != nil { return nil, errors.WithStack(err) } return deployment, nil } func (c *Client) ApplyDeployment(deployment *kapps.Deployment) (*kapps.Deployment, error) { existing, err := c.GetDeployment(deployment.Name) if err != nil { return nil, err } if existing == nil { return c.CreateDeployment(deployment) } return c.UpdateDeployment(deployment) } func (c *Client) GetDeployment(name string) (*kapps.Deployment, error) { deployment, err := c.deploymentClient.Get(context.Background(), name, kmeta.GetOptions{}) if err != nil { if kerrors.IsNotFound(err) { return nil, nil } return nil, errors.WithStack(err) } deployment.TypeMeta = _deploymentTypeMeta return deployment, nil } func (c *Client) DeleteDeployment(name string) (bool, error) { err := c.deploymentClient.Delete(context.Background(), name, _deleteOpts) if err != nil { if kerrors.IsNotFound(err) { return false, nil } return false, errors.WithStack(err) } return true, nil } func (c *Client) ListDeployments(opts *kmeta.ListOptions) ([]kapps.Deployment, error) { if opts == nil { opts = &kmeta.ListOptions{} } deploymentList, err := c.deploymentClient.List(context.Background(), *opts) if err != nil { return nil, errors.WithStack(err) } for i := range deploymentList.Items { deploymentList.Items[i].TypeMeta = _deploymentTypeMeta } return deploymentList.Items, nil } func (c *Client) ListDeploymentsByLabels(labels map[string]string) ([]kapps.Deployment, error) { opts := &kmeta.ListOptions{ LabelSelector: klabels.SelectorFromSet(labels).String(), } return c.ListDeployments(opts) } func (c *Client) ListDeploymentsByLabel(labelKey string, labelValue string) ([]kapps.Deployment, error) { return c.ListDeploymentsByLabels(map[string]string{labelKey: labelValue}) } func (c *Client) ListDeploymentsWithLabelKeys(labelKeys ...string) ([]kapps.Deployment, error) { opts := &kmeta.ListOptions{ LabelSelector: LabelExistsSelector(labelKeys...), } return c.ListDeployments(opts) } func DeploymentMap(deployments []kapps.Deployment) map[string]kapps.Deployment { deploymentMap := map[string]kapps.Deployment{} for _, deployment := range deployments { deploymentMap[deployment.Name] = deployment } return deploymentMap } func DeploymentStartTime(deployment *kapps.Deployment) *time.Time { if deployment == nil { return nil } t := deployment.CreationTimestamp.Time if t.IsZero() { return nil } return &deployment.CreationTimestamp.Time } func DeploymentStrategiesMatch(s1, s2 kapps.DeploymentStrategy) bool { if s1.Type != s2.Type { return false } if s1.RollingUpdate == nil && s2.RollingUpdate == nil { return true } if s1.RollingUpdate == nil || s2.RollingUpdate == nil { return false } if !intOrStrPtrsMatch(s1.RollingUpdate.MaxUnavailable, s2.RollingUpdate.MaxUnavailable) { return false } if !intOrStrPtrsMatch(s1.RollingUpdate.MaxSurge, s2.RollingUpdate.MaxSurge) { return false } return true } func intOrStrPtrsMatch(intStr1, intStr2 *intstr.IntOrString) bool { if intStr1 == nil && intStr2 == nil { return true } if intStr1 == nil || intStr2 == nil { return false } return (*intStr1).String() == (*intStr2).String() } ================================================ FILE: pkg/lib/k8s/errors.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package k8s import ( "fmt" "github.com/cortexlabs/cortex/pkg/consts" "github.com/cortexlabs/cortex/pkg/lib/errors" s "github.com/cortexlabs/cortex/pkg/lib/strings" ) const ( ErrLabelNotFound = "k8s.label_not_found" ErrAnnotationNotFound = "k8s.annotation_not_found" ErrParseLabel = "k8s.parse_label" ErrParseAnnotation = "k8s.parse_annotation" ErrParseQuantity = "k8s.parse_quantity" ErrMissingMetrics = "k8s.missing_metrics" ErrServiceNotFound = "k8s.service_not_found" ) func ErrorLabelNotFound(labelName string) error { return errors.WithStack(&errors.Error{ Kind: ErrLabelNotFound, Message: fmt.Sprintf("label %s not found", s.UserStr(labelName)), }) } func ErrorAnnotationNotFound(annotationName string) error { return errors.WithStack(&errors.Error{ Kind: ErrAnnotationNotFound, Message: fmt.Sprintf("annotation %s not found", s.UserStr(annotationName)), }) } func ErrorParseLabel(labelName string, labelVal string, desiredType string) error { return errors.WithStack(&errors.Error{ Kind: ErrParseLabel, Message: fmt.Sprintf("unable to parse value %s from label %s as type %s", s.UserStr(labelVal), labelName, desiredType), }) } func ErrorParseAnnotation(annotationName string, annotationVal string, desiredType string) error { return errors.WithStack(&errors.Error{ Kind: ErrParseAnnotation, Message: fmt.Sprintf("unable to parse value %s from annotation %s as type %s", s.UserStr(annotationVal), annotationName, desiredType), }) } func ErrorParseQuantity(qtyStr string) error { return errors.WithStack(&errors.Error{ Kind: ErrParseQuantity, Message: fmt.Sprintf("%s: invalid kubernetes quantity, some valid examples are 1, 200m, 500Mi, 2G (see here for more information: https://docs.cortexlabs.com/v/%s/)", qtyStr, consts.CortexVersionMinor), }) } func ErrorMissingMetrics() error { return errors.WithStack(&errors.Error{ Kind: ErrMissingMetrics, Message: "must specify at least one metric", }) } func ErrorServiceNotFound(serviceName string) error { return errors.WithStack(&errors.Error{ Kind: ErrServiceNotFound, Message: fmt.Sprintf("service %s couldn't be found", serviceName), }) } ================================================ FILE: pkg/lib/k8s/hpa.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package k8s import ( "context" "github.com/cortexlabs/cortex/pkg/lib/errors" kautoscaling "k8s.io/api/autoscaling/v2beta2" kcore "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" kmeta "k8s.io/apimachinery/pkg/apis/meta/v1" klabels "k8s.io/apimachinery/pkg/labels" ) var _hpaTypeMeta = kmeta.TypeMeta{ APIVersion: "autoscaling/v1", Kind: "HorizontalPodAutoscaler", } type HPASpec struct { DeploymentName string MinReplicas int32 MaxReplicas int32 TargetCPUUtilization int32 TargetMemUtilization int32 Labels map[string]string Annotations map[string]string } func HPA(spec *HPASpec) (*kautoscaling.HorizontalPodAutoscaler, error) { metrics := []kautoscaling.MetricSpec{} if spec.TargetCPUUtilization > 0 { metrics = append(metrics, kautoscaling.MetricSpec{ Type: kautoscaling.ResourceMetricSourceType, Resource: &kautoscaling.ResourceMetricSource{ Name: kcore.ResourceCPU, Target: kautoscaling.MetricTarget{ Type: kautoscaling.UtilizationMetricType, AverageUtilization: &spec.TargetCPUUtilization, }, }, }) } if spec.TargetMemUtilization > 0 { metrics = append(metrics, kautoscaling.MetricSpec{ Type: kautoscaling.ResourceMetricSourceType, Resource: &kautoscaling.ResourceMetricSource{ Name: kcore.ResourceMemory, Target: kautoscaling.MetricTarget{ Type: kautoscaling.UtilizationMetricType, AverageUtilization: &spec.TargetMemUtilization, }, }, }) } if len(metrics) == 0 { return nil, ErrorMissingMetrics() } hpa := &kautoscaling.HorizontalPodAutoscaler{ TypeMeta: _hpaTypeMeta, ObjectMeta: kmeta.ObjectMeta{ Name: spec.DeploymentName, Labels: spec.Labels, Annotations: spec.Annotations, }, Spec: kautoscaling.HorizontalPodAutoscalerSpec{ MinReplicas: &spec.MinReplicas, MaxReplicas: spec.MaxReplicas, Metrics: metrics, ScaleTargetRef: kautoscaling.CrossVersionObjectReference{ Kind: _deploymentTypeMeta.Kind, Name: spec.DeploymentName, APIVersion: _deploymentTypeMeta.APIVersion, }, }, } return hpa, nil } func (c *Client) CreateHPA(hpa *kautoscaling.HorizontalPodAutoscaler) (*kautoscaling.HorizontalPodAutoscaler, error) { hpa.TypeMeta = _hpaTypeMeta hpa, err := c.hpaClient.Create(context.Background(), hpa, kmeta.CreateOptions{}) if err != nil { return nil, errors.WithStack(err) } return hpa, nil } func (c *Client) UpdateHPA(hpa *kautoscaling.HorizontalPodAutoscaler) (*kautoscaling.HorizontalPodAutoscaler, error) { hpa.TypeMeta = _hpaTypeMeta hpa, err := c.hpaClient.Update(context.Background(), hpa, kmeta.UpdateOptions{}) if err != nil { return nil, errors.WithStack(err) } return hpa, nil } func (c *Client) ApplyHPA(hpa *kautoscaling.HorizontalPodAutoscaler) (*kautoscaling.HorizontalPodAutoscaler, error) { existing, err := c.GetHPA(hpa.Name) if err != nil { return nil, err } if existing == nil { return c.CreateHPA(hpa) } return c.UpdateHPA(hpa) } func (c *Client) GetHPA(name string) (*kautoscaling.HorizontalPodAutoscaler, error) { hpa, err := c.hpaClient.Get(context.Background(), name, kmeta.GetOptions{}) if err != nil { if kerrors.IsNotFound(err) { return nil, nil } return nil, errors.WithStack(err) } hpa.TypeMeta = _hpaTypeMeta return hpa, nil } func (c *Client) DeleteHPA(name string) (bool, error) { err := c.hpaClient.Delete(context.Background(), name, _deleteOpts) if err != nil { if kerrors.IsNotFound(err) { return false, nil } return false, errors.WithStack(err) } return true, nil } func (c *Client) ListHPAs(opts *kmeta.ListOptions) ([]kautoscaling.HorizontalPodAutoscaler, error) { if opts == nil { opts = &kmeta.ListOptions{} } hpaList, err := c.hpaClient.List(context.Background(), *opts) if err != nil { return nil, errors.WithStack(err) } for i := range hpaList.Items { hpaList.Items[i].TypeMeta = _hpaTypeMeta } return hpaList.Items, nil } func (c *Client) ListHPAsByLabels(labels map[string]string) ([]kautoscaling.HorizontalPodAutoscaler, error) { opts := &kmeta.ListOptions{ LabelSelector: klabels.SelectorFromSet(labels).String(), } return c.ListHPAs(opts) } func (c *Client) ListHPAsByLabel(labelKey string, labelValue string) ([]kautoscaling.HorizontalPodAutoscaler, error) { return c.ListHPAsByLabels(map[string]string{labelKey: labelValue}) } func (c *Client) ListHPAsWithLabelKeys(labelKeys ...string) ([]kautoscaling.HorizontalPodAutoscaler, error) { opts := &kmeta.ListOptions{ LabelSelector: LabelExistsSelector(labelKeys...), } return c.ListHPAs(opts) } func HPAMap(hpas []kautoscaling.HorizontalPodAutoscaler) map[string]kautoscaling.HorizontalPodAutoscaler { hpaMap := map[string]kautoscaling.HorizontalPodAutoscaler{} for _, hpa := range hpas { hpaMap[hpa.Name] = hpa } return hpaMap } func IsHPAUpToDate(hpa *kautoscaling.HorizontalPodAutoscaler, minReplicas, maxReplicas, targetCPUUtilization, targetMemUtilization int32) bool { if hpa == nil { return false } if hpa.Spec.MinReplicas == nil || *hpa.Spec.MinReplicas != minReplicas { return false } if hpa.Spec.MaxReplicas != maxReplicas { return false } if len(hpa.Spec.Metrics) != 2 { return false } for _, metric := range hpa.Spec.Metrics { if metric.Type != kautoscaling.ResourceMetricSourceType || metric.Resource == nil { return false } if metric.Resource.Target.Type != kautoscaling.UtilizationMetricType || metric.Resource.Target.AverageUtilization == nil { return false } if metric.Resource.Name != kcore.ResourceCPU && metric.Resource.Name != kcore.ResourceMemory { return false } if metric.Resource.Name == kcore.ResourceCPU && *metric.Resource.Target.AverageUtilization != targetCPUUtilization { return false } if metric.Resource.Name == kcore.ResourceMemory && *metric.Resource.Target.AverageUtilization != targetMemUtilization { return false } } return true } ================================================ FILE: pkg/lib/k8s/ingress.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package k8s import ( "context" "github.com/cortexlabs/cortex/pkg/lib/errors" kextensions "k8s.io/api/extensions/v1beta1" kerrors "k8s.io/apimachinery/pkg/api/errors" kmeta "k8s.io/apimachinery/pkg/apis/meta/v1" klabels "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/intstr" ) var _ingressTypeMeta = kmeta.TypeMeta{ APIVersion: "extensions/v1beta1", Kind: "Ingress", } type IngressSpec struct { Name string IngressClass string ServiceName string ServicePort int32 Path string Labels map[string]string Annotations map[string]string } func Ingress(spec *IngressSpec) *kextensions.Ingress { if spec.Annotations == nil { spec.Annotations = make(map[string]string) } spec.Annotations["kubernetes.io/ingress.class"] = spec.IngressClass ingress := &kextensions.Ingress{ TypeMeta: _ingressTypeMeta, ObjectMeta: kmeta.ObjectMeta{ Name: spec.Name, Annotations: spec.Annotations, Labels: spec.Labels, }, Spec: kextensions.IngressSpec{ Rules: []kextensions.IngressRule{ { IngressRuleValue: kextensions.IngressRuleValue{ HTTP: &kextensions.HTTPIngressRuleValue{ Paths: []kextensions.HTTPIngressPath{ { Path: spec.Path, Backend: kextensions.IngressBackend{ ServiceName: spec.ServiceName, ServicePort: intstr.IntOrString{ IntVal: spec.ServicePort, }, }, }, }, }, }, }, }, }, } return ingress } func (c *Client) CreateIngress(ingress *kextensions.Ingress) (*kextensions.Ingress, error) { ingress.TypeMeta = _ingressTypeMeta ingress, err := c.ingressClient.Create(context.Background(), ingress, kmeta.CreateOptions{}) if err != nil { return nil, errors.WithStack(err) } return ingress, nil } func (c *Client) UpdateIngress(ingress *kextensions.Ingress) (*kextensions.Ingress, error) { ingress.TypeMeta = _ingressTypeMeta ingress, err := c.ingressClient.Update(context.Background(), ingress, kmeta.UpdateOptions{}) if err != nil { return nil, errors.WithStack(err) } return ingress, nil } func (c *Client) ApplyIngress(ingress *kextensions.Ingress) (*kextensions.Ingress, error) { existing, err := c.GetIngress(ingress.Name) if err != nil { return nil, err } if existing == nil { return c.CreateIngress(ingress) } return c.UpdateIngress(ingress) } func (c *Client) GetIngress(name string) (*kextensions.Ingress, error) { ingress, err := c.ingressClient.Get(context.Background(), name, kmeta.GetOptions{}) if err != nil { if kerrors.IsNotFound(err) { return nil, nil } return nil, errors.WithStack(err) } ingress.TypeMeta = _ingressTypeMeta return ingress, nil } func (c *Client) DeleteIngress(name string) (bool, error) { err := c.ingressClient.Delete(context.Background(), name, _deleteOpts) if err != nil { if kerrors.IsNotFound(err) { return false, nil } return false, errors.WithStack(err) } return true, nil } func (c *Client) ListIngresses(opts *kmeta.ListOptions) ([]kextensions.Ingress, error) { if opts == nil { opts = &kmeta.ListOptions{} } ingressList, err := c.ingressClient.List(context.Background(), *opts) if err != nil { return nil, errors.WithStack(err) } for i := range ingressList.Items { ingressList.Items[i].TypeMeta = _ingressTypeMeta } return ingressList.Items, nil } func (c *Client) ListIngressesByLabels(labels map[string]string) ([]kextensions.Ingress, error) { opts := &kmeta.ListOptions{ LabelSelector: klabels.SelectorFromSet(labels).String(), } return c.ListIngresses(opts) } func (c *Client) ListIngressesByLabel(labelKey string, labelValue string) ([]kextensions.Ingress, error) { return c.ListIngressesByLabels(map[string]string{labelKey: labelValue}) } func (c *Client) ListIngressesWithLabelKeys(labelKeys ...string) ([]kextensions.Ingress, error) { opts := &kmeta.ListOptions{ LabelSelector: LabelExistsSelector(labelKeys...), } return c.ListIngresses(opts) } func IngressMap(ingresses []kextensions.Ingress) map[string]kextensions.Ingress { ingressMap := map[string]kextensions.Ingress{} for _, ingress := range ingresses { ingressMap[ingress.Name] = ingress } return ingressMap } ================================================ FILE: pkg/lib/k8s/job.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package k8s import ( "context" "github.com/cortexlabs/cortex/pkg/lib/errors" kbatch "k8s.io/api/batch/v1" kcore "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" kmeta "k8s.io/apimachinery/pkg/apis/meta/v1" klabels "k8s.io/apimachinery/pkg/labels" ) var _jobTypeMeta = kmeta.TypeMeta{ APIVersion: "batch/v1", Kind: "Job", } type JobSpec struct { Name string Namespace string PodSpec PodSpec Parallelism int32 BackoffLimit int32 Labels map[string]string Annotations map[string]string } func Job(spec *JobSpec) *kbatch.Job { if spec.PodSpec.Name == "" { spec.PodSpec.Name = spec.Name } job := &kbatch.Job{ TypeMeta: _jobTypeMeta, ObjectMeta: kmeta.ObjectMeta{ Name: spec.Name, Namespace: spec.Namespace, Labels: spec.Labels, Annotations: spec.Annotations, }, Spec: kbatch.JobSpec{ BackoffLimit: &spec.BackoffLimit, Parallelism: &spec.Parallelism, Template: kcore.PodTemplateSpec{ ObjectMeta: kmeta.ObjectMeta{ Name: spec.PodSpec.Name, Labels: spec.PodSpec.Labels, Annotations: spec.PodSpec.Annotations, }, Spec: spec.PodSpec.K8sPodSpec, }, }, } return job } func (c *Client) CreateJob(job *kbatch.Job) (*kbatch.Job, error) { job.TypeMeta = _jobTypeMeta job, err := c.jobClient.Create(context.Background(), job, kmeta.CreateOptions{}) if err != nil { return nil, errors.WithStack(err) } return job, nil } func (c *Client) UpdateJob(job *kbatch.Job) (*kbatch.Job, error) { job.TypeMeta = _jobTypeMeta job, err := c.jobClient.Update(context.Background(), job, kmeta.UpdateOptions{}) if err != nil { return nil, errors.WithStack(err) } return job, nil } func (c *Client) ApplyJob(job *kbatch.Job) (*kbatch.Job, error) { existing, err := c.GetJob(job.Name) if err != nil { return nil, err } if existing == nil { return c.CreateJob(job) } return c.UpdateJob(job) } func (c *Client) GetJob(name string) (*kbatch.Job, error) { job, err := c.jobClient.Get(context.Background(), name, kmeta.GetOptions{}) if err != nil { if kerrors.IsNotFound(err) { return nil, nil } return nil, errors.WithStack(err) } job.TypeMeta = _jobTypeMeta return job, nil } func (c *Client) DeleteJob(name string) (bool, error) { err := c.jobClient.Delete(context.Background(), name, _deleteOpts) if err != nil { if kerrors.IsNotFound(err) { return false, nil } return false, errors.WithStack(err) } return true, nil } func (c *Client) DeleteJobs(opts *kmeta.ListOptions) (bool, error) { if opts == nil { opts = &kmeta.ListOptions{} } err := c.jobClient.DeleteCollection(context.Background(), _deleteOpts, *opts) if err != nil { return false, errors.WithStack(err) } return true, nil } func (c *Client) ListJobs(opts *kmeta.ListOptions) ([]kbatch.Job, error) { if opts == nil { opts = &kmeta.ListOptions{} } jobList, err := c.jobClient.List(context.Background(), *opts) if err != nil { return nil, errors.WithStack(err) } for i := range jobList.Items { jobList.Items[i].TypeMeta = _jobTypeMeta } return jobList.Items, nil } func (c *Client) ListJobsByLabels(labels map[string]string) ([]kbatch.Job, error) { opts := &kmeta.ListOptions{ LabelSelector: klabels.SelectorFromSet(labels).String(), } return c.ListJobs(opts) } func (c *Client) ListJobsByLabel(labelKey string, labelValue string) ([]kbatch.Job, error) { return c.ListJobsByLabels(map[string]string{labelKey: labelValue}) } func (c *Client) ListJobsWithLabelKeys(labelKeys ...string) ([]kbatch.Job, error) { opts := &kmeta.ListOptions{ LabelSelector: LabelExistsSelector(labelKeys...), } return c.ListJobs(opts) } func JobMap(jobs []kbatch.Job) map[string]kbatch.Job { jobMap := map[string]kbatch.Job{} for _, job := range jobs { jobMap[job.Name] = job } return jobMap } func (c *Client) IsJobRunning(name string) (bool, error) { job, err := c.GetJob(name) if err != nil { return false, err } if job == nil { return false, nil } return job.Status.CompletionTime == nil, nil } ================================================ FILE: pkg/lib/k8s/k8s.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package k8s import ( "path" "regexp" "strings" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/random" istioclient "istio.io/client-go/pkg/clientset/versioned" istionetworkingclient "istio.io/client-go/pkg/clientset/versioned/typed/networking/v1beta1" kresource "k8s.io/apimachinery/pkg/api/resource" kmeta "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" kclientdynamic "k8s.io/client-go/dynamic" kclientset "k8s.io/client-go/kubernetes" kclientapps "k8s.io/client-go/kubernetes/typed/apps/v1" kclientautoscaling "k8s.io/client-go/kubernetes/typed/autoscaling/v2beta2" kclientbatch "k8s.io/client-go/kubernetes/typed/batch/v1" kclientcore "k8s.io/client-go/kubernetes/typed/core/v1" kclientextensions "k8s.io/client-go/kubernetes/typed/extensions/v1beta1" _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" kclientrest "k8s.io/client-go/rest" kclientcmd "k8s.io/client-go/tools/clientcmd" kclienthomedir "k8s.io/client-go/util/homedir" ctrl "sigs.k8s.io/controller-runtime/pkg/client" ) var ( _home = kclienthomedir.HomeDir() _deletePolicy = kmeta.DeletePropagationBackground _deleteOpts = kmeta.DeleteOptions{ PropagationPolicy: &_deletePolicy, } ) type Client struct { ctrl.Client RestConfig *kclientrest.Config clientSet *kclientset.Clientset istioClientSet *istioclient.Clientset dynamicClient kclientdynamic.Interface podClient kclientcore.PodInterface nodeClient kclientcore.NodeInterface serviceClient kclientcore.ServiceInterface configMapClient kclientcore.ConfigMapInterface secretClient kclientcore.SecretInterface deploymentClient kclientapps.DeploymentInterface jobClient kclientbatch.JobInterface ingressClient kclientextensions.IngressInterface hpaClient kclientautoscaling.HorizontalPodAutoscalerInterface virtualServiceClient istionetworkingclient.VirtualServiceInterface Namespace string } func New(namespace string, inCluster bool, restConfig *kclientrest.Config, scheme *runtime.Scheme) (*Client, error) { var err error client := &Client{ Namespace: namespace, } if restConfig != nil { client.RestConfig = restConfig } else if inCluster { client.RestConfig, err = kclientrest.InClusterConfig() } else { kubeConfig := path.Join(_home, ".kube", "config") client.RestConfig, err = kclientcmd.BuildConfigFromFlags("", kubeConfig) } if err != nil { return nil, errors.Wrap(err, "kubeconfig") } client.clientSet, err = kclientset.NewForConfig(client.RestConfig) if err != nil { return nil, errors.Wrap(err, "kubeconfig") } client.dynamicClient, err = kclientdynamic.NewForConfig(client.RestConfig) if err != nil { return nil, errors.Wrap(err, "kubeconfig") } client.Client, err = ctrl.New(client.RestConfig, ctrl.Options{Scheme: scheme}) if err != nil { return nil, errors.Wrap(err, "kubeconfig") } client.istioClientSet, err = istioclient.NewForConfig(client.RestConfig) if err != nil { return nil, errors.Wrap(err, "kubeconfig") } client.virtualServiceClient = client.istioClientSet.NetworkingV1beta1().VirtualServices(namespace) client.podClient = client.clientSet.CoreV1().Pods(namespace) client.nodeClient = client.clientSet.CoreV1().Nodes() client.serviceClient = client.clientSet.CoreV1().Services(namespace) client.configMapClient = client.clientSet.CoreV1().ConfigMaps(namespace) client.secretClient = client.clientSet.CoreV1().Secrets(namespace) client.deploymentClient = client.clientSet.AppsV1().Deployments(namespace) client.jobClient = client.clientSet.BatchV1().Jobs(namespace) client.ingressClient = client.clientSet.ExtensionsV1beta1().Ingresses(namespace) client.hpaClient = client.clientSet.AutoscalingV2beta2().HorizontalPodAutoscalers(namespace) return client, nil } func (c *Client) ClientSet() *kclientset.Clientset { return c.clientSet } func (c *Client) IstioClientSet() *istioclient.Clientset { return c.istioClientSet } // to be safe, k8s sometimes needs all characters to be lower case, and the first to be a letter func RandomName() string { return random.LowercaseLetters(1) + random.LowercaseString(62) } // ValidName ensures name contains only lower case alphanumeric, '-', or '.' func ValidName(name string) string { re := regexp.MustCompile(`[^a-zA-Z0-9\-\.]`) name = re.ReplaceAllLiteralString(name, "-") name = strings.ToLower(name) return name } // ValidNameContainer ensures name contains only lower case alphanumeric or '-', must start with alphabetic, end with alphanumeric func ValidNameContainer(name string) string { name = ValidName(name) dots := regexp.MustCompile(`[\.]`) name = dots.ReplaceAllLiteralString(name, "-") leading := regexp.MustCompile(`^[^a-z]*`) name = leading.ReplaceAllLiteralString(name, "") trailing := regexp.MustCompile(`[^a-z0-9]*$`) name = trailing.ReplaceAllLiteralString(name, "") if len(name) == 0 { name = "x" } return name } func CPU(cpu string) kresource.Quantity { return kresource.MustParse(cpu) } func Mem(mem string) kresource.Quantity { return kresource.MustParse(mem) } func LabelExistsSelector(labelKeys ...string) string { if len(labelKeys) == 0 { return "" } return strings.Join(labelKeys, ",") } func FieldSelectorNotIn(key string, values []string) string { selectors := make([]string, len(values)) for i, value := range values { selectors[i] = key + "!=" + value } return strings.Join(selectors, ",") } ================================================ FILE: pkg/lib/k8s/node.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package k8s import ( "context" "github.com/cortexlabs/cortex/pkg/lib/errors" kcore "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" kmeta "k8s.io/apimachinery/pkg/apis/meta/v1" klabels "k8s.io/apimachinery/pkg/labels" ) var _nodeTypeMeta = kmeta.TypeMeta{ APIVersion: "v1", Kind: "Node", } func (c *Client) ListNodes(opts *kmeta.ListOptions) ([]kcore.Node, error) { if opts == nil { opts = &kmeta.ListOptions{} } nodeList, err := c.nodeClient.List(context.Background(), *opts) if err != nil { return nil, errors.WithStack(err) } for i := range nodeList.Items { nodeList.Items[i].TypeMeta = _nodeTypeMeta } return nodeList.Items, nil } func (c *Client) ListNodesByLabels(labels map[string]string) ([]kcore.Node, error) { opts := &kmeta.ListOptions{ LabelSelector: klabels.SelectorFromSet(labels).String(), } return c.ListNodes(opts) } func (c *Client) ListNodesByLabel(labelKey string, labelValue string) ([]kcore.Node, error) { return c.ListNodesByLabels(map[string]string{labelKey: labelValue}) } func (c *Client) ListNodesWithLabelKeys(labelKeys ...string) ([]kcore.Node, error) { opts := &kmeta.ListOptions{ LabelSelector: LabelExistsSelector(labelKeys...), } return c.ListNodes(opts) } func HowManyPodsFitOnNode(podSpec kcore.PodSpec, node kcore.Node, cpuReserved resource.Quantity, memoryReserved resource.Quantity) int64 { cpuQty := node.Status.Allocatable[v1.ResourceCPU] memoryQty := node.Status.Allocatable[v1.ResourceMemory] gpuQty := node.Status.Allocatable["nvidia.com/gpu"] infQty := node.Status.Allocatable["aws.amazon.com/neuron"] podsQty := node.Status.Allocatable[v1.ResourcePods] cpuQty.Sub(cpuReserved) memoryReserved.Sub(memoryReserved) cpuInt64 := cpuQty.MilliValue() memoryInt64 := memoryQty.MilliValue() gpuInt64 := gpuQty.Value() infInt64 := infQty.Value() maxPodsInt64 := podsQty.Value() cpuPodQty, memoryPodQty, podGPUInt64, podInfInt64 := TotalPodCompute(&podSpec) podCPUInt64 := cpuPodQty.MilliValue() podMemoryInt64 := memoryPodQty.MilliValue() if podCPUInt64 > 0 && float64(cpuInt64)/float64(podCPUInt64) < float64(maxPodsInt64) { maxPodsInt64 = int64(float64(cpuInt64) / float64(podCPUInt64)) } if podMemoryInt64 > 0 && float64(memoryInt64)/float64(podMemoryInt64) < float64(maxPodsInt64) { maxPodsInt64 = int64(float64(memoryInt64) / float64(podMemoryInt64)) } if podGPUInt64 > 0 && float64(gpuInt64)/float64(podGPUInt64) < float64(maxPodsInt64) { maxPodsInt64 = int64(float64(gpuInt64) / float64(podGPUInt64)) } if podInfInt64 > 0 && float64(infInt64)/float64(podInfInt64) < float64(maxPodsInt64) { maxPodsInt64 = int64(float64(infInt64) / float64(podInfInt64)) } return maxPodsInt64 } ================================================ FILE: pkg/lib/k8s/parsers.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package k8s import ( "time" s "github.com/cortexlabs/cortex/pkg/lib/strings" kmeta "k8s.io/apimachinery/pkg/apis/meta/v1" _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" ) func GetLabel(obj kmeta.Object, key string) (string, error) { labels := obj.GetLabels() if labels == nil { return "", ErrorLabelNotFound(key) } val, ok := labels[key] if !ok { return "", ErrorLabelNotFound(key) } return val, nil } func GetAnnotation(obj kmeta.Object, key string) (string, error) { annotations := obj.GetAnnotations() if annotations == nil { return "", ErrorAnnotationNotFound(key) } val, ok := annotations[key] if !ok { return "", ErrorAnnotationNotFound(key) } return val, nil } func ParseBoolLabel(obj kmeta.Object, key string) (bool, error) { val, err := GetLabel(obj, key) if err != nil { return false, err } casted, ok := s.ParseBool(val) if !ok { return false, ErrorParseLabel(key, val, "bool") } return casted, nil } func ParseBoolAnnotation(obj kmeta.Object, key string) (bool, error) { val, err := GetAnnotation(obj, key) if err != nil { return false, err } casted, ok := s.ParseBool(val) if !ok { return false, ErrorParseAnnotation(key, val, "bool") } return casted, nil } func ParseIntLabel(obj kmeta.Object, key string) (int, error) { val, err := GetLabel(obj, key) if err != nil { return 0, err } casted, ok := s.ParseInt(val) if !ok { return 0, ErrorParseLabel(key, val, "int") } return casted, nil } func ParseIntAnnotation(obj kmeta.Object, key string) (int, error) { val, err := GetAnnotation(obj, key) if err != nil { return 0, err } casted, ok := s.ParseInt(val) if !ok { return 0, ErrorParseAnnotation(key, val, "int") } return casted, nil } func ParseInt32Label(obj kmeta.Object, key string) (int32, error) { val, err := GetLabel(obj, key) if err != nil { return 0, err } casted, ok := s.ParseInt32(val) if !ok { return 0, ErrorParseLabel(key, val, "int32") } return casted, nil } func ParseInt32Annotation(obj kmeta.Object, key string) (int32, error) { val, err := GetAnnotation(obj, key) if err != nil { return 0, err } casted, ok := s.ParseInt32(val) if !ok { return 0, ErrorParseAnnotation(key, val, "int32") } return casted, nil } func ParseInt64Label(obj kmeta.Object, key string) (int64, error) { val, err := GetLabel(obj, key) if err != nil { return 0, err } casted, ok := s.ParseInt64(val) if !ok { return 0, ErrorParseLabel(key, val, "int64") } return casted, nil } func ParseInt64Annotation(obj kmeta.Object, key string) (int64, error) { val, err := GetAnnotation(obj, key) if err != nil { return 0, err } casted, ok := s.ParseInt64(val) if !ok { return 0, ErrorParseAnnotation(key, val, "int64") } return casted, nil } func ParseFloat32Label(obj kmeta.Object, key string) (float32, error) { val, err := GetLabel(obj, key) if err != nil { return 0, err } casted, ok := s.ParseFloat32(val) if !ok { return 0, ErrorParseLabel(key, val, "float32") } return casted, nil } func ParseFloat32Annotation(obj kmeta.Object, key string) (float32, error) { val, err := GetAnnotation(obj, key) if err != nil { return 0, err } casted, ok := s.ParseFloat32(val) if !ok { return 0, ErrorParseAnnotation(key, val, "float32") } return casted, nil } func ParseFloat64Label(obj kmeta.Object, key string) (float64, error) { val, err := GetLabel(obj, key) if err != nil { return 0, err } casted, ok := s.ParseFloat64(val) if !ok { return 0, ErrorParseLabel(key, val, "float64") } return casted, nil } func ParseFloat64Annotation(obj kmeta.Object, key string) (float64, error) { val, err := GetAnnotation(obj, key) if err != nil { return 0, err } casted, ok := s.ParseFloat64(val) if !ok { return 0, ErrorParseAnnotation(key, val, "float64") } return casted, nil } func ParseDurationLabel(obj kmeta.Object, key string) (time.Duration, error) { val, err := GetLabel(obj, key) if err != nil { return 0, err } casted, err := time.ParseDuration(val) if err != nil { return 0, ErrorParseLabel(key, val, "duration") } return casted, nil } func ParseDurationAnnotation(obj kmeta.Object, key string) (time.Duration, error) { val, err := GetAnnotation(obj, key) if err != nil { return 0, err } casted, err := time.ParseDuration(val) if err != nil { return 0, ErrorParseAnnotation(key, val, "duration") } return casted, nil } ================================================ FILE: pkg/lib/k8s/pod.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package k8s import ( "bytes" "context" "regexp" "time" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/pointer" "github.com/cortexlabs/cortex/pkg/lib/sets/strset" kcore "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" kmeta "k8s.io/apimachinery/pkg/apis/meta/v1" klabels "k8s.io/apimachinery/pkg/labels" kscheme "k8s.io/client-go/kubernetes/scheme" kremotecommand "k8s.io/client-go/tools/remotecommand" ) var _podTypeMeta = kmeta.TypeMeta{ APIVersion: "v1", Kind: "Pod", } // pod termination reasons // https://github.com/kubernetes/kube-state-metrics/blob/master/docs/pod-metrics.md const ( ReasonEvicted = "Evicted" ReasonOOMKilled = "OOMKilled" ReasonCompleted = "Completed" ) type PodSpec struct { Name string K8sPodSpec kcore.PodSpec Labels map[string]string Annotations map[string]string } type PodStatus string const ( PodStatusPending PodStatus = "Pending" PodStatusCreating PodStatus = "Creating" PodStatusNotReady PodStatus = "NotReady" PodStatusReady PodStatus = "Ready" PodStatusErrImagePull PodStatus = "ErrImagePull" PodStatusTerminating PodStatus = "Terminating" PodStatusFailed PodStatus = "Failed" PodStatusKilled PodStatus = "Killed" PodStatusKilledOOM PodStatus = "KilledOOM" PodStatusStalled PodStatus = "Stalled" PodStatusSucceeded PodStatus = "Succeeded" PodStatusUnknown PodStatus = "Unknown" ) var ( _killStatuses = map[int32]bool{ 137: true, // SIGKILL 143: true, // SIGTERM 130: true, // SIGINT 129: true, // SIGHUP } _evictedMemoryMessageRegex = regexp.MustCompile(`(?i)low\W+on\W+resource\W+memory`) // https://github.com/kubernetes/kubernetes/blob/master/pkg/kubelet/images/types.go#L27 _imagePullErrorStrings = strset.New("ErrImagePull", "ImagePullBackOff", "RegistryUnavailable") // https://github.com/kubernetes/kubernetes/blob/9f47110aa29094ed2878cf1d85874cb59214664a/staging/src/k8s.io/api/core/v1/types.go#L76-L77 _creatingReasons = strset.New("ContainerCreating", "PodInitializing") _waitForCreatingPodTimeout = time.Minute * 15 ) func Pod(spec *PodSpec) *kcore.Pod { pod := &kcore.Pod{ TypeMeta: _podTypeMeta, ObjectMeta: kmeta.ObjectMeta{ Name: spec.Name, Labels: spec.Labels, Annotations: spec.Annotations, }, Spec: spec.K8sPodSpec, } return pod } func GetPodConditionOf(pod *kcore.Pod, podType kcore.PodConditionType) (*bool, *kcore.PodCondition) { if pod == nil { return nil, nil } var conditionState *bool var condition *kcore.PodCondition for i := range pod.Status.Conditions { if pod.Status.Conditions[i].Type == podType { if pod.Status.Conditions[i].Status == kcore.ConditionTrue { conditionState = pointer.Bool(true) } if pod.Status.Conditions[i].Status == kcore.ConditionFalse { conditionState = pointer.Bool(false) } condition = &pod.Status.Conditions[i] break } } return conditionState, condition } func (c *Client) CreatePod(pod *kcore.Pod) (*kcore.Pod, error) { pod.TypeMeta = _podTypeMeta pod, err := c.podClient.Create(context.Background(), pod, kmeta.CreateOptions{}) if err != nil { return nil, errors.WithStack(err) } return pod, nil } func (c *Client) UpdatePod(pod *kcore.Pod) (*kcore.Pod, error) { pod.TypeMeta = _podTypeMeta pod, err := c.podClient.Update(context.Background(), pod, kmeta.UpdateOptions{}) if err != nil { return nil, errors.WithStack(err) } return pod, nil } func (c *Client) ApplyPod(pod *kcore.Pod) (*kcore.Pod, error) { existing, err := c.GetPod(pod.Name) if err != nil { return nil, err } if existing == nil { return c.CreatePod(pod) } return c.UpdatePod(pod) } func IsPodReady(pod *kcore.Pod) bool { if GetPodStatus(pod) != PodStatusReady { return false } podConditionState, _ := GetPodConditionOf(pod, kcore.PodReady) if podConditionState != nil && *podConditionState { return true } return false } func IsPodStalled(pod *kcore.Pod) bool { if GetPodStatus(pod) != PodStatusPending { return false } podConditionState, podCondition := GetPodConditionOf(pod, kcore.PodScheduled) if podConditionState != nil && !*podConditionState && !podCondition.LastTransitionTime.Time.IsZero() && time.Since(podCondition.LastTransitionTime.Time) >= _waitForCreatingPodTimeout { return true } return false } func GetPodReadyTime(pod *kcore.Pod) *time.Time { for i := range pod.Status.Conditions { condition := pod.Status.Conditions[i] if condition.Type == kcore.PodReady && condition.Status == kcore.ConditionTrue { if condition.LastTransitionTime.Time.IsZero() { return nil } return &condition.LastTransitionTime.Time } } return nil } func WasPodOOMKilled(pod *kcore.Pod) bool { if pod.Status.Reason == ReasonEvicted && _evictedMemoryMessageRegex.MatchString(pod.Status.Message) { return true } for _, containerStatus := range pod.Status.ContainerStatuses { var reason string if containerStatus.LastTerminationState.Terminated != nil { reason = containerStatus.LastTerminationState.Terminated.Reason } else if containerStatus.State.Terminated != nil { reason = containerStatus.State.Terminated.Reason } if reason == ReasonOOMKilled { return true } } return false } func GetPodStatus(pod *kcore.Pod) PodStatus { if pod == nil { return PodStatusUnknown } switch pod.Status.Phase { case kcore.PodPending: podConditionState, podCondition := GetPodConditionOf(pod, kcore.PodScheduled) if podConditionState != nil && !*podConditionState && !podCondition.LastTransitionTime.Time.IsZero() && time.Since(podCondition.LastTransitionTime.Time) >= _waitForCreatingPodTimeout { return PodStatusStalled } return PodStatusFromContainerStatuses(append(pod.Status.InitContainerStatuses, pod.Status.ContainerStatuses...)) case kcore.PodSucceeded: return PodStatusSucceeded case kcore.PodFailed: if pod.Status.Reason == ReasonEvicted && _evictedMemoryMessageRegex.MatchString(pod.Status.Message) { return PodStatusKilledOOM } for _, containerStatus := range pod.Status.ContainerStatuses { var reason string var exitCode int32 if containerStatus.LastTerminationState.Terminated != nil { reason = containerStatus.LastTerminationState.Terminated.Reason exitCode = containerStatus.LastTerminationState.Terminated.ExitCode } else if containerStatus.State.Terminated != nil { reason = containerStatus.State.Terminated.Reason exitCode = containerStatus.State.Terminated.ExitCode } if reason == ReasonOOMKilled { return PodStatusKilledOOM } else if _killStatuses[exitCode] { return PodStatusKilled } } return PodStatusFailed case kcore.PodRunning: if pod.ObjectMeta.DeletionTimestamp != nil { return PodStatusTerminating } podConditionState, _ := GetPodConditionOf(pod, kcore.PodReady) if podConditionState != nil && *podConditionState { return PodStatusReady } status := PodStatusFromContainerStatuses(pod.Status.ContainerStatuses) if status == PodStatusReady { return PodStatusNotReady } return status default: return PodStatusUnknown } } func PodStatusFromContainerStatuses(containerStatuses []kcore.ContainerStatus) PodStatus { numContainers := len(containerStatuses) numWaiting := 0 numCreating := 0 numNotReady := 0 numReady := 0 numSucceeded := 0 numFailed := 0 numKilled := 0 numKilledOOM := 0 if len(containerStatuses) == 0 { return PodStatusPending } for _, containerStatus := range containerStatuses { if containerStatus.State.Running != nil && containerStatus.Ready { numReady++ } else if containerStatus.State.Running != nil && !containerStatus.Ready { numNotReady++ } else if containerStatus.State.Terminated != nil { exitCode := containerStatus.State.Terminated.ExitCode reason := containerStatus.State.Terminated.Reason if reason == ReasonOOMKilled { numKilledOOM++ } else if exitCode == 0 { numSucceeded++ } else if _killStatuses[exitCode] { numKilled++ } else { numFailed++ } } else if containerStatus.LastTerminationState.Terminated != nil { exitCode := containerStatus.LastTerminationState.Terminated.ExitCode reason := containerStatus.LastTerminationState.Terminated.Reason if reason == ReasonOOMKilled { numKilledOOM++ } else if exitCode == 0 { numSucceeded++ } else if _killStatuses[exitCode] { numKilled++ } else { numFailed++ } } else if containerStatus.State.Waiting != nil && _imagePullErrorStrings.Has(containerStatus.State.Waiting.Reason) { return PodStatusErrImagePull } else if containerStatus.State.Waiting != nil && _creatingReasons.Has(containerStatus.State.Waiting.Reason) { numCreating++ } else { // either containerStatus.State.Waiting != nil or all containerStatus.States are nil (which implies waiting) numWaiting++ } } if numKilledOOM > 0 { return PodStatusKilledOOM } else if numKilled > 0 { return PodStatusKilled } else if numFailed > 0 { return PodStatusFailed } else if numWaiting > 0 { return PodStatusPending } else if numSucceeded == numContainers { return PodStatusSucceeded } else if numCreating > 0 { return PodStatusCreating } else if numNotReady > 0 { return PodStatusNotReady } else { return PodStatusReady } } func (c *Client) WaitForPodRunning(name string, numSeconds int) error { for true { pod, err := c.GetPod(name) if err != nil { return err } if pod != nil && pod.Status.Phase == kcore.PodRunning { return nil } time.Sleep(time.Duration(numSeconds) * time.Second) } return nil } func (c *Client) GetPod(name string) (*kcore.Pod, error) { pod, err := c.podClient.Get(context.Background(), name, kmeta.GetOptions{}) if err != nil { if kerrors.IsNotFound(err) { return nil, nil } return nil, errors.WithStack(err) } pod.TypeMeta = _podTypeMeta return pod, nil } func (c *Client) DeletePod(name string) (bool, error) { err := c.podClient.Delete(context.Background(), name, _deleteOpts) if err != nil { if kerrors.IsNotFound(err) { return false, nil } return false, errors.WithStack(err) } return true, nil } func (c *Client) ListPods(opts *kmeta.ListOptions) ([]kcore.Pod, error) { if opts == nil { opts = &kmeta.ListOptions{} } podList, err := c.podClient.List(context.Background(), *opts) if err != nil { return nil, errors.WithStack(err) } for i := range podList.Items { podList.Items[i].TypeMeta = _podTypeMeta } return podList.Items, nil } func (c *Client) ListPodsByLabels(labels map[string]string) ([]kcore.Pod, error) { opts := &kmeta.ListOptions{ LabelSelector: klabels.SelectorFromSet(labels).String(), } return c.ListPods(opts) } func (c *Client) ListPodsByLabel(labelKey string, labelValue string) ([]kcore.Pod, error) { return c.ListPodsByLabels(map[string]string{labelKey: labelValue}) } func (c *Client) ListPodsWithLabelKeys(labelKeys ...string) ([]kcore.Pod, error) { opts := &kmeta.ListOptions{ LabelSelector: LabelExistsSelector(labelKeys...), } return c.ListPods(opts) } func PodMap(pods []kcore.Pod) map[string]kcore.Pod { podMap := map[string]kcore.Pod{} for _, pod := range pods { podMap[pod.Name] = pod } return podMap } func PodComputesEqual(podSpec1, podSpec2 *kcore.PodSpec) bool { cpu1, mem1, gpu1, inf1 := TotalPodCompute(podSpec1) cpu2, mem2, gpu2, inf2 := TotalPodCompute(podSpec2) return cpu1.Equal(cpu2) && mem1.Equal(mem2) && gpu1 == gpu2 && inf1 == inf2 } func TotalPodCompute(podSpec *kcore.PodSpec) (Quantity, Quantity, int64, int64) { totalCPU := Quantity{} totalMem := Quantity{} var totalGPU, totalInf int64 if podSpec == nil { return totalCPU, totalMem, totalGPU, totalInf } for _, container := range podSpec.Containers { requests := container.Resources.Requests if len(requests) == 0 { continue } totalCPU.Add(requests[kcore.ResourceCPU]) totalMem.Add(requests[kcore.ResourceMemory]) if gpu, ok := requests["nvidia.com/gpu"]; ok { totalGPU += gpu.Value() } if inf, ok := requests["aws.amazon.com/neuron"]; ok { totalInf += inf.Value() } } return totalCPU, totalMem, totalGPU, totalInf } // Example of running a shell command: []string{"/bin/bash", "-c", "ps aux | grep my-proc"} func (c *Client) Exec(podName string, containerName string, command []string) (string, error) { options := &kcore.PodExecOptions{ Container: containerName, Command: command, Stdin: false, Stdout: true, Stderr: true, TTY: true, } req := c.clientSet.CoreV1().RESTClient().Post().Namespace(c.Namespace).Resource("pods").Name(podName).SubResource("exec") req.VersionedParams(options, kscheme.ParameterCodec) exec, err := kremotecommand.NewSPDYExecutor(c.RestConfig, "POST", req.URL()) if err != nil { return "", err } buf := &bytes.Buffer{} err = exec.Stream(kremotecommand.StreamOptions{ Stdin: nil, Stdout: buf, Stderr: nil, // TTY merges stdout and stderr Tty: true, }) if err != nil { return "", err } return buf.String(), nil } ================================================ FILE: pkg/lib/k8s/quantity.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package k8s import ( "encoding/json" "math" "github.com/cortexlabs/cortex/pkg/lib/configreader" s "github.com/cortexlabs/cortex/pkg/lib/strings" kresource "k8s.io/apimachinery/pkg/api/resource" ) type Quantity struct { kresource.Quantity UserString string } type QuantityValidation struct { GreaterThan *kresource.Quantity GreaterThanOrEqualTo *kresource.Quantity LessThan *kresource.Quantity LessThanOrEqualTo *kresource.Quantity } func QuantityParser(v *QuantityValidation) func(string) (interface{}, error) { return func(str string) (interface{}, error) { k8sQuantity, err := kresource.ParseQuantity(str) if err != nil { return Quantity{}, ErrorParseQuantity(str) } if v.GreaterThan != nil { if k8sQuantity.Cmp(*v.GreaterThan) <= 0 { return nil, configreader.ErrorMustBeGreaterThan(str, *v.GreaterThan) } } if v.GreaterThanOrEqualTo != nil { if k8sQuantity.Cmp(*v.GreaterThanOrEqualTo) < 0 { return nil, configreader.ErrorMustBeGreaterThanOrEqualTo(str, *v.GreaterThanOrEqualTo) } } if v.LessThan != nil { if k8sQuantity.Cmp(*v.LessThan) >= 0 { return nil, configreader.ErrorMustBeLessThan(str, *v.LessThan) } } if v.LessThanOrEqualTo != nil { if k8sQuantity.Cmp(*v.LessThanOrEqualTo) > 0 { return nil, configreader.ErrorMustBeLessThanOrEqualTo(str, *v.LessThanOrEqualTo) } } return Quantity{ Quantity: k8sQuantity, UserString: str, }, nil } } func WrapQuantity(k8sQuantity kresource.Quantity) *Quantity { return &Quantity{ Quantity: k8sQuantity, } } func NewQuantity(value int64) *Quantity { k8sQuantity := kresource.NewQuantity(value, kresource.DecimalSI) return &Quantity{ Quantity: *k8sQuantity, } } func NewMilliQuantity(milliValue int64) *Quantity { k8sQuantity := kresource.NewMilliQuantity(milliValue, kresource.DecimalSI) return &Quantity{ Quantity: *k8sQuantity, UserString: s.Int64(milliValue) + "m", } } // Returns nil if no quantities are passed in func NewSummed(quantities ...kresource.Quantity) *Quantity { if len(quantities) == 0 { return nil } k8sQuantity := kresource.Quantity{} for _, q := range quantities { k8sQuantity.Add(q) } return &Quantity{ Quantity: k8sQuantity, } } func (quantity *Quantity) MilliString() string { return s.Int64(quantity.Quantity.MilliValue()) + "m" } func (quantity *Quantity) ToFloat32() float32 { return float32(quantity.Quantity.MilliValue()) / float32(1000) } func ToKiRounded(k8sQuantity kresource.Quantity) int64 { kiFloat := float64(k8sQuantity.Value()) / float64(1024) return int64(math.Round(kiFloat)) } func ToKiRoundedStr(k8sQuantity kresource.Quantity) string { return s.Int64(ToKiRounded(k8sQuantity)) + "Ki" } func (quantity *Quantity) ToKiRounded() int64 { return ToKiRounded(quantity.Quantity) } func (quantity *Quantity) ToKiRoundedStr() string { return ToKiRoundedStr(quantity.Quantity) } func ToKiCeil(k8sQuantity kresource.Quantity) int64 { kiFloat := float64(k8sQuantity.Value()) / float64(1024) return int64(math.Ceil(kiFloat)) } func ToKiCeilStr(k8sQuantity kresource.Quantity) string { return s.Int64(ToKiCeil(k8sQuantity)) + "Ki" } func (quantity *Quantity) ToKiCeil() int64 { return ToKiCeil(quantity.Quantity) } func (quantity *Quantity) ToKiCeilStr() string { return ToKiCeilStr(quantity.Quantity) } func ToKiFloor(k8sQuantity kresource.Quantity) int64 { kiFloat := float64(k8sQuantity.Value()) / float64(1024) return int64(math.Floor(kiFloat)) } func ToKiFloorStr(k8sQuantity kresource.Quantity) string { return s.Int64(ToKiFloor(k8sQuantity)) + "Ki" } func (quantity *Quantity) ToKiFloor() int64 { return ToKiFloor(quantity.Quantity) } func (quantity *Quantity) ToKiFloorStr() string { return ToKiFloorStr(quantity.Quantity) } func ToMiRounded(k8sQuantity kresource.Quantity) int64 { miFloat := float64(k8sQuantity.Value()) / float64(1024*1024) return int64(math.Round(miFloat)) } func ToMiRoundedStr(k8sQuantity kresource.Quantity) string { return s.Int64(ToMiRounded(k8sQuantity)) + "Mi" } func (quantity *Quantity) ToMiRounded() int64 { return ToMiRounded(quantity.Quantity) } func (quantity *Quantity) ToMiRoundedStr() string { return ToMiRoundedStr(quantity.Quantity) } func ToMiCeil(k8sQuantity kresource.Quantity) int64 { miFloat := float64(k8sQuantity.Value()) / float64(1024*1024) return int64(math.Ceil(miFloat)) } func ToMiCeilStr(k8sQuantity kresource.Quantity) string { return s.Int64(ToMiCeil(k8sQuantity)) + "Mi" } func (quantity *Quantity) ToMiCeil() int64 { return ToMiCeil(quantity.Quantity) } func (quantity *Quantity) ToMiCeilStr() string { return ToMiCeilStr(quantity.Quantity) } func ToMiFloor(k8sQuantity kresource.Quantity) int64 { miFloat := float64(k8sQuantity.Value()) / float64(1024*1024) return int64(math.Floor(miFloat)) } func ToMiFloorStr(k8sQuantity kresource.Quantity) string { return s.Int64(ToMiFloor(k8sQuantity)) + "Mi" } func (quantity *Quantity) ToMiFloor() int64 { return ToMiFloor(quantity.Quantity) } func (quantity *Quantity) ToMiFloorStr() string { return ToMiFloorStr(quantity.Quantity) } func (quantity *Quantity) Sub(q2 kresource.Quantity) { quantity.Quantity.Sub(q2) quantity.UserString = "" } func (quantity *Quantity) SubQty(q2 Quantity) { quantity.Quantity.Sub(q2.Quantity) quantity.UserString = "" } func (quantity *Quantity) Add(q2 kresource.Quantity) { quantity.Quantity.Add(q2) quantity.UserString = "" } func (quantity *Quantity) AddQty(q2 Quantity) { quantity.Quantity.Add(q2.Quantity) quantity.UserString = "" } func (quantity *Quantity) String() string { if quantity == nil { return "" } if quantity.UserString != "" { return quantity.UserString } return quantity.Quantity.String() } func (quantity *Quantity) Equal(quantity2 Quantity) bool { return quantity.Quantity.Cmp(quantity2.Quantity) == 0 } func (quantity *Quantity) ID() string { return s.Int64(quantity.MilliValue()) } func (quantity *Quantity) DeepCopy() Quantity { return Quantity{ Quantity: quantity.Quantity.DeepCopy(), UserString: quantity.UserString, } } func QuantityPtr(k8sQuantity kresource.Quantity) *kresource.Quantity { return &k8sQuantity } func QuantityPtrID(quantity *Quantity) string { if quantity == nil { return "nil" } return quantity.ID() } func QuantityPtrsEqual(quantity *Quantity, quantity2 *Quantity) bool { if quantity == nil && quantity2 == nil { return true } if quantity == nil || quantity2 == nil { return false } return quantity.Equal(*quantity2) } type quantityMarshalable struct { Quantity kresource.Quantity UserString string } func (quantity Quantity) MarshalYAML() (interface{}, error) { return quantity.String(), nil } func (quantity *Quantity) UnmarshalYAML(unmarshal func(interface{}) error) error { var userString string err := unmarshal(&userString) if err != nil { return err } err = quantity.UnmarshalJSON([]byte(userString)) if err != nil { return err } return nil } func (quantity Quantity) MarshalJSON() ([]byte, error) { return json.Marshal(quantity.String()) } func (quantity *Quantity) UnmarshalJSON(data []byte) error { var userString string err := json.Unmarshal(data, &userString) quantity.UserString = userString parsedQuantity, err := kresource.ParseQuantity(userString) if err != nil { return err } quantity.Quantity = parsedQuantity quantity.UserString = userString return nil } func (quantity Quantity) MarshalBinary() ([]byte, error) { return json.Marshal(quantity) } func (quantity *Quantity) UnmarshalBinary(data []byte) error { return json.Unmarshal(data, quantity) } func (quantity Quantity) MarshalText() ([]byte, error) { return json.Marshal(quantity) } func (quantity *Quantity) UnmarshalText(data []byte) error { return json.Unmarshal(data, quantity) } ================================================ FILE: pkg/lib/k8s/secret.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package k8s import ( "context" "github.com/cortexlabs/cortex/pkg/lib/errors" kcore "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" kmeta "k8s.io/apimachinery/pkg/apis/meta/v1" klabels "k8s.io/apimachinery/pkg/labels" ) var _secretTypeMeta = kmeta.TypeMeta{ APIVersion: "v1", Kind: "Secret", } type SecretSpec struct { Name string Data map[string][]byte Labels map[string]string Annotations map[string]string } func Secret(spec *SecretSpec) *kcore.Secret { secret := &kcore.Secret{ TypeMeta: _secretTypeMeta, ObjectMeta: kmeta.ObjectMeta{ Name: spec.Name, Labels: spec.Labels, Annotations: spec.Annotations, }, Data: spec.Data, } return secret } func (c *Client) CreateSecret(secret *kcore.Secret) (*kcore.Secret, error) { secret.TypeMeta = _secretTypeMeta secret, err := c.secretClient.Create(context.Background(), secret, kmeta.CreateOptions{}) if err != nil { return nil, errors.WithStack(err) } return secret, nil } func (c *Client) UpdateSecret(secret *kcore.Secret) (*kcore.Secret, error) { secret.TypeMeta = _secretTypeMeta secret, err := c.secretClient.Update(context.Background(), secret, kmeta.UpdateOptions{}) if err != nil { return nil, errors.WithStack(err) } return secret, nil } func (c *Client) ApplySecret(secret *kcore.Secret) (*kcore.Secret, error) { existing, err := c.GetSecret(secret.Name) if err != nil { return nil, err } if existing == nil { return c.CreateSecret(secret) } return c.UpdateSecret(secret) } func (c *Client) GetSecret(name string) (*kcore.Secret, error) { secret, err := c.secretClient.Get(context.Background(), name, kmeta.GetOptions{}) if err != nil { if kerrors.IsNotFound(err) { return nil, nil } return nil, errors.WithStack(err) } secret.TypeMeta = _secretTypeMeta return secret, nil } func (c *Client) GetSecretData(name string) (map[string][]byte, error) { secret, err := c.GetSecret(name) if err != nil { return nil, err } if secret == nil { return nil, nil } return secret.Data, nil } func (c *Client) DeleteSecret(name string) (bool, error) { err := c.secretClient.Delete(context.Background(), name, _deleteOpts) if err != nil { if kerrors.IsNotFound(err) { return false, nil } return false, errors.WithStack(err) } return true, nil } func (c *Client) ListSecrets(opts *kmeta.ListOptions) ([]kcore.Secret, error) { if opts == nil { opts = &kmeta.ListOptions{} } secretList, err := c.secretClient.List(context.Background(), *opts) if err != nil { return nil, errors.WithStack(err) } for i := range secretList.Items { secretList.Items[i].TypeMeta = _secretTypeMeta } return secretList.Items, nil } func (c *Client) ListSecretsByLabels(labels map[string]string) ([]kcore.Secret, error) { opts := &kmeta.ListOptions{ LabelSelector: klabels.SelectorFromSet(labels).String(), } return c.ListSecrets(opts) } func (c *Client) ListSecretsByLabel(labelKey string, labelValue string) ([]kcore.Secret, error) { return c.ListSecretsByLabels(map[string]string{labelKey: labelValue}) } func (c *Client) ListSecretsWithLabelKeys(labelKeys ...string) ([]kcore.Secret, error) { opts := &kmeta.ListOptions{ LabelSelector: LabelExistsSelector(labelKeys...), } return c.ListSecrets(opts) } func SecretMap(secrets []kcore.Secret) map[string]kcore.Secret { secretMap := map[string]kcore.Secret{} for _, secret := range secrets { secretMap[secret.Name] = secret } return secretMap } ================================================ FILE: pkg/lib/k8s/service.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package k8s import ( "context" "fmt" "github.com/cortexlabs/cortex/pkg/lib/errors" kcore "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" kmeta "k8s.io/apimachinery/pkg/apis/meta/v1" klabels "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/intstr" ) var _serviceTypeMeta = kmeta.TypeMeta{ APIVersion: "v1", Kind: "Service", } type ServiceSpec struct { Name string PortName string Port int32 TargetPort int32 ServiceType kcore.ServiceType Selector map[string]string Labels map[string]string Annotations map[string]string } func Service(spec *ServiceSpec) *kcore.Service { service := &kcore.Service{ TypeMeta: _serviceTypeMeta, ObjectMeta: kmeta.ObjectMeta{ Name: spec.Name, Labels: spec.Labels, Annotations: spec.Annotations, }, Spec: kcore.ServiceSpec{ Selector: spec.Selector, Type: spec.ServiceType, Ports: []kcore.ServicePort{ { Protocol: kcore.ProtocolTCP, Name: spec.PortName, Port: spec.Port, TargetPort: intstr.IntOrString{ IntVal: spec.TargetPort, }, }, }, }, } return service } func (c *Client) CreateService(service *kcore.Service) (*kcore.Service, error) { service.TypeMeta = _serviceTypeMeta service, err := c.serviceClient.Create(context.Background(), service, kmeta.CreateOptions{}) if err != nil { return nil, errors.WithStack(err) } return service, nil } func (c *Client) UpdateService(existing, updated *kcore.Service) (*kcore.Service, error) { updated.TypeMeta = _serviceTypeMeta updated.Spec.ClusterIP = existing.Spec.ClusterIP updated.ResourceVersion = existing.ResourceVersion service, err := c.serviceClient.Update(context.Background(), updated, kmeta.UpdateOptions{}) if err != nil { return nil, errors.WithStack(err) } return service, nil } func (c *Client) ApplyService(service *kcore.Service) (*kcore.Service, error) { existing, err := c.GetService(service.Name) if err != nil { return nil, err } if existing == nil { return c.CreateService(service) } return c.UpdateService(existing, service) } func (c *Client) GetService(name string) (*kcore.Service, error) { service, err := c.serviceClient.Get(context.Background(), name, kmeta.GetOptions{}) if err != nil { if kerrors.IsNotFound(err) { return nil, nil } return nil, errors.WithStack(err) } service.TypeMeta = _serviceTypeMeta return service, nil } func (c *Client) DeleteService(name string) (bool, error) { err := c.serviceClient.Delete(context.Background(), name, _deleteOpts) if err != nil { if kerrors.IsNotFound(err) { return false, nil } return false, errors.WithStack(err) } return true, nil } func (c *Client) ListServices(opts *kmeta.ListOptions) ([]kcore.Service, error) { if opts == nil { opts = &kmeta.ListOptions{} } serviceList, err := c.serviceClient.List(context.Background(), *opts) if err != nil { return nil, errors.WithStack(err) } for i := range serviceList.Items { serviceList.Items[i].TypeMeta = _serviceTypeMeta } return serviceList.Items, nil } func (c *Client) ListServicesByLabels(labels map[string]string) ([]kcore.Service, error) { opts := &kmeta.ListOptions{ LabelSelector: klabels.SelectorFromSet(labels).String(), } return c.ListServices(opts) } func (c *Client) ListServicesByLabel(labelKey string, labelValue string) ([]kcore.Service, error) { return c.ListServicesByLabels(map[string]string{labelKey: labelValue}) } func (c *Client) ListServicesWithLabelKeys(labelKeys ...string) ([]kcore.Service, error) { opts := &kmeta.ListOptions{ LabelSelector: LabelExistsSelector(labelKeys...), } return c.ListServices(opts) } func (c *Client) InternalServiceEndpoint(serviceName string, portNumber int32) string { return fmt.Sprintf("http://%s.%s.svc.cluster.local:%d", serviceName, c.Namespace, portNumber) } func ServiceMap(services []kcore.Service) map[string]kcore.Service { serviceMap := map[string]kcore.Service{} for _, service := range services { serviceMap[service.Name] = service } return serviceMap } ================================================ FILE: pkg/lib/k8s/virtual_service.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package k8s import ( "context" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/sets/strset" "github.com/cortexlabs/cortex/pkg/lib/urls" istionetworking "istio.io/api/networking/v1beta1" istioclientnetworking "istio.io/client-go/pkg/apis/networking/v1beta1" istionetworkingclient "istio.io/client-go/pkg/clientset/versioned/typed/networking/v1beta1" kerrors "k8s.io/apimachinery/pkg/api/errors" kmeta "k8s.io/apimachinery/pkg/apis/meta/v1" klabels "k8s.io/apimachinery/pkg/labels" ) var _virtualServiceTypeMeta = kmeta.TypeMeta{ APIVersion: "v1beta1", Kind: "VirtualService", } type VirtualServiceSpec struct { Name string Gateways []string ExactPath *string // either this or PrefixPath PrefixPath *string // either this or ExactPath Destinations []Destination Rewrite *string Labels map[string]string Annotations map[string]string Headers *istionetworking.Headers Retries *int32 } type Destination struct { ServiceName string Weight int32 Port uint32 Shadow bool Headers *istionetworking.Headers } func VirtualService(spec *VirtualServiceSpec) *istioclientnetworking.VirtualService { destinations := []*istionetworking.HTTPRouteDestination{} var mirror *istionetworking.Destination var mirrorWeight *istionetworking.Percent for _, destination := range spec.Destinations { if destination.Shadow { mirror = &istionetworking.Destination{ Host: destination.ServiceName, Port: &istionetworking.PortSelector{ Number: destination.Port, }, } mirrorWeight = &istionetworking.Percent{Value: float64(destination.Weight)} } else { destinations = append(destinations, &istionetworking.HTTPRouteDestination{ Destination: &istionetworking.Destination{ Host: destination.ServiceName, Port: &istionetworking.PortSelector{ Number: destination.Port, }, }, Weight: destination.Weight, Headers: destination.Headers, }) } } var httpRoutes []*istionetworking.HTTPRoute if spec.ExactPath != nil { httpRoutes = append(httpRoutes, &istionetworking.HTTPRoute{ Match: []*istionetworking.HTTPMatchRequest{ { Uri: &istionetworking.StringMatch{ MatchType: &istionetworking.StringMatch_Exact{ Exact: urls.CanonicalizeEndpoint(*spec.ExactPath), }, }, }, }, Route: destinations, Mirror: mirror, MirrorPercentage: mirrorWeight, Headers: spec.Headers, }) if spec.Rewrite != nil { httpRoutes[0].Rewrite = &istionetworking.HTTPRewrite{ Uri: urls.CanonicalizeEndpoint(*spec.Rewrite), } } } else { exactMatch := &istionetworking.HTTPRoute{ Match: []*istionetworking.HTTPMatchRequest{ { Uri: &istionetworking.StringMatch{ MatchType: &istionetworking.StringMatch_Exact{ Exact: urls.CanonicalizeEndpoint(*spec.PrefixPath), }, }, }, }, Route: destinations, Mirror: mirror, MirrorPercentage: mirrorWeight, Headers: spec.Headers, } prefixMatch := &istionetworking.HTTPRoute{ Match: []*istionetworking.HTTPMatchRequest{ { Uri: &istionetworking.StringMatch{ MatchType: &istionetworking.StringMatch_Prefix{ Prefix: urls.CanonicalizeEndpointWithTrailingSlash(*spec.PrefixPath), }, }, }, }, Route: destinations, Mirror: mirror, MirrorPercentage: mirrorWeight, Headers: spec.Headers, } if spec.Rewrite != nil { exactMatch.Rewrite = &istionetworking.HTTPRewrite{ Uri: urls.CanonicalizeEndpoint(*spec.Rewrite), } prefixMatch.Rewrite = &istionetworking.HTTPRewrite{ Uri: urls.CanonicalizeEndpointWithTrailingSlash(*spec.Rewrite), } } httpRoutes = append(httpRoutes, exactMatch, prefixMatch) } if spec.Retries != nil { for i := range httpRoutes { httpRoutes[i].Retries = &istionetworking.HTTPRetry{ Attempts: *spec.Retries, } } } virtualService := &istioclientnetworking.VirtualService{ TypeMeta: _virtualServiceTypeMeta, ObjectMeta: kmeta.ObjectMeta{ Name: spec.Name, Labels: spec.Labels, Annotations: spec.Annotations, }, Spec: istionetworking.VirtualService{ Hosts: []string{"*"}, Gateways: spec.Gateways, Http: httpRoutes, }, } return virtualService } func (c *Client) VirtualServiceClient() istionetworkingclient.VirtualServiceInterface { return c.virtualServiceClient } func (c *Client) CreateVirtualService(virtualService *istioclientnetworking.VirtualService) (*istioclientnetworking.VirtualService, error) { virtualService.TypeMeta = _virtualServiceTypeMeta virtualService, err := c.virtualServiceClient.Create(context.Background(), virtualService, kmeta.CreateOptions{}) if err != nil { return nil, errors.WithStack(err) } return virtualService, nil } func (c *Client) UpdateVirtualService(existing, updated *istioclientnetworking.VirtualService) (*istioclientnetworking.VirtualService, error) { updated.TypeMeta = _virtualServiceTypeMeta updated.ResourceVersion = existing.ResourceVersion virtualService, err := c.virtualServiceClient.Update(context.Background(), updated, kmeta.UpdateOptions{}) if err != nil { return nil, errors.WithStack(err) } return virtualService, nil } func (c *Client) ApplyVirtualService(virtualService *istioclientnetworking.VirtualService) (*istioclientnetworking.VirtualService, error) { existing, err := c.GetVirtualService(virtualService.Name) if err != nil { return nil, err } if existing == nil { return c.CreateVirtualService(virtualService) } return c.UpdateVirtualService(existing, virtualService) } func (c *Client) GetVirtualService(name string) (*istioclientnetworking.VirtualService, error) { virtualService, err := c.virtualServiceClient.Get(context.Background(), name, kmeta.GetOptions{}) if err != nil { if kerrors.IsNotFound(err) { return nil, nil } return nil, errors.WithStack(err) } virtualService.TypeMeta = _virtualServiceTypeMeta return virtualService, nil } func (c *Client) DeleteVirtualService(name string) (bool, error) { err := c.virtualServiceClient.Delete(context.Background(), name, _deleteOpts) if err != nil { if kerrors.IsNotFound(err) { return false, nil } return false, errors.WithStack(err) } return true, nil } func (c *Client) ListVirtualServices(opts *kmeta.ListOptions) ([]istioclientnetworking.VirtualService, error) { if opts == nil { opts = &kmeta.ListOptions{} } vsList, err := c.virtualServiceClient.List(context.Background(), *opts) if err != nil { return nil, errors.WithStack(err) } for i := range vsList.Items { vsList.Items[i].TypeMeta = _virtualServiceTypeMeta } return vsList.Items, nil } func (c *Client) ListVirtualServicesByLabels(labels map[string]string) ([]istioclientnetworking.VirtualService, error) { opts := &kmeta.ListOptions{ LabelSelector: klabels.SelectorFromSet(labels).String(), } return c.ListVirtualServices(opts) } func (c *Client) ListVirtualServicesByLabel(labelKey string, labelValue string) ([]istioclientnetworking.VirtualService, error) { return c.ListVirtualServicesByLabels(map[string]string{labelKey: labelValue}) } func (c *Client) ListVirtualServicesWithLabelKeys(labelKeys ...string) ([]istioclientnetworking.VirtualService, error) { opts := &kmeta.ListOptions{ LabelSelector: LabelExistsSelector(labelKeys...), } return c.ListVirtualServices(opts) } func ExtractVirtualServiceGateways(virtualService *istioclientnetworking.VirtualService) strset.Set { return strset.FromSlice(virtualService.Spec.Gateways) } func ExtractVirtualServiceEndpoints(virtualService *istioclientnetworking.VirtualService) strset.Set { endpoints := strset.New() for _, http := range virtualService.Spec.Http { for _, match := range http.Match { if match.Uri.GetExact() != "" { endpoints.Add(urls.CanonicalizeEndpoint(match.Uri.GetExact())) } if match.Uri.GetPrefix() != "" { endpoints.Add(urls.CanonicalizeEndpoint(match.Uri.GetPrefix())) } } } return endpoints } ================================================ FILE: pkg/lib/k8s/volume.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package k8s import ( kcore "k8s.io/api/core/v1" ) func EmptyDirVolume(volumeName string) kcore.Volume { return kcore.Volume{ Name: volumeName, VolumeSource: kcore.VolumeSource{ EmptyDir: &kcore.EmptyDirVolumeSource{}, }, } } func EmptyDirVolumeMount(volumeName string, mountPath string) kcore.VolumeMount { return kcore.VolumeMount{ Name: volumeName, MountPath: mountPath, } } ================================================ FILE: pkg/lib/logging/errors.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package logging import ( "fmt" "github.com/cortexlabs/cortex/pkg/lib/errors" s "github.com/cortexlabs/cortex/pkg/lib/strings" ) const ( ErrInvalidLogLevel = "logging.invalid_log_level" ) func ErrorInvalidLogLevel(provided string, loglevels []string) error { return errors.WithStack(&errors.Error{ Kind: ErrInvalidLogLevel, Message: fmt.Sprintf("invalid log level %s; must be one of %s", provided, s.StrsOr(loglevels)), }) } ================================================ FILE: pkg/lib/logging/logging.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package logging import ( "os" "strings" "sync" "github.com/cortexlabs/cortex/pkg/types/userconfig" "go.uber.org/zap" ) var logger *zap.SugaredLogger var loggerLock sync.Mutex func initializeLogger() { logLevel := strings.ToLower(os.Getenv("CORTEX_LOG_LEVEL")) if logLevel == "" { logLevel = "info" } cortexLogLevel := userconfig.LogLevelFromString(logLevel) if cortexLogLevel == userconfig.UnknownLogLevel { panic(ErrorInvalidLogLevel(logLevel, userconfig.LogLevelTypes())) } zapConfig := DefaultZapConfig(cortexLogLevel) disableJSONLogging := strings.ToLower(os.Getenv("CORTEX_DISABLE_JSON_LOGGING")) if disableJSONLogging == "true" { zapConfig.Encoding = "console" } zapLogger, err := zapConfig.Build() if err != nil { panic(err) } logger = zapLogger.Sugar() } func GetLogger() *zap.SugaredLogger { loggerLock.Lock() defer loggerLock.Unlock() if logger == nil { initializeLogger() } return logger } func DefaultZapConfig(level userconfig.LogLevel, fields ...map[string]interface{}) zap.Config { encoderConfig := zap.NewProductionEncoderConfig() encoderConfig.MessageKey = "message" labels := map[string]interface{}{} for _, m := range fields { for k, v := range m { labels[k] = v } } initialFields := map[string]interface{}{} if len(labels) > 0 { initialFields["cortex.labels"] = labels } return zap.Config{ Level: zap.NewAtomicLevelAt(userconfig.ToZapLogLevel(level)), Encoding: "json", EncoderConfig: encoderConfig, OutputPaths: []string{"stdout"}, ErrorOutputPaths: []string{"stderr"}, InitialFields: initialFields, } } ================================================ FILE: pkg/lib/maps/interface.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package maps import ( "reflect" "sort" ) func InterfaceMapKeys(myMap map[string]interface{}) []string { keys := make([]string, len(myMap)) i := 0 for key := range myMap { keys[i] = key i++ } return keys } func InterfaceMapSortedKeys(myMap map[string]interface{}) []string { keys := InterfaceMapKeys(myMap) sort.Strings(keys) return keys } func InterfaceMapKeysUnsafe(myMap interface{}) []string { keyValues := reflect.ValueOf(myMap).MapKeys() keys := make([]string, len(keyValues)) for i := range keyValues { keys[i] = keyValues[i].String() } return keys } func InterfaceMapsKeysMatch(map1 map[string]interface{}, map2 map[string]interface{}) bool { if len(map1) != len(map2) { return false } for key := range map1 { if _, ok := map2[key]; !ok { return false } } return true } func MergeStrInterfaceMaps(maps ...map[string]interface{}) map[string]interface{} { merged := map[string]interface{}{} for _, m := range maps { for k, v := range m { merged[k] = v } } return merged } ================================================ FILE: pkg/lib/maps/string.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package maps func StrMapKeysString(myMap map[string]string) []string { keys := make([]string, len(myMap)) i := 0 for key := range myMap { keys[i] = key i++ } return keys } func StrMapValuesString(myMap map[string]string) []string { values := make([]string, len(myMap)) i := 0 for _, value := range myMap { values[i] = value i++ } return values } func MergeStrMapsString(maps ...map[string]string) map[string]string { merged := map[string]string{} for _, m := range maps { for k, v := range m { merged[k] = v } } return merged } func StrMapsEqualString(m1, m2 map[string]string) bool { if len(m1) != len(m2) { return false } if len(m1) == 0 && len(m2) == 0 { return true } if len(m1) == 0 || len(m2) == 0 { return false } for k, v1 := range m1 { if v2, ok := m2[k]; !ok || v2 != v1 { return false } } return true } func StrMapKeysInt(myMap map[string]int) []string { keys := make([]string, len(myMap)) i := 0 for key := range myMap { keys[i] = key i++ } return keys } func StrMapValuesInt(myMap map[string]int) []int { values := make([]int, len(myMap)) i := 0 for _, value := range myMap { values[i] = value i++ } return values } func MergeStrMapsInt(maps ...map[string]int) map[string]int { merged := map[string]int{} for _, m := range maps { for k, v := range m { merged[k] = v } } return merged } func StrMapsEqualInt(m1, m2 map[string]int) bool { if len(m1) != len(m2) { return false } if len(m1) == 0 && len(m2) == 0 { return true } if len(m1) == 0 || len(m2) == 0 { return false } for k, v1 := range m1 { if v2, ok := m2[k]; !ok || v2 != v1 { return false } } return true } ================================================ FILE: pkg/lib/math/float32.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package math func MinFloat32(val float32, vals ...float32) float32 { min := val for _, v := range vals { if v < min { min = v } } return min } func MaxFloat32(val float32, vals ...float32) float32 { max := val for _, v := range vals { if v > max { max = v } } return max } ================================================ FILE: pkg/lib/math/float64.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package math func MinFloat64(val float64, vals ...float64) float64 { min := val for _, v := range vals { if v < min { min = v } } return min } func MaxFloat64(val float64, vals ...float64) float64 { max := val for _, v := range vals { if v > max { max = v } } return max } ================================================ FILE: pkg/lib/math/int.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package math import ( "math" "sort" ) func MinInt(val int, vals ...int) int { min := val for _, v := range vals { if v < min { min = v } } return min } func MaxInt(val int, vals ...int) int { max := val for _, v := range vals { if v > max { max = v } } return max } func IsDivisibleByInt(num int, divisor int) bool { return num%divisor == 0 } func FactorsInt(num int) []int { divisibleNumbers := []int{} maxDivisor := int(math.Sqrt(float64(num))) incrementer := 1 // Skip even numbers if num is odd if num%2 == 1 { incrementer = 2 } for divisor := 1; divisor <= maxDivisor; divisor += incrementer { if num%divisor == 0 { divisibleNumbers = append(divisibleNumbers, divisor) complementaryDivisor := num / divisor if divisor != complementaryDivisor { divisibleNumbers = append(divisibleNumbers, complementaryDivisor) } } } sort.Ints(divisibleNumbers) return divisibleNumbers } ================================================ FILE: pkg/lib/math/int32.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package math import ( "math" "sort" ) func MinInt32(val int32, vals ...int32) int32 { min := val for _, v := range vals { if v < min { min = v } } return min } func MaxInt32(val int32, vals ...int32) int32 { max := val for _, v := range vals { if v > max { max = v } } return max } func IsDivisibleByInt32(num int32, divisor int32) bool { return num%divisor == 0 } func FactorsInt32(num int32) []int32 { divisibleNumbers := []int32{} maxDivisor := int32(math.Sqrt(float64(num))) incrementer := int32(1) // Skip even numbers if num is odd if num%2 == 1 { incrementer = int32(2) } for divisor := int32(1); divisor <= maxDivisor; divisor += incrementer { if num%divisor == 0 { divisibleNumbers = append(divisibleNumbers, divisor) complementaryDivisor := num / divisor if divisor != complementaryDivisor { divisibleNumbers = append(divisibleNumbers, complementaryDivisor) } } } sort.Slice(divisibleNumbers, func(i, j int) bool { return divisibleNumbers[i] < divisibleNumbers[j] }) return divisibleNumbers } ================================================ FILE: pkg/lib/math/int64.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package math import ( "math" "sort" ) func MinInt64(val int64, vals ...int64) int64 { min := val for _, v := range vals { if v < min { min = v } } return min } func MaxInt64(val int64, vals ...int64) int64 { max := val for _, v := range vals { if v > max { max = v } } return max } func IsDivisibleByInt64(num int64, divisor int64) bool { return num%divisor == 0 } func FactorsInt64(num int64) []int64 { divisibleNumbers := []int64{} maxDivisor := int64(math.Sqrt(float64(num))) incrementer := int64(1) // Skip even numbers if num is odd if num%2 == 1 { incrementer = int64(2) } for divisor := int64(1); divisor <= maxDivisor; divisor += incrementer { if num%divisor == 0 { divisibleNumbers = append(divisibleNumbers, divisor) complementaryDivisor := num / divisor if divisor != complementaryDivisor { divisibleNumbers = append(divisibleNumbers, complementaryDivisor) } } } sort.Slice(divisibleNumbers, func(i, j int) bool { return divisibleNumbers[i] < divisibleNumbers[j] }) return divisibleNumbers } ================================================ FILE: pkg/lib/msgpack/errors.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package msgpack import ( "github.com/cortexlabs/cortex/pkg/lib/errors" ) const ( ErrUnmarshalMsgpack = "msgpack.unmarshal_msgpack" ErrMarshalMsgpack = "msgpack.marshal_msgpack" ) func ErrorUnmarshalMsgpack() error { return errors.WithStack(&errors.Error{ Kind: ErrUnmarshalMsgpack, Message: "invalid messagepack", }) } func ErrorMarshalMsgpack() error { return errors.WithStack(&errors.Error{ Kind: ErrMarshalMsgpack, Message: "invalid messagepack cannot be serialized", }) } ================================================ FILE: pkg/lib/msgpack/msgpack.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package msgpack import ( "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/ugorji/go/codec" ) var _mh codec.MsgpackHandle func init() { _mh.RawToString = true } func Marshal(obj interface{}) ([]byte, error) { var bytes []byte enc := codec.NewEncoderBytes(&bytes, &_mh) err := enc.Encode(obj) if err != nil { return nil, errors.Wrap(err, errors.Message(ErrorMarshalMsgpack())) } return bytes, nil } func MustMarshal(obj interface{}) []byte { msgpackBytes, err := Marshal(obj) if err != nil { panic(err) } return msgpackBytes } func UnmarshalToInterface(b []byte) (interface{}, error) { var obj interface{} err := Unmarshal(b, &obj) if err != nil { return nil, errors.Wrap(err, errors.Message(ErrorUnmarshalMsgpack())) } return obj, nil } func Unmarshal(b []byte, obj interface{}) error { dec := codec.NewDecoderBytes(b, &_mh) return dec.Decode(&obj) } ================================================ FILE: pkg/lib/parallel/parallel.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package parallel import ( "github.com/cortexlabs/cortex/pkg/lib/errors" ) // Alternative: https://golang.org/pkg/sync/#WaitGroup (with error channel) // Alternative: https://godoc.org/golang.org/x/sync/errgroup func Run(fn func() error, fns ...func() error) []error { allFns := append(fns, fn) errChannels := make([]chan error, len(allFns)) for i := range errChannels { errChannels[i] = make(chan error) } for i := range allFns { fn := allFns[i] errChannel := errChannels[i] if fn == nil { errChannel <- nil continue } go func() { defer func() { if r := recover(); r != nil { errChannel <- errors.CastRecoverError(r) } }() errChannel <- fn() }() } errors := make([]error, len(allFns)) for i := range allFns { errors[i] = <-errChannels[i] } return errors } func RunFirstErr(fn func() error, fns ...func() error) error { errs := Run(fn, fns...) return errors.FirstError(errs...) } ================================================ FILE: pkg/lib/parallel/parallel_test.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package parallel // // These tests must be run and verified manually: // go test github.com/cortexlabs/cortex/pkg/lib/parallel -run TestRunInParallel -v // // import ( // "testing" // "github.com/stretchr/testify/require" // "github.com/cortexlabs/cortex/utils/util" // ) // func delayAndPrint(jobID string, secs int) error { // time.Sleep(time.Duration(secs) * time.Second) // t := time.Now() // fmt.Printf("Done with job %s at %d:%d:%s\n", jobID, t.Minute(), t.Second(), util.MillisecsStr(t)) // return nil // } // func delayAndError(jobID string, secs int) error { // delayAndPrint(jobID, secs) // return errors.New("Error in job " + jobID) // } // func TestRunInParallel(t *testing.T) { // var errs []error // var err error // errs = util.RunInParallel( // func() error { // return delayAndPrint("1", 1) // }, // func() error { // return delayAndPrint("2", 2) // }, // func() error { // return delayAndPrint("3", 3) // }, // ) // err = util.FirstError(errs...) // require.NoError(t, err) // errs = util.RunInParallel( // func() error { // return delayAndPrint("3", 3) // }, // func() error { // return delayAndPrint("2", 2) // }, // func() error { // return delayAndPrint("1", 1) // }, // ) // err = util.FirstError(errs...) // require.NoError(t, err) // errs = util.RunInParallel( // func() error { // return delayAndError("3", 3) // }, // func() error { // return delayAndError("2", 2) // }, // func() error { // return delayAndError("1", 1) // }, // ) // expectedErrs := []error{ // errors.New("Error in job 3"), // errors.New("Error in job 2"), // errors.New("Error in job 1"), // } // require.Equal(t, expectedErrs, errs) // } ================================================ FILE: pkg/lib/pointer/equal.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package pointer import ( "time" ) func AreIntsEqual(v1 *int, v2 *int) bool { if v1 == nil && v2 == nil { return true } if v1 == nil || v2 == nil { return false } return *v1 == *v2 } func AreInt8sEqual(v1 *int8, v2 *int8) bool { if v1 == nil && v2 == nil { return true } if v1 == nil || v2 == nil { return false } return *v1 == *v2 } func AreInt16sEqual(v1 *int16, v2 *int16) bool { if v1 == nil && v2 == nil { return true } if v1 == nil || v2 == nil { return false } return *v1 == *v2 } func AreInt32sEqual(v1 *int32, v2 *int32) bool { if v1 == nil && v2 == nil { return true } if v1 == nil || v2 == nil { return false } return *v1 == *v2 } func AreInt64sEqual(v1 *int64, v2 *int64) bool { if v1 == nil && v2 == nil { return true } if v1 == nil || v2 == nil { return false } return *v1 == *v2 } func AreFloat64sEqual(v1 *float64, v2 *float64) bool { if v1 == nil && v2 == nil { return true } if v1 == nil || v2 == nil { return false } return *v1 == *v2 } func AreFloat32sEqual(v1 *float32, v2 *float32) bool { if v1 == nil && v2 == nil { return true } if v1 == nil || v2 == nil { return false } return *v1 == *v2 } func AreStringsEqual(v1 *string, v2 *string) bool { if v1 == nil && v2 == nil { return true } if v1 == nil || v2 == nil { return false } return *v1 == *v2 } func AreBoolsEqual(v1 *bool, v2 *bool) bool { if v1 == nil && v2 == nil { return true } if v1 == nil || v2 == nil { return false } return *v1 == *v2 } func AreTimesEqual(v1 *time.Time, v2 *time.Time) bool { if v1 == nil && v2 == nil { return true } if v1 == nil || v2 == nil { return false } return v1.Equal(*v2) } func AreDurationsEqual(v1 *time.Duration, v2 *time.Duration) bool { if v1 == nil && v2 == nil { return true } if v1 == nil || v2 == nil { return false } return *v1 == *v2 } ================================================ FILE: pkg/lib/pointer/pointer.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package pointer import ( "reflect" "time" ) func Int(val int) *int { return &val } func Int8(val int8) *int8 { return &val } func Int16(val int16) *int16 { return &val } func Int32(val int32) *int32 { return &val } func Int64(val int64) *int64 { return &val } func Float64(val float64) *float64 { return &val } func Float32(val float32) *float32 { return &val } func String(val string) *string { return &val } func Bool(val bool) *bool { return &val } func Time(val time.Time) *time.Time { return &val } func Duration(val time.Duration) *time.Duration { return &val } // IndirectSafe dereferences if obj is a pointer, otherwise no-op func IndirectSafe(obj interface{}) interface{} { if obj == nil { return nil } return reflect.Indirect(reflect.ValueOf(obj)).Interface() } ================================================ FILE: pkg/lib/pointer/pointer_test.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package pointer import ( "testing" ) func TestIndirectSafe(t *testing.T) { IndirectSafe(nil) IndirectSafe("") IndirectSafe([]string{}) var s []string IndirectSafe(s) } ================================================ FILE: pkg/lib/print/print.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package print import ( "fmt" "os" "strings" "github.com/cortexlabs/cortex/pkg/lib/console" ) var _maxBoldLength = 200 func BoldFirstLine(msg string) { msgParts := strings.Split(msg, "\n") if len(msgParts[0]) > _maxBoldLength { fmt.Println(msg) return } fmt.Println(console.Bold(msgParts[0])) if len(msgParts) > 1 { fmt.Println(strings.Join(msgParts[1:], "\n")) } } func StderrBoldFirstLine(msg string) { msgParts := strings.Split(msg, "\n") if len(msgParts[0]) > _maxBoldLength { StderrPrintln(msg) return } StderrPrintln(console.Bold(msgParts[0])) if len(msgParts) > 1 { StderrPrintln(strings.Join(msgParts[1:], "\n")) } } func BoldFirstBlock(msg string) { msgParts := strings.Split(msg, "\n\n") if len(msgParts[0]) > _maxBoldLength { fmt.Println(msg) return } fmt.Println(console.Bold(msgParts[0])) if len(msgParts) > 1 { fmt.Println("\n" + strings.Join(msgParts[1:], "\n\n")) } } func StderrBoldFirstBlock(msg string) { msgParts := strings.Split(msg, "\n\n") if len(msgParts[0]) > _maxBoldLength { StderrPrintln(msg) return } StderrPrintln(console.Bold(msgParts[0])) if len(msgParts) > 1 { StderrPrintln("\n" + strings.Join(msgParts[1:], "\n\n")) } } func Dot() error { fmt.Print(".") return nil } func StderrPrintln(str string) { os.Stderr.WriteString(str + "\n") } ================================================ FILE: pkg/lib/prompt/errors.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package prompt import ( "github.com/cortexlabs/cortex/pkg/lib/errors" ) const ( ErrUserNoContinue = "prompt.user_no_continue" ErrUserCtrlC = "prompt.user_ctrl_c" ) func ErrorUserNoContinue() error { return errors.WithStack(&errors.Error{ Kind: ErrUserNoContinue, NoPrint: true, NoTelemetry: true, }) } func ErrorUserCtrlC() error { return errors.WithStack(&errors.Error{ Kind: ErrUserCtrlC, NoPrint: true, NoTelemetry: true, }) } ================================================ FILE: pkg/lib/prompt/prompt.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package prompt import ( "fmt" "os" "strings" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/exit" s "github.com/cortexlabs/cortex/pkg/lib/strings" input "github.com/cortexlabs/go-input" ) var _ui = &input.UI{ Writer: os.Stdout, Reader: os.Stdin, } type Options struct { Prompt string DefaultStr string HideDefault bool MaskDefault bool HideTyping bool MaskTyping bool TypingMaskVal string SkipTrailingNewline bool } func Prompt(opts *Options) string { prompt := opts.Prompt if opts.DefaultStr != "" && !opts.HideDefault { defaultStr := opts.DefaultStr if opts.MaskDefault { defaultStr = s.MaskString(defaultStr, 4) } prompt = fmt.Sprintf("%s [%s]", opts.Prompt, defaultStr) } val, err := _ui.Ask(prompt, &input.Options{ Default: opts.DefaultStr, Hide: opts.HideTyping, Mask: opts.MaskTyping, MaskVal: opts.TypingMaskVal, Required: false, HideDefault: true, HideOrder: true, Loop: false, SkipNewline: opts.SkipTrailingNewline, }) if err != nil { if errors.Message(err) == "interrupted" { exit.Error(ErrorUserCtrlC()) } if strings.Contains(errors.Message(err), "not a terminal") { err = errors.Append(err, "\n\nyou may be able to pass flags into this command to provide all required inputs and/or skip prompts (e.g. via `--yes`)") } exit.Error(err) } return val } func YesOrExit(prompt string, yesMessage string, noMessage string) { for { str := Prompt(&Options{ Prompt: prompt + " (y/n)", HideDefault: true, }) if strings.ToLower(str) == "y" { if yesMessage != "" { fmt.Println(yesMessage) } return } if strings.ToLower(str) == "n" { if noMessage != "" { fmt.Println(noMessage) } exit.Error(ErrorUserNoContinue()) } fmt.Println("please enter \"y\" or \"n\"") fmt.Println() } } func YesOrNo(prompt string, yesMessage string, noMessage string) bool { for true { str := Prompt(&Options{ Prompt: prompt + " (y/n)", HideDefault: true, }) if strings.ToLower(str) == "y" { if yesMessage != "" { fmt.Println(yesMessage) } return true } if strings.ToLower(str) == "n" { if noMessage != "" { fmt.Println(noMessage) } return false } fmt.Println("please enter \"y\" or \"n\"") fmt.Println() } return false } ================================================ FILE: pkg/lib/random/random.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package random import ( "math/rand" "time" ) const ( _uppercaseBytes = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" _lowercaseBytes = "abcdefghijklmnopqrstuvwxyz" _letterBytes = _lowercaseBytes + _uppercaseBytes _numberBytes = "0123456789" _stringBytes = _letterBytes + _numberBytes _letterIdxBits = 6 // 6 bits to represent a letter index _letterIdxMask = 1<<_letterIdxBits - 1 // All 1-bits, as many as _letterIdxBits _letterIdxMax = 63 / _letterIdxBits // # of letter indices fitting in 63 bits ) func randomString(n int, src rand.Source, charset string) string { b := make([]byte, n) for i, cache, remain := n-1, src.Int63(), _letterIdxMax; i >= 0; { if remain == 0 { cache, remain = src.Int63(), _letterIdxMax } if idx := int(cache & _letterIdxMask); idx < len(charset) { b[i] = charset[idx] i-- } cache >>= _letterIdxBits remain-- } return string(b) } // Digits generates a random string containing only digits func Digits(n int) string { return randomString(n, rand.NewSource(time.Now().UnixNano()), _numberBytes) } // Letters generates a random string containing only english alphabet characters (upper and lower case) func Letters(n int) string { return randomString(n, rand.NewSource(time.Now().UnixNano()), _letterBytes) } // LowercaseLetters generates a random string containing only lower case english alphabet characters func LowercaseLetters(n int) string { return randomString(n, rand.NewSource(time.Now().UnixNano()), _lowercaseBytes) } // String generates a random string containing both digits and english alphabet characters (upper and lower) func String(n int) string { return randomString(n, rand.NewSource(time.Now().UnixNano()), _stringBytes) } func LowercaseString(n int) string { return randomString(n, rand.NewSource(time.Now().UnixNano()), _lowercaseBytes+_numberBytes) } ================================================ FILE: pkg/lib/regex/regex.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package regex import ( "regexp" ) func MatchAnyRegex(s string, regexes []*regexp.Regexp) bool { for _, regex := range regexes { if regex.MatchString(s) { return true } } return false } var _leadingWhitespaceRegex = regexp.MustCompile(`^\s+`) func HasLeadingWhitespace(s string) bool { return _leadingWhitespaceRegex.MatchString(s) } var _trailingWhitespaceRegex = regexp.MustCompile(`\s+$`) func HasTrailingWhitespace(s string) bool { return _trailingWhitespaceRegex.MatchString(s) } // letters, numbers, spaces representable in UTF-8, and the following characters: _ . : / + - @ // = is not supported because it doesn't propagate to the NLB correctly (via the k8s service annotation) var _awsTagRegex = regexp.MustCompile(`^[\sa-zA-Z0-9_\-\.:/+@]+$`) func IsValidAWSTag(s string) bool { return _awsTagRegex.MatchString(s) } var _alphaNumericDashDotUnderscoreRegex = regexp.MustCompile(`^[a-zA-Z0-9_\-\.]+$`) func IsAlphaNumericDashDotUnderscore(s string) bool { return _alphaNumericDashDotUnderscoreRegex.MatchString(s) } var _alphaNumericDashUnderscoreRegex = regexp.MustCompile(`^[a-zA-Z0-9_\-]+$`) func IsAlphaNumericDashUnderscore(s string) bool { return _alphaNumericDashUnderscoreRegex.MatchString(s) } var _alphaNumericDotUnderscoreRegex = regexp.MustCompile(`^[a-zA-Z0-9_\.]+$`) func IsAlphaNumericDotUnderscore(s string) bool { return _alphaNumericDotUnderscoreRegex.MatchString(s) } var _alphaNumericDashRegex = regexp.MustCompile(`^[a-zA-Z0-9\-]+$`) func IsAlphaNumericDash(s string) bool { return _alphaNumericDashRegex.MatchString(s) } // used the evaluated form of // https://github.com/docker/distribution/blob/3150937b9f2b1b5b096b2634d0e7c44d4a0f89fb/reference/regexp.go#L68-L70 var _dockerValidImage = regexp.MustCompile( `^((?:(?:[a-z0-9]|[a-z0-9][a-z0-9-]*[a-z0-9])` + `(?:(?:\.(?:[a-z0-9]|[a-z0-9][a-z0-9-]*[a-z0-9]))+)` + `?(?::[0-9]+)?/)?[a-z0-9]` + `+(?:(?:(?:[._]|__|[-]*)[a-z0-9]+)+)` + `?(?:(?:/[a-z0-9]+(?:(?:(?:[._]|__|[-]*)[a-z0-9]+)+)?)+)?)` + `(?::([\w][\w.-]{0,127}))` + `?(?:@([A-Za-z][A-Za-z0-9]*(?:[-_+.][A-Za-z][A-Za-z0-9]*)*[:][[:xdigit:]]{32,}))?$`, ) func IsValidDockerImage(s string) bool { return _dockerValidImage.MatchString(s) } var _ecrPattern = regexp.MustCompile( `(^[a-zA-Z0-9][a-zA-Z0-9-_]*)\.dkr\.ecr(\-fips)?\.([a-zA-Z0-9][a-zA-Z0-9-_]*)\.amazonaws\.com(\.cn)?`, ) func IsValidECRURL(s string) bool { return _ecrPattern.MatchString(s) } ================================================ FILE: pkg/lib/regex/regex_test.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package regex import ( "strings" "testing" "github.com/stretchr/testify/assert" ) type regexpMatch struct { input string match bool subs []string } func TestHasLeadingWhitespace(t *testing.T) { testcases := []regexpMatch{ { input: " test", match: true, }, { input: " test", match: true, }, { input: "\ntest", match: true, }, { input: " test ", match: true, }, { input: " test ", match: true, }, { input: "\ntest\n", match: true, }, { input: "test", match: false, }, { input: "te st", match: false, }, { input: "te st", match: false, }, { input: "te\nst", match: false, }, { input: "_", match: false, }, { input: "test ", match: false, }, { input: "test ", match: false, }, { input: "test\n", match: false, }, { input: " ", match: true, }, { input: " ", match: true, }, { input: "\n", match: true, }, { input: "", match: false, }, } for i := range testcases { match := HasLeadingWhitespace(testcases[i].input) assert.Equal(t, testcases[i].match, match, "input: "+testcases[i].input) } } func TestHasTrailingWhitespace(t *testing.T) { testcases := []regexpMatch{ { input: " test", match: false, }, { input: " test", match: false, }, { input: "\ntest", match: false, }, { input: " test ", match: true, }, { input: " test ", match: true, }, { input: "\ntest\n", match: true, }, { input: "test", match: false, }, { input: "te st", match: false, }, { input: "te st", match: false, }, { input: "te\nst", match: false, }, { input: "_", match: false, }, { input: "test ", match: true, }, { input: "test ", match: true, }, { input: "test\n", match: true, }, { input: " ", match: true, }, { input: " ", match: true, }, { input: "\n", match: true, }, { input: "", match: false, }, } for i := range testcases { match := HasTrailingWhitespace(testcases[i].input) assert.Equal(t, testcases[i].match, match, "input: "+testcases[i].input) } } func TestAlphaNumericDashDotUnderscoreRegex(t *testing.T) { testcases := []regexpMatch{ { input: "generic.package.com", match: true, }, { input: "generic_package.com", match: true, }, { input: "_value-123", match: true, }, { input: "data.generic.package()", match: false, }, { input: "!variable", match: false, }, { input: "assignment=value", match: false, }, { input: "LibrayPackage@model.net", match: false, }, { input: "aaaabbbbcccABCDZX-123456789", match: true, }, } for i := range testcases { match := _alphaNumericDashDotUnderscoreRegex.MatchString(testcases[i].input) if match != testcases[i].match { t.Errorf("No match for %q", testcases[i].input) } } } func TestAlphaNumericDashUnderscoreRegex(t *testing.T) { testcases := []regexpMatch{ { input: "generic.package.com", match: false, }, { input: "generic_package.com", match: false, }, { input: "_value-123", match: true, }, { input: "data.generic.package()", match: false, }, { input: "!variable", match: false, }, { input: "assignment=value", match: false, }, { input: "LibrayPackage@model.net", match: false, }, { input: "aaaabbbbcccABCDZX-123456789", match: true, }, { input: "word1-word2_word3_word4", match: true, }, { input: "____-----____", match: true, }, { input: "(word)", match: false, }, } for i := range testcases { match := _alphaNumericDashUnderscoreRegex.MatchString(testcases[i].input) if match != testcases[i].match { t.Errorf("No match for %q", testcases[i].input) } } } func TestValidDockerImage(t *testing.T) { testcases := []regexpMatch{ { input: "", match: false, }, { input: "short", match: true, }, { input: "simple/name", match: true, }, { input: "library/ubuntu", match: true, }, { input: "library/ubuntu:latest", match: true, }, { input: "docker/stevvooe/app", match: true, }, { input: "aa/aa/aa/aa/aa/aa/aa/aa/aa/bb/bb/bb/bb/bb/bb", match: true, }, { input: "aa/aa/bb/bb/bb", match: true, }, { input: "a/a/a/a", match: true, }, { input: "a/a/a/a/", match: false, }, { input: "a//a/a", match: false, }, { input: "a", match: true, }, { input: "a/aa", match: true, }, { input: "a/aa/a", match: true, }, { input: "foo.com", match: true, }, { input: "foo.com/", match: false, }, { input: "foo.com:8080/bar", match: true, }, { input: "foo.com:http/bar", match: false, }, { input: "foo.com/bar", match: true, }, { input: "foo.com/bar/baz", match: true, }, { input: "localhost:8080/bar", match: true, }, { input: "sub-dom1.foo.com/bar/baz/quux", match: true, }, { input: "blog.foo.com/bar/baz", match: true, }, { input: "a^a", match: false, }, { input: "aa/asdf$$^/aa", match: false, }, { input: "asdf$$^/aa", match: false, }, { input: "aa-a/a", match: true, }, { input: strings.Repeat("a/", 128) + "a", match: true, }, { input: "a-/a/a/a", match: false, }, { input: "foo.com/a-/a/a", match: false, }, { input: "-foo/bar", match: false, }, { input: "foo/bar-", match: false, }, { input: "foo-/bar", match: false, }, { input: "foo/-bar", match: false, }, { input: "_foo/bar", match: false, }, { input: "foo_bar", match: true, }, { input: "foo_bar.com", match: true, }, { input: "foo_bar.com:8080/app", match: false, }, { input: "foo.com/foo_bar", match: true, }, { input: "____/____", match: false, }, { input: "_docker/_docker", match: false, }, { input: "docker_/docker_", match: false, }, { input: "b.gcr.io/test.example.com/my-app", match: true, }, { input: "xn--n3h.com/myimage", // ☃.com in punycode match: true, }, { input: "xn--7o8h.com/myimage", // 🐳.com in punycode match: true, }, { input: "example.com/xn--7o8h.com/myimage", // 🐳.com in punycode match: true, }, { input: "example.com/some_separator__underscore/myimage", match: true, }, { input: "example.com/__underscore/myimage", match: false, }, { input: "example.com/..dots/myimage", match: false, }, { input: "example.com/.dots/myimage", match: false, }, { input: "example.com/nodouble..dots/myimage", match: false, }, { input: "example.com/nodouble..dots/myimage", match: false, }, { input: "docker./docker", match: false, }, { input: ".docker/docker", match: false, }, { input: "docker-/docker", match: false, }, { input: "-docker/docker", match: false, }, { input: "do..cker/docker", match: false, }, { input: "do__cker:8080/docker", match: false, }, { input: "do__cker/docker", match: true, }, { input: "b.gcr.io/test.example.com/my-app", match: true, }, { input: "registry.io/foo/project--id.module--name.ver---sion--name", match: true, }, { input: "registry.com:8080/myapp:tag", match: true, subs: []string{"registry.com:8080/myapp", "tag", ""}, }, { input: "registry.com:8080/myapp@sha256:be178c0543eb17f5f3043021c9e5fcf30285e557a4fc309cce97ff9ca6182912", match: true, subs: []string{"registry.com:8080/myapp", "", "sha256:be178c0543eb17f5f3043021c9e5fcf30285e557a4fc309cce97ff9ca6182912"}, }, { input: "registry.com:8080/myapp:tag2@sha256:be178c0543eb17f5f3043021c9e5fcf30285e557a4fc309cce97ff9ca6182912", match: true, subs: []string{"registry.com:8080/myapp", "tag2", "sha256:be178c0543eb17f5f3043021c9e5fcf30285e557a4fc309cce97ff9ca6182912"}, }, { input: "registry.com:8080/myapp@sha256:badbadbadbad", match: false, }, { input: "registry.com:8080/myapp:invalid~tag", match: false, }, { input: "bad_hostname.com:8080/myapp:tag", match: false, }, { input:// localhost treated as name, missing tag with 8080 as tag "localhost:8080@sha256:be178c0543eb17f5f3043021c9e5fcf30285e557a4fc309cce97ff9ca6182912", match: true, subs: []string{"localhost", "8080", "sha256:be178c0543eb17f5f3043021c9e5fcf30285e557a4fc309cce97ff9ca6182912"}, }, { input: "localhost:8080/name@sha256:be178c0543eb17f5f3043021c9e5fcf30285e557a4fc309cce97ff9ca6182912", match: true, subs: []string{"localhost:8080/name", "", "sha256:be178c0543eb17f5f3043021c9e5fcf30285e557a4fc309cce97ff9ca6182912"}, }, { input: "localhost:http/name@sha256:be178c0543eb17f5f3043021c9e5fcf30285e557a4fc309cce97ff9ca6182912", match: false, }, { // localhost will be treated as an image name without a host input: "localhost@sha256:be178c0543eb17f5f3043021c9e5fcf30285e557a4fc309cce97ff9ca6182912", match: true, subs: []string{"localhost", "", "sha256:be178c0543eb17f5f3043021c9e5fcf30285e557a4fc309cce97ff9ca6182912"}, }, { input: "registry.com:8080/myapp@bad", match: false, }, { input: "registry.com:8080/myapp@2bad", match: false, // Support this as valid? }, { input: "680880929103.dkr.ecr.eu-central-1.amazonaws.com/cortexlabs/async-gateway:latest", match: true, }, } for i := range testcases { match := _dockerValidImage.MatchString(testcases[i].input) if match != testcases[i].match { t.Errorf("No match for %q", testcases[i].input) } } } func TestValidECR(t *testing.T) { testcases := []regexpMatch{ { input: "", match: false, }, { input: "library/ubuntu:latest", match: false, }, { input: "registry.com:8080/myapp:tag", match: false, }, { input: "680880929102.dkr.ecr.eu-central-1.amazonaws.com/cortexlabs/python-handler-cpu:latest", match: true, }, { input: "localhost@sha256:be178c0543eb17f5f3043021c9e5fcf30285e557a4fc309cce97ff9ca6182912", match: false, }, { input: "680880929102.dkr.ecr.eu-central-1.com/cortexlabs/image", match: false, }, { input: "680880929102.dkr.ecr.us-east-1.amazonaws.com/registry", match: true, }, { input: "1234567.dkr.ecr.us-west-1.amazonaws.com/registry/image:123", match: true, }, { input: "680880929102.dkr.ecr.us-east-1.amazonaws.com", match: true, }, } for i := range testcases { match := _ecrPattern.MatchString(testcases[i].input) if match != testcases[i].match { t.Errorf("No match for %q", testcases[i].input) } } } ================================================ FILE: pkg/lib/requests/errors.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package requests import ( "fmt" "net/url" "strings" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/urls" ) const ( _errStrCantMakeRequest = "unable to make request" _errStrRead = "unable to read" ) func errStrFailedToConnect(u url.URL) string { return "failed to connect to " + urls.TrimQueryParamsURL(u) } const ( ErrResponseUnknown = "requests.response_unknown" ) func ErrorResponseUnknown(body string, statusCode int) error { msg := body if strings.TrimSpace(body) == "" { msg = fmt.Sprintf("empty response (status code %d)", statusCode) } return errors.WithStack(&errors.Error{ Kind: ErrResponseUnknown, Message: msg, }) } ================================================ FILE: pkg/lib/requests/requests.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package requests import ( "crypto/tls" "io/ioutil" "net/http" "time" "github.com/cortexlabs/cortex/pkg/lib/errors" ) func MakeRequest(request *http.Request) (http.Header, []byte, error) { client := http.Client{ Timeout: 5 * time.Second, Transport: &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, }, } response, err := client.Do(request) if err != nil { return nil, nil, errors.Wrap(err, errStrFailedToConnect(*request.URL)) } defer response.Body.Close() if response.StatusCode != 200 { bodyBytes, err := ioutil.ReadAll(response.Body) if err != nil { return nil, nil, errors.Wrap(err, _errStrRead) } return nil, nil, ErrorResponseUnknown(string(bodyBytes), response.StatusCode) } bodyBytes, err := ioutil.ReadAll(response.Body) if err != nil { return nil, nil, errors.Wrap(err, _errStrRead) } return response.Header, bodyBytes, nil } ================================================ FILE: pkg/lib/sets/strset/strset.go ================================================ /* Copyright 2017 ScyllaDB Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. Modifications Copyright 2022 Cortex Labs, Inc. */ package strset import ( "fmt" "math" "sort" "strings" ) type Set map[string]struct{} var _keyExists = struct{}{} // New creates and initializes a new Set. func New(ts ...string) Set { s := make(Set) s.Add(ts...) return s } func FromSlice(items []string) Set { return New(items...) } // NewWithSize creates a new Set and gives make map a size hint. func NewWithSize(size int) Set { return make(Set, size) } // Add includes the specified items (one or more) to the Set. The underlying // Set s is modified. If passed nothing it silently returns. func (s Set) Add(items ...string) { for _, item := range items { s[item] = _keyExists } } // Remove deletes the specified items from the Set. The underlying Set s is // modified. If passed nothing it silently returns. func (s Set) Remove(items ...string) { for _, item := range items { delete(s, item) } } // GetOne returns an item from the set or "" if the set is empty. func (s Set) GetOne() string { for item := range s { return item } return "" } // GetOne2 returns an item from the set. The second value is a bool that is // true if an item exists in the set, or false if the set is empty. func (s Set) GetOne2() (string, bool) { for item := range s { return item, true } return "", false } // Pop deletes and returns an item from the Set. The underlying Set s is // modified. If Set is empty, the zero value is returned. func (s Set) Pop() string { for item := range s { delete(s, item) return item } return "" } // Pop2 tries to delete and return an item from the Set. The underlying Set s // is modified. The second value is a bool that is true if the item existed in // the set, and false if not. If Set is empty, the zero value and false are // returned. func (s Set) Pop2() (string, bool) { for item := range s { delete(s, item) return item, true } return "", false } // Has looks for the existence of items passed. It returns false if nothing is // passed. For multiple items it returns true only if all of the items exist. func (s Set) Has(items ...string) bool { has := false for _, item := range items { if _, has = s[item]; !has { break } } return has } // HasWithPrefix checks if at least one element of the set is the prefix of any of the passed items. // It returns false if nothing is passed. func (s Set) HasWithPrefix(items ...string) bool { for _, prefix := range items { for k := range s { if strings.HasPrefix(prefix, k) { return true } } } return false } // HasAny looks for the existence of any of the items passed. // It returns false if nothing is passed. // For multiple items it returns true if any of the items exist. func (s Set) HasAny(items ...string) bool { has := false for _, item := range items { if _, has = s[item]; has { break } } return has } // Clear removes all items from the Set. func (s *Set) Clear() { *s = make(Set) } // IsEqual test whether s and t are the same in size and have the same items. func (s Set) IsEqual(t Set) bool { if len(s) != len(t) { return false } for item := range s { if !t.Has(item) { return false } } return true } // IsSubset tests whether t is a subset of s. func (s Set) IsSubset(t Set) bool { if len(s) < len(t) { return false } for item := range t { if !s.Has(item) { return false } } return true } // IsSuperset tests whether t is a superset of s. func (s Set) IsSuperset(t Set) bool { return t.IsSubset(s) } // Copy returns a new Set with a copy of s. func (s Set) Copy() Set { u := make(Set, len(s)) for item := range s { u[item] = _keyExists } return u } // String returns a string representation of s func (s Set) String() string { v := make([]string, 0, len(s)) for item := range s { v = append(v, fmt.Sprintf("%v", item)) } return fmt.Sprintf("[%s]", strings.Join(v, ", ")) } // List returns a slice of all items. func (s Set) Slice() []string { v := make([]string, 0, len(s)) for item := range s { v = append(v, item) } return v } // List returns a sorted slice of all items (a to z). func (s Set) SliceSorted() []string { v := s.Slice() sort.Strings(v) return v } // Merge is like Union, however it modifies the current Set it's applied on // with the given t Set. func (s Set) Merge(sets ...Set) { for _, set := range sets { for item := range set { s[item] = _keyExists } } } // Subtract removes the Set items contained in sets from Set s func (s Set) Subtract(sets ...Set) { for _, set := range sets { for item := range set { delete(s, item) } } } // Remove items until len(s) <= targetLen func (s Set) Shrink(targetLen int) { for len(s) > targetLen { s.Pop() } } // remove items alphabetically until len(s) <= targetLen func (s Set) ShrinkSorted(targetLen int) { if len(s) <= targetLen { return } sorted := s.SliceSorted() extras := sorted[targetLen:] s.Remove(extras...) } // Union is the merger of multiple sets. It returns a new set with all the // elements present in all the sets that are passed. func Union(sets ...Set) Set { maxPos := -1 maxSize := 0 // find which set is the largest and its size for i, set := range sets { if l := len(set); l > maxSize { maxSize = l maxPos = i } } if maxSize == 0 { return make(Set) } u := sets[maxPos].Copy() for i, set := range sets { if i == maxPos { continue } for item := range set { u[item] = _keyExists } } return u } // Difference returns a new set which contains items which are in the first // set but not in the others. func Difference(set1 Set, sets ...Set) Set { s := set1.Copy() for _, set := range sets { s.Subtract(set) } return s } // Intersection returns a new set which contains items that only exist in all // given sets. func Intersection(sets ...Set) Set { minPos := -1 minSize := math.MaxInt64 for i, set := range sets { if l := len(set); l < minSize { minSize = l minPos = i } } if minSize == math.MaxInt64 || minSize == 0 { return make(Set) } t := sets[minPos].Copy() for i, set := range sets { if i == minPos { continue } for item := range t { if _, has := set[item]; !has { delete(t, item) } } } return t } // SymmetricDifference returns a new set which s is the difference of items // which are in one of either, but not in both. func SymmetricDifference(s Set, t Set) Set { u := Difference(s, t) v := Difference(t, s) return Union(u, v) } ================================================ FILE: pkg/lib/sets/strset/strset_test.go ================================================ /* Copyright 2017 ScyllaDB Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. Modifications Copyright 2022 Cortex Labs, Inc. */ package strset_test import ( "testing" "github.com/cortexlabs/cortex/pkg/lib/sets/strset" "github.com/stretchr/testify/require" ) // Also tests Add func TestNew(t *testing.T) { set := strset.New() require.Equal(t, 0, len(set)) set = strset.New("a", "b", "a") require.Equal(t, 2, len(set)) if _, ok := set["a"]; !ok { require.FailNow(t, "a not found in set") } if _, ok := set["b"]; !ok { require.FailNow(t, "b not found in set") } } func TestAdd(t *testing.T) { set := strset.New() set.Add("a") set.Add("b", "c") require.Equal(t, set, strset.New("a", "b", "c")) } func TestRemove(t *testing.T) { set := strset.New("a", "b") set.Remove("c") require.Equal(t, set, strset.New("a", "b")) set.Remove() require.Equal(t, set, strset.New("a", "b")) set.Remove("a") require.Equal(t, set, strset.New("b")) set.Add("a") set.Remove("a", "b") require.Equal(t, set, strset.New()) } func TestPop(t *testing.T) { set := strset.New("a", "b") p := set.Pop() require.Contains(t, []string{"a", "b"}, p) require.Equal(t, 1, len(set)) p = set.Pop() require.Contains(t, []string{"a", "b"}, p) require.Equal(t, 0, len(set)) p = set.Pop() require.Equal(t, "", p) require.Equal(t, 0, len(set)) } func TestPop2(t *testing.T) { set := strset.New("a", "b") p, ok := set.Pop2() require.Contains(t, []string{"a", "b"}, p) require.Equal(t, 1, len(set)) require.True(t, ok) p, ok = set.Pop2() require.Contains(t, []string{"a", "b"}, p) require.Equal(t, 0, len(set)) require.True(t, ok) p, ok = set.Pop2() require.Equal(t, "", p) require.Equal(t, 0, len(set)) require.False(t, ok) } func TestHas(t *testing.T) { set := strset.New("a", "b", "c") require.True(t, set.Has("a")) require.True(t, set.Has("a", "b")) require.False(t, set.Has("z")) require.False(t, set.Has("a", "z")) } func TestHasAny(t *testing.T) { set := strset.New("a", "b", "c") require.True(t, set.HasAny("a")) require.True(t, set.HasAny("a", "b")) require.False(t, set.HasAny("z")) require.True(t, set.HasAny("a", "z")) } func TestClear(t *testing.T) { set := strset.New("a", "b", "c") require.Equal(t, 3, len(set)) set.Clear() require.Equal(t, 0, len(set)) } func TestIsEqual(t *testing.T) { set1 := strset.New("a", "b", "c") set2 := strset.New("a", "b", "c") set3 := strset.New("a", "b") set4 := strset.New("a", "b", "z") require.True(t, set1.IsEqual(set2)) require.False(t, set1.IsEqual(set3)) require.False(t, set1.IsEqual(set4)) } func TestIsSubset(t *testing.T) { set1 := strset.New("a", "b", "c") set2 := strset.New("a", "b", "c") set3 := strset.New("a", "b") set4 := strset.New("a", "b", "z") set5 := strset.New("a", "b", "c", "d") require.True(t, set1.IsSubset(set2)) require.True(t, set1.IsSubset(set3)) require.False(t, set1.IsSubset(set4)) require.False(t, set1.IsSubset(set5)) } func TestIsSuperset(t *testing.T) { set1 := strset.New("a", "b", "c") set2 := strset.New("a", "b", "c") set3 := strset.New("a", "b") set4 := strset.New("a", "b", "z") set5 := strset.New("a", "b", "c", "d") require.True(t, set1.IsSuperset(set2)) require.False(t, set1.IsSuperset(set3)) require.False(t, set1.IsSuperset(set4)) require.True(t, set1.IsSuperset(set5)) } func TestCopy(t *testing.T) { set := strset.New() cset := set.Copy() require.Equal(t, 0, len(cset)) set = strset.New("a", "b") cset = set.Copy() require.Equal(t, 2, len(cset)) if _, ok := set["a"]; !ok { require.FailNow(t, "a not found in set") } if _, ok := set["b"]; !ok { require.FailNow(t, "b not found in set") } } func TestSlice(t *testing.T) { set := strset.New() require.Equal(t, set.Slice(), []string{}) set.Add("a") require.Equal(t, set.Slice(), []string{"a"}) set.Add("a", "b") require.ElementsMatch(t, set.Slice(), []string{"a", "b"}) } func TestMerge(t *testing.T) { set := strset.New() emptySet := strset.New() set.Merge(emptySet) require.Equal(t, 0, len(set)) set.Merge(strset.New("a")) require.Equal(t, 1, len(set)) set.Merge(emptySet) require.Equal(t, 1, len(set)) set.Add("a", "b", "c") set.Merge(strset.New("e", "e", "d")) require.Equal(t, set, strset.New("a", "b", "c", "e", "d")) set.Merge(strset.New("a", "e")) require.Equal(t, set, strset.New("a", "b", "c", "e", "d")) set.Merge(strset.New("a", "e", "i"), strset.New("o", "u"), strset.New("sometimes y")) require.Equal(t, set, strset.New("a", "b", "c", "e", "d", "i", "o", "u", "sometimes y")) } func TestSubtract(t *testing.T) { set := strset.New("a", "b", "c") set.Subtract(strset.New("z", "a", "b")) require.Equal(t, set, strset.New("c")) set.Subtract(strset.New("x")) require.Equal(t, set, strset.New("c")) } func TestShrink(t *testing.T) { set := strset.New("a", "b", "c", "d") set.Shrink(2) require.Len(t, set, 2) set = strset.New("g", "f", "e", "d", "c", "b", "a") set.ShrinkSorted(3) require.Len(t, set, 3) set = strset.New("a", "b") set.Shrink(2) require.Equal(t, set, strset.New("a", "b")) set = strset.New("a") set.Shrink(2) require.Equal(t, set, strset.New("a")) set = strset.New() set.Shrink(2) require.Len(t, set, 0) } func TestShrinkSorted(t *testing.T) { for i := 0; i < 10; i++ { set := strset.New("g", "f", "e", "d", "c", "b", "a") set.ShrinkSorted(2) require.Equal(t, set, strset.New("a", "b")) set = strset.New("g", "f", "e", "d", "c", "b", "a") set.ShrinkSorted(3) require.Equal(t, set, strset.New("a", "b", "c")) } set := strset.New("a", "b") set.ShrinkSorted(2) require.Equal(t, set, strset.New("a", "b")) set = strset.New("a") set.ShrinkSorted(2) require.Equal(t, set, strset.New("a")) set = strset.New() set.ShrinkSorted(2) require.Len(t, set, 0) } func TestUnion(t *testing.T) { require.Equal(t, len(strset.Union(strset.New(), strset.New())), 0) require.Equal(t, strset.Union(strset.New("a", "b"), strset.New()), strset.New("a", "b")) require.Equal(t, strset.Union(strset.New(), strset.New("a", "b")), strset.New("a", "b")) require.Equal(t, strset.Union(strset.New("a", "a"), strset.New("a", "b")), strset.New("a", "b")) require.Equal(t, strset.Union(strset.New("a"), strset.New("b")), strset.New("a", "b")) } func TestDifference(t *testing.T) { set1 := strset.New("a", "b", "c") set2 := strset.New("z", "a", "b") d := strset.Difference(set1, set2) require.Equal(t, d, strset.New("c")) d = strset.Difference(set2, set1) require.Equal(t, d, strset.New("z")) d = strset.Difference(set1, set1) require.Equal(t, d, strset.New()) } func TestIntersection(t *testing.T) { set1 := strset.New("a", "b", "c") set2 := strset.New("a", "x", "y") set3 := strset.New("z", "b", "c") set4 := strset.New("z", "b", "w") d := strset.Intersection(set1, set2) require.Equal(t, d, strset.New("a")) d = strset.Intersection(set1, set3) require.Equal(t, d, strset.New("b", "c")) d = strset.Intersection(set2, set3) require.Equal(t, d, strset.New()) d = strset.Intersection(set1, set3, set4) require.Equal(t, d, strset.New("b")) } ================================================ FILE: pkg/lib/sets/strset/threadsafe/strset.go ================================================ /* Copyright 2017 ScyllaDB Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. Modifications Copyright 2022 Cortex Labs, Inc. */ package threadsafe import ( "sync" "github.com/cortexlabs/cortex/pkg/lib/sets/strset" ) // New creates and initializes a new Set. type Set struct { sync.RWMutex s strset.Set } func New(ts ...string) *Set { set := Set{} set.s = strset.New(ts...) return &set } func FromSlice(items []string) *Set { return New(items...) } // NewWithSize creates a new Set and gives make map a size hint. func NewWithSize(size int) *Set { set := Set{} set.s = strset.NewWithSize(size) return &set } func (s *Set) Len() int { s.RLock() defer s.RUnlock() return len(s.s) } func (s *Set) ToStrset() strset.Set { s.RLock() defer s.RUnlock() return s.s.Copy() } // Add includes the specified items (one or more) to the Set. The underlying // Set s is modified. If passed nothing it silently returns. func (s *Set) Add(items ...string) { s.Lock() defer s.Unlock() s.s.Add(items...) } // Remove deletes the specified items from the Set. The underlying Set s is // modified. If passed nothing it silently returns. func (s *Set) Remove(items ...string) { s.Lock() defer s.Unlock() s.s.Remove(items...) } // GetOne returns an item from the set or "" if the set is empty. func (s *Set) GetOne() string { s.RLock() defer s.RUnlock() return s.s.GetOne() } // GetOne2 returns an item from the set. The second value is a bool that is // true if an item exists in the set, or false if the set is empty. func (s *Set) GetOne2() (string, bool) { s.RLock() defer s.RUnlock() return s.s.GetOne2() } // Pop deletes and returns an item from the Set. The underlying Set s is // modified. If Set is empty, the zero value is returned. func (s *Set) Pop() string { s.Lock() defer s.Unlock() return s.s.Pop() } // Pop2 tries to delete and return an item from the Set. The underlying Set s // is modified. The second value is a bool that is true if the item existed in // the set, and false if not. If Set is empty, the zero value and false are // returned. func (s *Set) Pop2() (string, bool) { s.Lock() defer s.Unlock() return s.s.Pop2() } // Has looks for the existence of items passed. It returns false if nothing is // passed. For multiple items it returns true only if all of the items exist. func (s *Set) Has(items ...string) bool { s.RLock() defer s.RUnlock() return s.s.Has(items...) } // HasAny looks for the existence of any of the items passed. // It returns false if nothing is passed. // For multiple items it returns true if any of the items exist. func (s *Set) HasAny(items ...string) bool { s.RLock() defer s.RUnlock() return s.s.HasAny(items...) } // Clear removes all items from the Set. func (s *Set) Clear() { s.Lock() defer s.Unlock() s.s.Clear() } // IsEqual test whether s and t are the same in size and have the same items. func (s *Set) IsEqual(t strset.Set) bool { s.RLock() defer s.RUnlock() return s.s.IsEqual(t) } // IsEqualThreadsafe test whether s and t are the same in size and have the same items. func (s *Set) IsEqualThreadsafe(t *Set) bool { s.RLock() defer s.RUnlock() t.RLock() defer t.RUnlock() return s.s.IsEqual(t.s) } // IsSubset tests whether t is a subset of s. func (s *Set) IsSubset(t strset.Set) bool { s.RLock() defer s.RUnlock() return s.s.IsSubset(t) } // IsSubsetThreadsafe tests whether t is a subset of s. func (s *Set) IsSubsetThreadsafe(t *Set) bool { s.RLock() defer s.RUnlock() t.RLock() defer t.RUnlock() return s.s.IsSubset(t.s) } // IsSuperset tests whether t is a superset of s. func (s *Set) IsSuperset(t strset.Set) bool { s.RLock() defer s.RUnlock() return s.s.IsSuperset(t) } // IsSupersetThreadsafe tests whether t is a superset of s. func (s *Set) IsSupersetThreadsafe(t *Set) bool { s.RLock() defer s.RUnlock() t.RLock() defer t.RUnlock() return s.s.IsSuperset(t.s) } // Copy returns a new Set with a copy of s. func (s *Set) Copy() strset.Set { s.RLock() defer s.RUnlock() return s.s.Copy() } // CopyToThreadsafe returns a new Set with a copy of s. func (s *Set) CopyToThreadsafe() *Set { s.RLock() defer s.RUnlock() newSet := Set{} newSet.s = s.s.Copy() return &newSet } // String returns a string representation of s func (s *Set) String() string { s.RLock() defer s.RUnlock() return s.s.String() } // List returns a slice of all items. func (s *Set) Slice() []string { s.RLock() defer s.RUnlock() return s.s.Slice() } // List returns a sorted slice of all items (a to z). func (s *Set) SliceSorted() []string { s.RLock() defer s.RUnlock() return s.s.SliceSorted() } // Merge is like Union, however it modifies the current Set it's applied on // with the given t Set. func (s *Set) Merge(sets ...strset.Set) { s.Lock() defer s.Unlock() s.s.Merge(sets...) } // MergeThreadsafe is like UnionThreadsafe, however it modifies the current Set it's applied on // with the given t Set. func (s *Set) MergeThreadsafe(sets ...*Set) { s.Lock() defer s.Unlock() for _, set := range sets { set.RLock() s.s.Merge(set.s) set.RUnlock() } } // Subtract removes the Set items contained in sets from Set s func (s *Set) Subtract(sets ...strset.Set) { s.Lock() defer s.Unlock() s.s.Subtract(sets...) } // SubtractThreadsafe removes the Set items contained in sets from Set s func (s *Set) SubtractThreadsafe(sets ...*Set) { s.Lock() defer s.Unlock() for _, set := range sets { set.RLock() s.s.Subtract(set.s) set.RUnlock() } } // Remove items until len(s) <= targetLen func (s *Set) Shrink(targetLen int) { s.Lock() defer s.Unlock() s.s.Shrink(targetLen) } // remove items alphabetically until len(s) <= targetLen func (s *Set) ShrinkSorted(targetLen int) { s.Lock() defer s.Unlock() s.s.ShrinkSorted(targetLen) } // Union is the merger of multiple sets. It returns a new set with all the // elements present in all the sets that are passed. func Union(set1 *Set, sets ...strset.Set) *Set { finalSet := set1.CopyToThreadsafe() for _, set := range sets { finalSet.s.Merge(set) } return finalSet } // UnionThreadsafe is the merger of multiple sets. It returns a new set with all the // elements present in all the sets that are passed. func UnionThreadsafe(sets ...*Set) *Set { finalSet := New() for _, set := range sets { set.RLock() finalSet.s.Merge(set.s) set.RUnlock() } return finalSet } // Difference returns a new set which contains items which are in the first // set but not in the others. func Difference(set1 *Set, sets ...strset.Set) *Set { s := set1.CopyToThreadsafe() for _, set := range sets { s.s.Subtract(set) } return s } // DifferenceThreadsafe returns a new set which contains items which are in in the first // set but not in the others. func DifferenceThreadsafe(set1 *Set, sets ...*Set) *Set { s := set1.CopyToThreadsafe() for _, set := range sets { set.RLock() s.s.Subtract(set.s) set.RUnlock() } return s } // Intersection returns a new set which contains items that only exist in all // given sets. func Intersection(set1 *Set, sets ...strset.Set) *Set { t := set1.CopyToThreadsafe() for _, set := range sets { for item := range t.s { if _, has := set[item]; !has { delete(t.s, item) } } } return t } // IntersectionThreadsafe returns a new set which contains items that only exist in all // given sets. func IntersectionThreadsafe(set1 *Set, sets ...*Set) *Set { t := set1.CopyToThreadsafe() for _, set := range sets { set.RLock() for item := range t.s { if _, has := set.s[item]; !has { delete(t.s, item) } } set.RUnlock() } return t } // SymmetricDifferenceThreadsafe returns a new set which s is the difference of items // which are in one of either, but not in both. func SymmetricDifferenceThreadsafe(s *Set, t *Set) *Set { u := DifferenceThreadsafe(s, t) v := DifferenceThreadsafe(t, s) return UnionThreadsafe(u, v) } ================================================ FILE: pkg/lib/sets/strset/threadsafe/strset_test.go ================================================ /* Copyright 2017 ScyllaDB Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. Modifications Copyright 2022 Cortex Labs, Inc. */ package threadsafe_test import ( "testing" "github.com/cortexlabs/cortex/pkg/lib/sets/strset" "github.com/cortexlabs/cortex/pkg/lib/sets/strset/threadsafe" "github.com/stretchr/testify/require" ) // Also tests Add func TestNew(t *testing.T) { set := threadsafe.New() require.Equal(t, 0, set.Len()) set = threadsafe.New("a", "b", "a") require.Equal(t, 2, set.Len()) if !set.Has("a") { require.FailNow(t, "a not found in set") } if !set.Has("b") { require.FailNow(t, "b not found in set") } } func TestAdd(t *testing.T) { set := threadsafe.New() set.Add("a") set.Add("b", "c") require.Equal(t, set, threadsafe.New("a", "b", "c")) } func TestRemove(t *testing.T) { set := threadsafe.New("a", "b") set.Remove("c") require.Equal(t, set, threadsafe.New("a", "b")) set.Remove() require.Equal(t, set, threadsafe.New("a", "b")) set.Remove("a") require.Equal(t, set, threadsafe.New("b")) set.Add("a") set.Remove("a", "b") require.Equal(t, set, threadsafe.New()) } func TestPop(t *testing.T) { set := threadsafe.New("a", "b") p := set.Pop() require.Contains(t, []string{"a", "b"}, p) require.Equal(t, 1, set.Len()) p = set.Pop() require.Contains(t, []string{"a", "b"}, p) require.Equal(t, 0, set.Len()) p = set.Pop() require.Equal(t, "", p) require.Equal(t, 0, set.Len()) } func TestPop2(t *testing.T) { set := threadsafe.New("a", "b") p, ok := set.Pop2() require.Contains(t, []string{"a", "b"}, p) require.Equal(t, 1, set.Len()) require.True(t, ok) p, ok = set.Pop2() require.Contains(t, []string{"a", "b"}, p) require.Equal(t, 0, set.Len()) require.True(t, ok) p, ok = set.Pop2() require.Equal(t, "", p) require.Equal(t, 0, set.Len()) require.False(t, ok) } func TestHas(t *testing.T) { set := threadsafe.New("a", "b", "c") require.True(t, set.Has("a")) require.True(t, set.Has("a", "b")) require.False(t, set.Has("z")) require.False(t, set.Has("a", "z")) } func TestHasAny(t *testing.T) { set := threadsafe.New("a", "b", "c") require.True(t, set.HasAny("a")) require.True(t, set.HasAny("a", "b")) require.False(t, set.HasAny("z")) require.True(t, set.HasAny("a", "z")) } func TestClear(t *testing.T) { set := threadsafe.New("a", "b", "c") require.Equal(t, 3, set.Len()) set.Clear() require.Equal(t, 0, set.Len()) } func TestIsEqual(t *testing.T) { set1 := threadsafe.New("a", "b", "c") set2 := strset.New("a", "b", "c") set3 := strset.New("a", "b") set4 := strset.New("a", "b", "z") require.True(t, set1.IsEqual(set2)) require.False(t, set1.IsEqual(set3)) require.False(t, set1.IsEqual(set4)) } func IsEqualThreadsafe(t *testing.T) { set1 := threadsafe.New("a", "b", "c") set2 := threadsafe.New("a", "b", "c") set3 := threadsafe.New("a", "b") set4 := threadsafe.New("a", "b", "z") require.True(t, set1.IsEqualThreadsafe(set2)) require.False(t, set1.IsEqualThreadsafe(set3)) require.False(t, set1.IsEqualThreadsafe(set4)) } func TestIsSubset(t *testing.T) { set1 := threadsafe.New("a", "b", "c") set2 := strset.New("a", "b", "c") set3 := strset.New("a", "b") set4 := strset.New("a", "b", "z") set5 := strset.New("a", "b", "c", "d") require.True(t, set1.IsSubset(set2)) require.True(t, set1.IsSubset(set3)) require.False(t, set1.IsSubset(set4)) require.False(t, set1.IsSubset(set5)) } func IsSubsetThreadsafe(t *testing.T) { set1 := threadsafe.New("a", "b", "c") set2 := threadsafe.New("a", "b", "c") set3 := threadsafe.New("a", "b") set4 := threadsafe.New("a", "b", "z") set5 := threadsafe.New("a", "b", "c", "d") require.True(t, set1.IsSubsetThreadsafe(set2)) require.True(t, set1.IsSubsetThreadsafe(set3)) require.False(t, set1.IsSubsetThreadsafe(set4)) require.False(t, set1.IsSubsetThreadsafe(set5)) } func TestIsSuperset(t *testing.T) { set1 := threadsafe.New("a", "b", "c") set2 := strset.New("a", "b", "c") set3 := strset.New("a", "b") set4 := strset.New("a", "b", "z") set5 := strset.New("a", "b", "c", "d") require.True(t, set1.IsSuperset(set2)) require.False(t, set1.IsSuperset(set3)) require.False(t, set1.IsSuperset(set4)) require.True(t, set1.IsSuperset(set5)) } func IsSupersetThreadsafe(t *testing.T) { set1 := threadsafe.New("a", "b", "c") set2 := threadsafe.New("a", "b", "c") set3 := threadsafe.New("a", "b") set4 := threadsafe.New("a", "b", "z") set5 := threadsafe.New("a", "b", "c", "d") require.True(t, set1.IsSupersetThreadsafe(set2)) require.False(t, set1.IsSupersetThreadsafe(set3)) require.False(t, set1.IsSupersetThreadsafe(set4)) require.True(t, set1.IsSupersetThreadsafe(set5)) } func TestCopy(t *testing.T) { set := threadsafe.New() cset := set.Copy() require.Equal(t, 0, len(cset)) set = threadsafe.New("a", "b") cset = set.Copy() require.Equal(t, 2, len(cset)) if !set.Has("a") { require.FailNow(t, "a not found in set") } if !set.Has("b") { require.FailNow(t, "b not found in set") } } func TestCopyToThreadsafe(t *testing.T) { set := threadsafe.New() cset := set.CopyToThreadsafe() require.Equal(t, 0, cset.Len()) set = threadsafe.New("a", "b") cset = set.CopyToThreadsafe() require.Equal(t, 2, cset.Len()) if !set.Has("a") { require.FailNow(t, "a not found in set") } if !set.Has("b") { require.FailNow(t, "b not found in set") } } func TestSlice(t *testing.T) { set := threadsafe.New() require.Equal(t, set.Slice(), []string{}) set.Add("a") require.Equal(t, set.Slice(), []string{"a"}) set.Add("a", "b") require.ElementsMatch(t, set.Slice(), []string{"a", "b"}) } func TestMerge(t *testing.T) { set := threadsafe.New() emptySet := strset.New() set.Merge(emptySet) require.Equal(t, 0, set.Len()) set.Merge(strset.New("a")) require.Equal(t, 1, set.Len()) set.Merge(emptySet) require.Equal(t, 1, set.Len()) set.Add("a", "b", "c") set.Merge(strset.New("e", "e", "d")) require.Equal(t, set, threadsafe.New("a", "b", "c", "e", "d")) set.Merge(strset.New("a", "e")) require.Equal(t, set, threadsafe.New("a", "b", "c", "e", "d")) set.Merge(strset.New("a", "e", "i"), strset.New("o", "u"), strset.New("sometimes y")) require.Equal(t, set, threadsafe.New("a", "b", "c", "e", "d", "i", "o", "u", "sometimes y")) } func TestMergeThreadsafe(t *testing.T) { set := threadsafe.New() emptySet := threadsafe.New() set.MergeThreadsafe(emptySet) require.Equal(t, 0, set.Len()) set.MergeThreadsafe(threadsafe.New("a")) require.Equal(t, 1, set.Len()) set.MergeThreadsafe(emptySet) require.Equal(t, 1, set.Len()) set.Add("a", "b", "c") set.MergeThreadsafe(threadsafe.New("e", "e", "d")) require.Equal(t, set, threadsafe.New("a", "b", "c", "e", "d")) set.MergeThreadsafe(threadsafe.New("a", "e")) require.Equal(t, set, threadsafe.New("a", "b", "c", "e", "d")) set.MergeThreadsafe(threadsafe.New("a", "e", "i"), threadsafe.New("o", "u"), threadsafe.New("sometimes y")) require.Equal(t, set, threadsafe.New("a", "b", "c", "e", "d", "i", "o", "u", "sometimes y")) } func TestSubtract(t *testing.T) { set := threadsafe.New("a", "b", "c") set.Subtract(strset.New("z", "a", "b")) require.Equal(t, set, threadsafe.New("c")) set.Subtract(strset.New("x")) require.Equal(t, set, threadsafe.New("c")) } func SubtractThreadsafe(t *testing.T) { set := threadsafe.New("a", "b", "c") set.SubtractThreadsafe(threadsafe.New("z", "a", "b")) require.Equal(t, set, threadsafe.New("c")) set.SubtractThreadsafe(threadsafe.New("x")) require.Equal(t, set, threadsafe.New("c")) } func TestShrink(t *testing.T) { set := threadsafe.New("a", "b", "c", "d") set.Shrink(2) require.Equal(t, set.Len(), 2) set = threadsafe.New("g", "f", "e", "d", "c", "b", "a") set.ShrinkSorted(3) require.Equal(t, set.Len(), 3) set = threadsafe.New("a", "b") set.Shrink(2) require.Equal(t, set, threadsafe.New("a", "b")) set = threadsafe.New("a") set.Shrink(2) require.Equal(t, set, threadsafe.New("a")) set = threadsafe.New() set.Shrink(2) require.Equal(t, set.Len(), 0) } func TestShrinkSorted(t *testing.T) { for i := 0; i < 10; i++ { set := threadsafe.New("g", "f", "e", "d", "c", "b", "a") set.ShrinkSorted(2) require.Equal(t, set, threadsafe.New("a", "b")) set = threadsafe.New("g", "f", "e", "d", "c", "b", "a") set.ShrinkSorted(3) require.Equal(t, set, threadsafe.New("a", "b", "c")) } set := threadsafe.New("a", "b") set.ShrinkSorted(2) require.Equal(t, set, threadsafe.New("a", "b")) set = threadsafe.New("a") set.ShrinkSorted(2) require.Equal(t, set, threadsafe.New("a")) set = threadsafe.New() set.ShrinkSorted(2) require.Equal(t, set.Len(), 0) } func TestUnion(t *testing.T) { require.Equal(t, threadsafe.Union(threadsafe.New(), strset.New()).Len(), 0) require.Equal(t, threadsafe.Union(threadsafe.New("a", "b"), strset.New()), threadsafe.New("a", "b")) require.Equal(t, threadsafe.Union(threadsafe.New(), strset.New("a", "b")), threadsafe.New("a", "b")) require.Equal(t, threadsafe.Union(threadsafe.New("a", "a"), strset.New("a", "b")), threadsafe.New("a", "b")) require.Equal(t, threadsafe.Union(threadsafe.New("a"), strset.New("b")), threadsafe.New("a", "b")) } func TestUnionThreadsafe(t *testing.T) { require.Equal(t, threadsafe.UnionThreadsafe(threadsafe.New(), threadsafe.New()).Len(), 0) require.Equal(t, threadsafe.UnionThreadsafe(threadsafe.New("a", "b"), threadsafe.New()), threadsafe.New("a", "b")) require.Equal(t, threadsafe.UnionThreadsafe(threadsafe.New(), threadsafe.New("a", "b")), threadsafe.New("a", "b")) require.Equal(t, threadsafe.UnionThreadsafe(threadsafe.New("a", "a"), threadsafe.New("a", "b")), threadsafe.New("a", "b")) require.Equal(t, threadsafe.UnionThreadsafe(threadsafe.New("a"), threadsafe.New("b")), threadsafe.New("a", "b")) } func TestDifference(t *testing.T) { set1 := threadsafe.New("a", "b", "c") set1Strset := strset.New("a", "b", "c") set2 := threadsafe.New("z", "a", "b") set2Strset := strset.New("z", "a", "b") d := threadsafe.Difference(set1, set2Strset) require.Equal(t, d, threadsafe.New("c")) d = threadsafe.Difference(set2, set1Strset) require.Equal(t, d, threadsafe.New("z")) d = threadsafe.Difference(set1, set1Strset) require.Equal(t, d, threadsafe.New()) } func TestDifferenceThreadsafe(t *testing.T) { set1 := threadsafe.New("a", "b", "c") set2 := threadsafe.New("z", "a", "b") d := threadsafe.DifferenceThreadsafe(set1, set2) require.Equal(t, d, threadsafe.New("c")) d = threadsafe.DifferenceThreadsafe(set2, set1) require.Equal(t, d, threadsafe.New("z")) d = threadsafe.DifferenceThreadsafe(set1, set1) require.Equal(t, d, threadsafe.New()) } func TestIntersection(t *testing.T) { set1 := threadsafe.New("a", "b", "c") set2 := threadsafe.New("a", "x", "y") set2Strset := strset.New("a", "x", "y") set3Strset := strset.New("z", "b", "c") set4Strset := strset.New("z", "b", "w") d := threadsafe.Intersection(set1, set2Strset) require.Equal(t, d, threadsafe.New("a")) d = threadsafe.Intersection(set1, set3Strset) require.Equal(t, d, threadsafe.New("b", "c")) d = threadsafe.Intersection(set2, set3Strset) require.Equal(t, d, threadsafe.New()) d = threadsafe.Intersection(set1, set3Strset, set4Strset) require.Equal(t, d, threadsafe.New("b")) } func TestIntersectionThreadsafe(t *testing.T) { set1 := threadsafe.New("a", "b", "c") set2 := threadsafe.New("a", "x", "y") set3 := threadsafe.New("z", "b", "c") set4 := threadsafe.New("z", "b", "w") d := threadsafe.IntersectionThreadsafe(set1, set2) require.Equal(t, d, threadsafe.New("a")) d = threadsafe.IntersectionThreadsafe(set1, set3) require.Equal(t, d, threadsafe.New("b", "c")) d = threadsafe.IntersectionThreadsafe(set2, set3) require.Equal(t, d, threadsafe.New()) d = threadsafe.IntersectionThreadsafe(set1, set3, set4) require.Equal(t, d, threadsafe.New("b")) } ================================================ FILE: pkg/lib/slices/bool.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package slices func HasTrue(list []bool) bool { for _, elem := range list { if elem { return true } } return false } ================================================ FILE: pkg/lib/slices/errors.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package slices import ( "github.com/cortexlabs/cortex/pkg/lib/errors" ) const ( ErrLenValuesWeightsMismatch = "slices.len_values_weights_mismatch" ) func ErrorLenValuesWeightsMismatch() error { return errors.WithStack(&errors.Error{ Kind: ErrLenValuesWeightsMismatch, Message: "length of values is not equal to length of weights", }) } ================================================ FILE: pkg/lib/slices/float32.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package slices import ( s "github.com/cortexlabs/cortex/pkg/lib/strings" ) func HasFloat32(list []float32, query float32) bool { for _, elem := range list { if elem == query { return true } } return false } func CopyFloat32s(vals []float32) []float32 { return append(vals[:0:0], vals...) } func Float32ToString(vals []float32) []string { stringSlice := []string{} for _, elem := range vals { stringSlice = append(stringSlice, s.Float32(elem)) } return stringSlice } ================================================ FILE: pkg/lib/slices/float64.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package slices import ( s "github.com/cortexlabs/cortex/pkg/lib/strings" ) func HasFloat64(list []float64, query float64) bool { for _, elem := range list { if elem == query { return true } } return false } func CopyFloat64s(vals []float64) []float64 { return append(vals[:0:0], vals...) } func Float64ToString(vals []float64) []string { stringSlice := []string{} for _, elem := range vals { stringSlice = append(stringSlice, s.Float64(elem)) } return stringSlice } ================================================ FILE: pkg/lib/slices/float64_ptr.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package slices // For adding integers stored in floats func Float64PtrSumInt(floats ...*float64) int { sum := 0 for _, num := range floats { if num != nil { sum += int(*num) } } return sum } func Float64PtrMin(floats ...*float64) *float64 { var min *float64 for _, num := range floats { switch { case num != nil && min != nil && *num < *min: min = num case num != nil && min == nil: min = num } } return min } func Float64PtrMax(floats ...*float64) *float64 { var max *float64 for _, num := range floats { switch { case num != nil && max != nil && *num > *max: max = num case num != nil && max == nil: max = num } } return max } func Float64PtrAvg(values []*float64, weights []*float64) (*float64, error) { if len(values) != len(weights) { return nil, ErrorLenValuesWeightsMismatch() } totalWeight := 0.0 for i, valPtr := range values { if valPtr != nil && weights[i] != nil && *weights[i] > 0 { totalWeight += *weights[i] } } if totalWeight == 0.0 { return nil, nil } avg := 0.0 for i, valPtr := range values { if valPtr != nil && weights[i] != nil && *weights[i] > 0 { avg += (*valPtr) * (*weights[i]) / float64(totalWeight) } } return &avg, nil } ================================================ FILE: pkg/lib/slices/float64_ptr_test.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package slices import ( "testing" "github.com/cortexlabs/cortex/pkg/lib/pointer" "github.com/stretchr/testify/require" ) var _float64NilPtr = (*float64)(nil) func TestFloat64PtrSumInt(t *testing.T) { require.Equal(t, 0, Float64PtrSumInt(nil)) require.Equal(t, 1, Float64PtrSumInt(pointer.Float64(1))) require.Equal(t, 2, Float64PtrSumInt(pointer.Float64(1), pointer.Float64(1.5))) } func TestFloat64PtrMin(t *testing.T) { require.Equal(t, _float64NilPtr, Float64PtrMin()) require.Equal(t, _float64NilPtr, Float64PtrMin(nil)) require.Equal(t, pointer.Float64(1), Float64PtrMin(pointer.Float64(1))) require.Equal(t, pointer.Float64(-1), Float64PtrMin(_float64NilPtr, pointer.Float64(1), pointer.Float64(-1))) } func TestFloat64PtrMax(t *testing.T) { require.Equal(t, _float64NilPtr, Float64PtrMax()) require.Equal(t, _float64NilPtr, Float64PtrMax(nil)) require.Equal(t, pointer.Float64(1), Float64PtrMax(pointer.Float64(1))) require.Equal(t, pointer.Float64(1.5), Float64PtrMax(pointer.Float64(1), pointer.Float64(1.5), _float64NilPtr)) } func TestFloat64PtrAvg(t *testing.T) { var err error var avg *float64 avg, err = Float64PtrAvg([]*float64{pointer.Float64(1)}, []*float64{pointer.Float64(10)}) require.Equal(t, pointer.Float64(1), avg) require.NoError(t, err) avg, err = Float64PtrAvg([]*float64{pointer.Float64(10)}, []*float64{pointer.Float64(1)}) require.Equal(t, pointer.Float64(10), avg) require.NoError(t, err) avg, err = Float64PtrAvg([]*float64{pointer.Float64(1), pointer.Float64(4), _float64NilPtr}, []*float64{pointer.Float64(2), pointer.Float64(1), pointer.Float64(1)}) require.Equal(t, pointer.Float64(2), avg) require.NoError(t, err) avg, err = Float64PtrAvg([]*float64{pointer.Float64(1), pointer.Float64(4), pointer.Float64(1)}, []*float64{pointer.Float64(2), pointer.Float64(1), _float64NilPtr}) require.Equal(t, pointer.Float64(2), avg) require.NoError(t, err) avg, err = Float64PtrAvg([]*float64{pointer.Float64(1), pointer.Float64(4), _float64NilPtr}, []*float64{pointer.Float64(2), pointer.Float64(1), _float64NilPtr}) require.Equal(t, pointer.Float64(2), avg) require.NoError(t, err) avg, err = Float64PtrAvg([]*float64{pointer.Float64(1)}, []*float64{pointer.Float64(2), pointer.Float64(1)}) require.Equal(t, _float64NilPtr, avg) require.Error(t, err) avg, err = Float64PtrAvg([]*float64{pointer.Float64(2)}, []*float64{pointer.Float64(0)}) require.Equal(t, _float64NilPtr, avg) require.NoError(t, err) avg, err = Float64PtrAvg([]*float64{pointer.Float64(0)}, []*float64{pointer.Float64(2)}) require.Equal(t, pointer.Float64(0), avg) require.NoError(t, err) avg, err = Float64PtrAvg([]*float64{nil}, []*float64{pointer.Float64(2)}) require.Equal(t, _float64NilPtr, avg) require.NoError(t, err) } ================================================ FILE: pkg/lib/slices/int.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package slices import ( s "github.com/cortexlabs/cortex/pkg/lib/strings" ) func HasInt(list []int, query int) bool { for _, elem := range list { if elem == query { return true } } return false } func CopyInts(vals []int) []int { return append(vals[:0:0], vals...) } func AreNGreaterThanZero(minCount int, val int, vals ...int) bool { count := 0 allVals := append(vals, val) for _, val := range allVals { if val > 0 { count++ if count >= minCount { return true } } } return false } func IntToString(vals []int) []string { stringSlice := []string{} for _, elem := range vals { stringSlice = append(stringSlice, s.Int(elem)) } return stringSlice } ================================================ FILE: pkg/lib/slices/int32.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package slices import ( s "github.com/cortexlabs/cortex/pkg/lib/strings" ) func HasInt32(list []int32, query int32) bool { for _, elem := range list { if elem == query { return true } } return false } func CopyInt32s(vals []int32) []int32 { return append(vals[:0:0], vals...) } func Int32ToString(vals []int32) []string { stringSlice := []string{} for _, elem := range vals { stringSlice = append(stringSlice, s.Int32(elem)) } return stringSlice } ================================================ FILE: pkg/lib/slices/int64.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package slices import ( s "github.com/cortexlabs/cortex/pkg/lib/strings" ) func HasInt64(list []int64, query int64) bool { for _, elem := range list { if elem == query { return true } } return false } func CopyInt64s(vals []int64) []int64 { return append(vals[:0:0], vals...) } func UniqueInt64(vals []int64) []int64 { keys := make(map[int64]bool) list := []int64{} for _, entry := range vals { if _, value := keys[entry]; !value { keys[entry] = true list = append(list, entry) } } return list } func Int64ToString(vals []int64) []string { stringSlice := []string{} for _, elem := range vals { stringSlice = append(stringSlice, s.Int64(elem)) } return stringSlice } ================================================ FILE: pkg/lib/slices/sort.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package slices import ( "math" "sort" ) // Inspiration: https://golang.org/src/sort/sort.go type Int32Slice []int32 func (p Int32Slice) Len() int { return len(p) } func (p Int32Slice) Less(i, j int) bool { return p[i] < p[j] } func (p Int32Slice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } func SortInt32s(a []int32) { sort.Sort(Int32Slice(a)) } type Int64Slice []int64 func (p Int64Slice) Len() int { return len(p) } func (p Int64Slice) Less(i, j int) bool { return p[i] < p[j] } func (p Int64Slice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } func SortInt64s(a []int64) { sort.Sort(Int64Slice(a)) } type Float32Slice []float32 func (p Float32Slice) Len() int { return len(p) } func (p Float32Slice) Less(i, j int) bool { return p[i] < p[j] || math.IsNaN(float64(p[i])) && !math.IsNaN(float64(p[j])) } func (p Float32Slice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } func SortFloat32s(a []float32) { sort.Sort(Float32Slice(a)) } // Sort with copying func SortStrsCopy(a []string) []string { aCopy := CopyStrings(a) sort.Strings(aCopy) return aCopy } func SortIntsCopy(a []int) []int { aCopy := CopyInts(a) sort.Ints(aCopy) return aCopy } func SortInt32sCopy(a []int32) []int32 { aCopy := CopyInt32s(a) SortInt32s(aCopy) return aCopy } func SortInt64sCopy(a []int64) []int64 { aCopy := CopyInt64s(a) SortInt64s(aCopy) return aCopy } func SortFloat32sCopy(a []float32) []float32 { aCopy := CopyFloat32s(a) SortFloat32s(aCopy) return aCopy } func SortFloat64sCopy(a []float64) []float64 { aCopy := CopyFloat64s(a) sort.Float64s(aCopy) return aCopy } ================================================ FILE: pkg/lib/slices/string.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package slices import ( "strconv" libmath "github.com/cortexlabs/cortex/pkg/lib/math" "github.com/cortexlabs/cortex/pkg/lib/sets/strset" ) // HasString checks if a string slice contains a target string func HasString(list []string, query string) bool { for _, elem := range list { if elem == query { return true } } return false } // HasAnyStrings checks if a string slice contains any string from the query string slice func HasAnyStrings(queries []string, list []string) bool { keys := strset.New() for _, elem := range queries { keys.Add(elem) } for _, elem := range list { if keys.Has(elem) { return true } } return false } // HasAllStrings checks if a string slice contains all the strings from the query string slice func HasAllStrings(queries []string, list []string) bool { keys := strset.New() for _, elem := range list { keys.Add(elem) } for _, elem := range queries { if !keys.Has(elem) { return false } } return true } func CopyStrings(vals []string) []string { return append(vals[:0:0], vals...) } func UniqueStrings(strs []string) []string { keys := strset.New() out := []string{} for _, elem := range strs { if !keys.Has(elem) { keys.Add(elem) out = append(out, elem) } } return out } func RemoveEmpties(strs []string) []string { var cleanStrs []string for _, str := range strs { if str != "" { cleanStrs = append(cleanStrs, str) } } return cleanStrs } func RemoveEmptiesAndUnique(strs []string) []string { keys := strset.New() out := []string{} for _, elem := range strs { if elem != "" { if !keys.Has(elem) { keys.Add(elem) out = append(out, elem) } } } return out } // RemoveString removes a target string from a string slice if it exists func RemoveString(strs []string, target string) []string { var result []string for _, item := range strs { if item == target { continue } result = append(result, item) } return result } func HasDuplicateStr(in []string) bool { keys := strset.New() for _, elem := range in { if keys.Has(elem) { return true } keys.Add(elem) } return false } func FindDuplicateStrs(in []string) []string { dups := []string{} keys := strset.New() for _, elem := range in { if keys.Has(elem) { dups = append(dups, elem) } keys.Add(elem) } return dups } func SubtractStrSlice(slice1 []string, slice2 []string) []string { result := []string{} for _, elem := range slice1 { if !HasString(slice2, elem) { result = append(result, elem) } } return result } func StrSliceElementsMatch(strs1 []string, strs2 []string) bool { if len(strs1) == 0 && len(strs2) == 0 { return true } if len(strs1) != len(strs2) { return false } return StrSlicesEqual(SortStrsCopy(strs1), SortStrsCopy(strs2)) } func StrSlicesEqual(strs1 []string, strs2 []string) bool { if len(strs1) == 0 && len(strs2) == 0 { return true } if len(strs1) != len(strs2) { return false } for i := range strs1 { if strs1[i] != strs2[i] { return false } } return true } func FilterStrs(strs []string, filterFn func(string) bool) []string { out := []string{} for _, elem := range strs { if filterFn(elem) { out = append(out, elem) } } return out } func MapStrs(strs []string, mapFn func(string) string) []string { out := make([]string, len(strs)) for i, elem := range strs { out[i] = mapFn(elem) } return out } func MergeStrSlices(slices ...[]string) []string { if slices == nil || len(slices) == 0 { return nil } var totalLen int for _, s := range slices { totalLen += len(s) } result := make([]string, totalLen) var i int for _, s := range slices { i += copy(result[i:], s) } return result } func ZipStrsToMap(strs1 []string, strs2 []string) map[string]string { strMap := map[string]string{} length := libmath.MinInt(len(strs1), len(strs2)) for i := 0; i < length; i++ { strMap[strs1[i]] = strs2[i] } return strMap } func StringToInt(vals []string) ([]int, error) { intSlice := []int{} for _, elem := range vals { i, err := strconv.Atoi(elem) if err != nil { return nil, err } intSlice = append(intSlice, i) } return intSlice, nil } func StringToInt32(vals []string) ([]int32, error) { intSlice := []int32{} for _, elem := range vals { i, err := strconv.ParseInt(elem, 10, 32) if err != nil { return nil, err } intSlice = append(intSlice, int32(i)) } return intSlice, nil } func StringToInt64(vals []string) ([]int64, error) { intSlice := []int64{} for _, elem := range vals { i, err := strconv.ParseInt(elem, 10, 64) if err != nil { return nil, err } intSlice = append(intSlice, i) } return intSlice, nil } ================================================ FILE: pkg/lib/slices/string_test.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package slices_test import ( "testing" "github.com/cortexlabs/cortex/pkg/lib/slices" "github.com/stretchr/testify/require" ) func TestStrSliceElementsMatch(t *testing.T) { var strs1 []string var strs2 []string strs1 = []string{} strs2 = []string{} require.True(t, slices.StrSliceElementsMatch(strs1, strs2)) strs1 = []string{"1"} strs2 = []string{"1"} require.True(t, slices.StrSliceElementsMatch(strs1, strs2)) strs1 = []string{"1", "2", "3"} strs2 = []string{"1", "2", "3"} require.True(t, slices.StrSliceElementsMatch(strs1, strs2)) strs1 = []string{"1", "2", "3"} strs2 = []string{"1", "2", "3", "4"} require.False(t, slices.StrSliceElementsMatch(strs1, strs2)) strs1 = []string{"1", "2", "3"} strs2 = []string{"1", "4", "3"} require.False(t, slices.StrSliceElementsMatch(strs1, strs2)) strs1 = []string{"1", "2", "3"} strs2 = []string{"3", "2", "1"} require.True(t, slices.StrSliceElementsMatch(strs1, strs2)) strs1 = []string{"2", "1", "2", "3"} strs2 = []string{"3", "2", "1", "2"} require.True(t, slices.StrSliceElementsMatch(strs1, strs2)) require.Equal(t, []string{"2", "1", "2", "3"}, strs1) // ensure sort didn't get applied require.Equal(t, []string{"3", "2", "1", "2"}, strs2) // ensure sort didn't get applied strs1 = []string{"2", "1", "2", "3"} strs2 = []string{"3", "2", "1"} require.False(t, slices.StrSliceElementsMatch(strs1, strs2)) } func TestHasString(t *testing.T) { cases := []struct { name string slice []string target string expected bool }{ { name: "exists", slice: []string{"a", "b", "c"}, target: "a", expected: true, }, { name: "doesn't exist", slice: []string{"a", "b", "c"}, target: "d", expected: false, }, { name: "nil", slice: nil, target: "a", expected: false, }, } for _, tt := range cases { t.Run(tt.name, func(t *testing.T) { result := slices.HasString(tt.slice, tt.target) require.Equal(t, tt.expected, result) }) } } func TestRemoveString(t *testing.T) { cases := []struct { name string slice []string target string expected []string }{ { name: "simple", slice: []string{"a", "b", "c"}, target: "a", expected: []string{"b", "c"}, }, { name: "repeated", slice: []string{"a", "b", "c", "c"}, target: "c", expected: []string{"a", "b"}, }, { name: "nil", slice: nil, target: "a", expected: nil, }, { name: "unchanged", slice: []string{"a", "b", "c", "c"}, target: "d", expected: []string{"a", "b", "c", "c"}, }, } for _, tt := range cases { t.Run(tt.name, func(t *testing.T) { result := slices.RemoveString(tt.slice, tt.target) require.Equal(t, tt.expected, result) }) } } ================================================ FILE: pkg/lib/strings/operations.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package strings import ( "regexp" "strings" "unicode" "github.com/cortexlabs/cortex/pkg/lib/cast" ) func ToTitle(str string) string { return strings.Title(strings.ToLower(str)) } func EnsureSingleOccurrenceCharPrefix(str string, character string) string { return character + strings.TrimLeft(str, character) } func EnsureSingleOccurrenceCharSuffix(str string, character string) string { return strings.TrimRight(str, character) + character } func EnsurePrefix(str string, prefix string) string { if prefix != "" && !strings.HasPrefix(str, prefix) { return prefix + str } return str } func EnsureSuffix(str string, suffix string) string { if suffix != "" && !strings.HasSuffix(str, suffix) { return str + suffix } return str } func EnsureBlankLineIfNotEmpty(str string) string { if str == "" { return str } if strings.HasSuffix(str, "\n\n") { return str } if strings.HasSuffix(str, "\n") { return str + "\n" } return str + "\n\n" } func TrimTrailingNewLines(str string) string { return strings.TrimRight(str, "\n") } func TrimTrailingWhitespace(str string) string { return strings.TrimRightFunc(str, unicode.IsSpace) } func EnsureSingleTrailingNewLine(str string) string { return strings.TrimRight(str, "\n") + "\n" } func HasPrefixAndSuffix(str string, substr string) bool { return strings.HasPrefix(str, substr) && strings.HasSuffix(str, substr) } func TrimPrefixAndSuffix(str string, substr string) string { return strings.TrimSuffix(strings.TrimPrefix(str, substr), substr) } // MaskString omits no more than half of the string func MaskString(str string, numPlain int) string { if numPlain > len(str)/2 { numPlain = len(str) / 2 } return strings.Repeat("*", len(str)-numPlain) + str[len(str)-numPlain:] } // Returns the portion str after the last occurrance of chars, or the entire str if chars are not found func LastSplit(str string, chars string) string { split := strings.Split(str, chars) return split[len(split)-1] } // Returns the last n chars, or the entire string if the requested length is greater than the length of the string func LastNChars(str string, n int) string { if len(str) < n { return str } return str[len(str)-n:] } func LongestCommonPrefix(strs ...string) string { if len(strs) == 0 { return "" } prefix := strs[0] if len(strs) == 1 { return prefix } for _, str := range strs[1:] { if len(prefix) == 0 || len(str) == 0 { return "" } maxLen := len(prefix) if len(str) < maxLen { maxLen = len(str) } for i := 0; i < maxLen; i++ { if prefix[i] != str[i] { prefix = prefix[:i] break } } } return prefix } func MaxLen(strs ...string) int { if len(strs) == 0 { return 0 } maxLen := len(strs[0]) for _, str := range strs { if len(str) > maxLen { maxLen = len(str) } } return maxLen } func TrimPrefixIfPresentInAll(strs []string, prefix string) ([]string, bool) { if prefix == "" { return strs, false } trimmedStrs := make([]string, len(strs)) for i, str := range strs { if !strings.HasPrefix(str, prefix) { return strs, false } trimmedStrs[i] = strings.TrimPrefix(str, prefix) } return trimmedStrs, true } func StrsOr(strs []string) string { return StrsSentence(strs, "or") } func StrsAnd(strs []string) string { return StrsSentence(strs, "and") } func UserStrsOr(vals interface{}) string { return StrsOr(UserStrs(vals)) } func UserStrsAnd(vals interface{}) string { return StrsAnd(UserStrs(vals)) } func StrsSentence(strs []string, lastJoinWord string) string { switch len(strs) { case 0: return "" case 1: return strs[0] case 2: return strings.Join(strs, " "+lastJoinWord+" ") default: lastIndex := len(strs) - 1 return strings.Join(strs[:lastIndex], ", ") + ", " + lastJoinWord + " " + strs[lastIndex] } } func SIfPlural(count interface{}) string { return StrIfPlural("s", count) } func EsIfPlural(count interface{}) string { return StrIfPlural("es", count) } func StrIfPlural(str string, count interface{}) string { countInt, _ := cast.InterfaceToInt64(count) if countInt > 1 { return str } return "" } func PluralS(str string, count interface{}) string { return PluralCustom(str, str+"s", count) } func PluralEs(str string, count interface{}) string { return PluralCustom(str, str+"es", count) } func PluralIs(count interface{}) string { return PluralCustom("is", "are", count) } func PluralCustom(singular string, plural string, count interface{}) string { countInt, _ := cast.InterfaceToInt64(count) if countInt == 1 { return singular } return plural } // RemoveDuplicates returns a filtered string slice without repeated entries. // The ignoreRegex parameter can optionally be used to ignore repeated patterns in each slice entry. func RemoveDuplicates(strs []string, ignoreRegex *regexp.Regexp) []string { var result []string counter := map[string]int64{} for _, str := range strs { filteredStr := str if ignoreRegex != nil { filteredStr = ignoreRegex.ReplaceAllString(str, "") } counter[filteredStr]++ if counter[filteredStr] > 1 { continue } result = append(result, str) } return result } ================================================ FILE: pkg/lib/strings/operations_test.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package strings import ( "regexp" "testing" "github.com/stretchr/testify/require" ) func TestLongestCommonPrefix(t *testing.T) { var strs []string var expected string strs = []string{ "12345", } expected = "12345" require.Equal(t, expected, LongestCommonPrefix(strs...)) strs = []string{ "12345", "12345678", } expected = "12345" require.Equal(t, expected, LongestCommonPrefix(strs...)) strs = []string{ "12345", "12345678", "1239", } expected = "123" require.Equal(t, expected, LongestCommonPrefix(strs...)) strs = []string{ "123", "456", } expected = "" require.Equal(t, expected, LongestCommonPrefix(strs...)) } func TestRemoveDuplicates(t *testing.T) { cases := []struct { name string input []string prefixRegex *regexp.Regexp expected []string }{ { name: "abc", input: []string{"a", "b", "a", "b", "a", "a", "a", "c"}, expected: []string{"a", "b", "c"}, }, { name: "nil", input: nil, expected: nil, }, { name: "eksctl", prefixRegex: regexp.MustCompile(`^.*[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2} \[.+] {2}`), input: []string{ "2021-03-26 00:03:50 [ℹ] eksctl version 0.40.0", "2021-03-26 00:03:50 [ℹ] using region us-east-1", "2021-03-26 00:03:50 [ℹ] subnets for us-east-1a - public:192.168.0.0/19 private:192.168.64.0/19", "2021-03-26 00:03:50 [ℹ] subnets for us-east-1b - public:192.168.32.0/19 private:192.168.96.0/19", "2021-03-26 00:03:50 [ℹ] nodegroup \"cx-operator\" will use \"ami-05edded4121b6bde8\" [AmazonLinux2/1.18]", "2021-03-26 00:03:50 [ℹ] nodegroup \"cx-ws-spot\" will use \"ami-05edded4121b6bde8\" [AmazonLinux2/1.18]", "2021-03-26 00:03:50 [ℹ] nodegroup \"cx-wd-cpu\" will use \"ami-05edded4121b6bde8\" [AmazonLinux2/1.18]", "2021-03-26 00:03:51 [ℹ] nodegroup \"cx-wd-gpu\" will use \"ami-00a430391abee258d\" [AmazonLinux2/1.18]", "2021-03-26 00:03:51 [ℹ] nodegroup \"cx-wd-inferentia\" will use \"ami-00a430391abee258d\" [AmazonLinux2/1.18]", "2021-03-26 00:03:51 [ℹ] using Kubernetes version 1.18", "2021-03-26 00:03:51 [ℹ] creating EKS cluster \"cortex\" in \"us-east-1\" region with un-managed nodes", "2021-03-26 00:03:51 [ℹ] 5 nodegroups (cx-operator, cx-wd-cpu, cx-wd-gpu, cx-wd-inferentia, cx-ws-spot) were included (based on the include/exclude rules)", "2021-03-26 00:03:51 [ℹ] will create a CloudFormation stack for cluster itself and 5 nodegroup stack(s)", "2021-03-26 00:03:51 [ℹ] will create a CloudFormation stack for cluster itself and 0 managed nodegroup stack(s)", "2021-03-26 00:03:51 [ℹ] if you encounter any issues, check CloudFormation console or try 'eksctl utils describe-stacks --region=us-east-1 --cluster=cortex'", "2021-03-26 00:03:51 [ℹ] CloudWatch logging will not be enabled for cluster \"cortex\" in \"us-east-1\"", "2021-03-26 00:03:51 [ℹ] you can enable it with 'eksctl utils update-cluster-logging --enable-types={SPECIFY-YOUR-LOG-TYPES-HERE (e.g. all)} --region=us-east-1 --cluster=cortex'", "2021-03-26 00:03:51 [ℹ] Kubernetes API endpoint access will use default of {publicAccess=true, privateAccess=false} for cluster \"cortex\" in \"us-east-1\"", "2021-03-26 00:03:51 [ℹ] 2 sequential tasks: { create cluster control plane \"cortex\", 3 sequential sub-tasks: { 2 sequential sub-tasks: { wait for control plane to become ready, tag cluster }, create addons, 5 parallel sub-tasks: { create nodegroup \"cx-operator\", create nodegroup \"cx-ws-spot\", create nodegroup \"cx-wd-cpu\", create nodegroup \"cx-wd-gpu\", create nodegroup \"cx-wd-inferentia\" } } }", "2021-03-26 00:03:51 [ℹ] building cluster stack \"eksctl-cortex-cluster\"", "2021-03-26 00:03:52 [ℹ] deploying stack \"eksctl-cortex-cluster\"", "2021-03-26 00:03:52 [ℹ] waiting for CloudFormation stack \"eksctl-cortex-cluster\"", "2021-03-26 00:04:09 [ℹ] waiting for CloudFormation stack \"eksctl-cortex-cluster\"", "2021-03-26 00:04:28 [ℹ] waiting for CloudFormation stack \"eksctl-cortex-cluster\"", "2021-03-26 00:04:47 [ℹ] waiting for CloudFormation stack \"eksctl-cortex-cluster\"", "2021-03-26 00:05:07 [ℹ] waiting for CloudFormation stack \"eksctl-cortex-cluster\"", "2021-03-26 00:05:25 [ℹ] waiting for CloudFormation stack \"eksctl-cortex-cluster\"", }, expected: []string{ "2021-03-26 00:03:50 [ℹ] eksctl version 0.40.0", "2021-03-26 00:03:50 [ℹ] using region us-east-1", "2021-03-26 00:03:50 [ℹ] subnets for us-east-1a - public:192.168.0.0/19 private:192.168.64.0/19", "2021-03-26 00:03:50 [ℹ] subnets for us-east-1b - public:192.168.32.0/19 private:192.168.96.0/19", "2021-03-26 00:03:50 [ℹ] nodegroup \"cx-operator\" will use \"ami-05edded4121b6bde8\" [AmazonLinux2/1.18]", "2021-03-26 00:03:50 [ℹ] nodegroup \"cx-ws-spot\" will use \"ami-05edded4121b6bde8\" [AmazonLinux2/1.18]", "2021-03-26 00:03:50 [ℹ] nodegroup \"cx-wd-cpu\" will use \"ami-05edded4121b6bde8\" [AmazonLinux2/1.18]", "2021-03-26 00:03:51 [ℹ] nodegroup \"cx-wd-gpu\" will use \"ami-00a430391abee258d\" [AmazonLinux2/1.18]", "2021-03-26 00:03:51 [ℹ] nodegroup \"cx-wd-inferentia\" will use \"ami-00a430391abee258d\" [AmazonLinux2/1.18]", "2021-03-26 00:03:51 [ℹ] using Kubernetes version 1.18", "2021-03-26 00:03:51 [ℹ] creating EKS cluster \"cortex\" in \"us-east-1\" region with un-managed nodes", "2021-03-26 00:03:51 [ℹ] 5 nodegroups (cx-operator, cx-wd-cpu, cx-wd-gpu, cx-wd-inferentia, cx-ws-spot) were included (based on the include/exclude rules)", "2021-03-26 00:03:51 [ℹ] will create a CloudFormation stack for cluster itself and 5 nodegroup stack(s)", "2021-03-26 00:03:51 [ℹ] will create a CloudFormation stack for cluster itself and 0 managed nodegroup stack(s)", "2021-03-26 00:03:51 [ℹ] if you encounter any issues, check CloudFormation console or try 'eksctl utils describe-stacks --region=us-east-1 --cluster=cortex'", "2021-03-26 00:03:51 [ℹ] CloudWatch logging will not be enabled for cluster \"cortex\" in \"us-east-1\"", "2021-03-26 00:03:51 [ℹ] you can enable it with 'eksctl utils update-cluster-logging --enable-types={SPECIFY-YOUR-LOG-TYPES-HERE (e.g. all)} --region=us-east-1 --cluster=cortex'", "2021-03-26 00:03:51 [ℹ] Kubernetes API endpoint access will use default of {publicAccess=true, privateAccess=false} for cluster \"cortex\" in \"us-east-1\"", "2021-03-26 00:03:51 [ℹ] 2 sequential tasks: { create cluster control plane \"cortex\", 3 sequential sub-tasks: { 2 sequential sub-tasks: { wait for control plane to become ready, tag cluster }, create addons, 5 parallel sub-tasks: { create nodegroup \"cx-operator\", create nodegroup \"cx-ws-spot\", create nodegroup \"cx-wd-cpu\", create nodegroup \"cx-wd-gpu\", create nodegroup \"cx-wd-inferentia\" } } }", "2021-03-26 00:03:51 [ℹ] building cluster stack \"eksctl-cortex-cluster\"", "2021-03-26 00:03:52 [ℹ] deploying stack \"eksctl-cortex-cluster\"", "2021-03-26 00:03:52 [ℹ] waiting for CloudFormation stack \"eksctl-cortex-cluster\"", }, }, } for _, tt := range cases { t.Run(tt.name, func(t *testing.T) { output := RemoveDuplicates(tt.input, tt.prefixRegex) require.Equal(t, tt.expected, output) }) } } ================================================ FILE: pkg/lib/strings/parse.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package strings import ( "strconv" ) func ParseBool(valStr string) (bool, bool) { casted, err := strconv.ParseBool(valStr) if err != nil { return false, false } return casted, true } func ParseFloat32(valStr string) (float32, bool) { casted, err := strconv.ParseFloat(valStr, 32) if err != nil { return 0, false } return float32(casted), true } func ParseFloat64(valStr string) (float64, bool) { casted, err := strconv.ParseFloat(valStr, 64) if err != nil { return 0, false } return casted, true } func ParseInt(valStr string) (int, bool) { casted, err := strconv.Atoi(valStr) if err != nil { return 0, false } return casted, true } func ParseInt64(valStr string) (int64, bool) { casted, err := strconv.ParseInt(valStr, 10, 64) if err != nil { return 0, false } return casted, true } func ParseInt32(valStr string) (int32, bool) { casted, err := strconv.ParseInt(valStr, 10, 32) if err != nil { return 0, false } return int32(casted), true } func ParseInt16(valStr string) (int16, bool) { casted, err := strconv.ParseInt(valStr, 10, 16) if err != nil { return 0, false } return int16(casted), true } func ParseInt8(valStr string) (int8, bool) { casted, err := strconv.ParseInt(valStr, 10, 8) if err != nil { return 0, false } return int8(casted), true } ================================================ FILE: pkg/lib/strings/stringify.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package strings import ( "encoding/json" "fmt" "math" "reflect" "sort" "strconv" "strings" "github.com/cortexlabs/yaml" ) func Bool(val bool) string { return strconv.FormatBool(val) } func Float32(val float32) string { str := strconv.FormatFloat(float64(val), 'f', -1, 32) if !strings.Contains(str, ".") { str = str + ".0" } return str } func Float64(val float64) string { str := strconv.FormatFloat(val, 'f', -1, 64) if !strings.Contains(str, ".") { str = str + ".0" } return str } func Int(val int) string { return strconv.Itoa(val) } func Int64(val int64) string { return strconv.FormatInt(val, 10) } func Int32(val int32) string { return strconv.FormatInt(int64(val), 10) } func Int16(val int16) string { return strconv.FormatInt(int64(val), 10) } func Int8(val int8) string { return strconv.FormatInt(int64(val), 10) } func Uint(val uint) string { return strconv.FormatUint(uint64(val), 10) } func Uint8(val uint8) string { return strconv.FormatUint(uint64(val), 10) } func Uint16(val uint16) string { return strconv.FormatUint(uint64(val), 10) } func Uint32(val uint32) string { return strconv.FormatUint(uint64(val), 10) } func Uint64(val uint64) string { return strconv.FormatUint(val, 10) } func Complex64(val complex64) string { return fmt.Sprint(val) } func Complex128(val complex128) string { return fmt.Sprint(val) } func Uintptr(val uintptr) string { return fmt.Sprint(val) } func Round(val float64, decimalPlaces int, padToDecimalPlaces int) string { rounded := math.Round(val*math.Pow10(decimalPlaces)) / math.Pow10(decimalPlaces) str := strconv.FormatFloat(rounded, 'f', -1, 64) if padToDecimalPlaces == 0 { return str } split := strings.Split(str, ".") intVal := split[0] decVal := "" if len(split) > 1 { decVal = split[1] } if len(decVal) >= padToDecimalPlaces { return str } numZeros := padToDecimalPlaces - len(decVal) return intVal + "." + decVal + strings.Repeat("0", numZeros) } // copied from https://yourbasic.org/golang/formatting-byte-size-to-human-readable-format/ func IntToBase2Byte(size int) string { return Int64ToBase2Byte(int64(size)) } func Int64ToBase2Byte(size int64) string { const unit int64 = 1024 if size < unit { return fmt.Sprintf("%d B", size) } div, exp := int64(unit), 0 for n := size / unit; n >= unit; n /= unit { div *= unit exp++ } return fmt.Sprintf("%.1f %ciB", float64(size)/float64(div), "KMGTPE"[exp]) } func DollarsAndCents(val float64) string { return "$" + Round(val, 2, 2) } func DollarsAndTenthsOfCents(val float64) string { return "$" + Round(val, 3, 2) } func DollarsMaxPrecision(val float64) string { return "$" + Round(val, 100, 2) } // This is similar to json.Marshal, but handles non-string keys (which we support). It should be valid YAML since we use it in templates func strIndent(val interface{}, indent string, currentIndent string, newlineChar string, quoteStr string) string { if val == nil { return "" } value := reflect.ValueOf(val) valueType := value.Type() if value.Kind() == reflect.Invalid { return "" } if value.Kind() == reflect.Chan || value.Kind() == reflect.Func || value.Kind() == reflect.Interface || value.Kind() == reflect.Map || value.Kind() == reflect.Ptr || value.Kind() == reflect.Slice { if value.IsNil() { return "" } } stringSep := "," + newlineChar if len(newlineChar) == 0 { stringSep += " " } // Use a String() method if one exists funcVal := value.MethodByName("String") if funcVal.IsValid() { t := funcVal.Type() if t.NumIn() == 0 && t.NumOut() == 1 && t.Out(0).Kind() == reflect.String { return strIndent(funcVal.Call(nil)[0].Interface().(string), indent, currentIndent, newlineChar, quoteStr) } } if _, ok := reflect.PtrTo(valueType).MethodByName("String"); ok { ptrValue := reflect.New(valueType) ptrValue.Elem().Set(value) funcVal := ptrValue.MethodByName("String") if funcVal.IsValid() { t := funcVal.Type() if t.NumIn() == 0 && t.NumOut() == 1 && t.Out(0).Kind() == reflect.String { return strIndent(funcVal.Call(nil)[0].Interface().(string), indent, currentIndent, newlineChar, quoteStr) } } } switch value.Kind() { case reflect.Bool: var t bool return Bool(value.Convert(reflect.TypeOf(t)).Interface().(bool)) case reflect.Float32: var t float32 return Float32(value.Convert(reflect.TypeOf(t)).Interface().(float32)) case reflect.Float64: var t float64 return Float64(value.Convert(reflect.TypeOf(t)).Interface().(float64)) case reflect.Int: var t int return Int(value.Convert(reflect.TypeOf(t)).Interface().(int)) case reflect.Int8: var t int8 return Int8(value.Convert(reflect.TypeOf(t)).Interface().(int8)) case reflect.Int16: var t int16 return Int16(value.Convert(reflect.TypeOf(t)).Interface().(int16)) case reflect.Int32: var t int32 return Int32(value.Convert(reflect.TypeOf(t)).Interface().(int32)) case reflect.Int64: var t int64 return Int64(value.Convert(reflect.TypeOf(t)).Interface().(int64)) case reflect.Uint: var t uint return Uint(value.Convert(reflect.TypeOf(t)).Interface().(uint)) case reflect.Uint8: var t uint8 return Uint8(value.Convert(reflect.TypeOf(t)).Interface().(uint8)) case reflect.Uint16: var t uint16 return Uint16(value.Convert(reflect.TypeOf(t)).Interface().(uint16)) case reflect.Uint32: var t uint32 return Uint32(value.Convert(reflect.TypeOf(t)).Interface().(uint32)) case reflect.Uint64: var t uint64 return Uint64(value.Convert(reflect.TypeOf(t)).Interface().(uint64)) case reflect.Complex64: var t complex64 return Complex64(value.Convert(reflect.TypeOf(t)).Interface().(complex64)) case reflect.Complex128: var t complex128 return Complex128(value.Convert(reflect.TypeOf(t)).Interface().(complex128)) case reflect.Uintptr: var t uintptr return Uintptr(value.Convert(reflect.TypeOf(t)).Interface().(uintptr)) case reflect.String: var t string casted := value.Convert(reflect.TypeOf(t)).Interface().(string) var ok bool casted, ok = yaml.UnescapeAtSymbolOk(casted) if ok { return casted } switch val.(type) { case json.Number: return casted default: return quoteStr + casted + quoteStr } case reflect.Slice: fallthrough case reflect.Array: if value.Len() == 0 { return "[]" } strs := make([]string, value.Len()) for i := 0; i < value.Len(); i++ { strs[i] = currentIndent + indent + strIndentValue(value.Index(i), indent, currentIndent+indent, newlineChar, quoteStr) } return "[" + newlineChar + strings.Join(strs, stringSep) + newlineChar + currentIndent + "]" case reflect.Map: if value.Len() == 0 { return "{}" } strs := make([]string, value.Len()) for i, keyValue := range value.MapKeys() { keyStr := strIndentValue(keyValue, indent, currentIndent+indent, newlineChar, quoteStr) valStr := strIndentValue(value.MapIndex(keyValue), indent, currentIndent+indent, newlineChar, quoteStr) strs[i] = currentIndent + indent + keyStr + ": " + valStr } sort.Strings(strs) return "{" + newlineChar + strings.Join(strs, stringSep) + newlineChar + currentIndent + "}" case reflect.Struct: if value.NumField() == 0 { return "{}" } strs := make([]string, value.NumField()) for i := 0; i < value.NumField(); i++ { structField := valueType.Field(i) keyStr := strIndent(structField.Name, indent, currentIndent+indent, newlineChar, quoteStr) if tag, ok := structField.Tag.Lookup("yaml"); ok { keyStr = strIndent(strings.Split(tag, ",")[0], indent, currentIndent+indent, newlineChar, quoteStr) } if tag, ok := structField.Tag.Lookup("json"); ok { keyStr = strIndent(strings.Split(tag, ",")[0], indent, currentIndent+indent, newlineChar, quoteStr) } valStr := strIndentValue(value.Field(i), indent, currentIndent+indent, newlineChar, quoteStr) strs[i] = currentIndent + indent + keyStr + ": " + valStr } return "{" + newlineChar + strings.Join(strs, stringSep) + newlineChar + currentIndent + "}" case reflect.Ptr: return strIndentValue(reflect.Indirect(value), indent, currentIndent, newlineChar, quoteStr) case reflect.UnsafePointer: return strIndentValue(reflect.Indirect(value), indent, currentIndent+indent, newlineChar, quoteStr) case reflect.Interface: return strIndentValue(value.Elem(), indent, currentIndent+indent, newlineChar, quoteStr) case reflect.Func: return "" case reflect.Chan: return "" case reflect.Invalid: return "" default: return fmt.Sprint(val) } } func strIndentValue(val reflect.Value, indent string, currentIndent string, newlineChar string, quoteStr string) string { if val.IsValid() && val.CanInterface() { return strIndent(val.Interface(), indent, currentIndent, newlineChar, quoteStr) } return "" } func YesNo(val bool) string { if val { return "yes" } return "no" } func Obj(val interface{}) string { return strIndent(val, " ", "", "\n", `"`) } // Same as Obj(), but trim leading and trailing quotes if it's just a string func ObjStripped(val interface{}) string { return TrimPrefixAndSuffix(Obj(val), `"`) } func ObjFlat(val interface{}) string { return strIndent(val, "", "", "", `"`) } func ObjFlatNoQuotes(val interface{}) string { return strIndent(val, "", "", "", "") } func UserStr(val interface{}) string { return strIndent(val, "", "", "", `"`) } func UserStrValue(val reflect.Value) string { return strIndentValue(val, "", "", "", `"`) } func UserStrStripped(val interface{}) string { return TrimPrefixAndSuffix(UserStr(val), `"`) } func UserStrs(val interface{}) []string { if val == nil { return nil } if reflect.TypeOf(val).Kind() != reflect.Slice { val = []interface{}{val} } inVal := reflect.ValueOf(val) if inVal.IsNil() { return nil } // Handle case where caller passed in a nested slice if inVal.Len() == 1 { if inVal.Index(0).Kind() == reflect.Slice { inVal = inVal.Index(0) } else if inVal.Index(0).Kind() == reflect.Interface { // Handle case where input is e.g. []interface{[]string{"test"}} firstElementVal := reflect.ValueOf(inVal.Index(0).Interface()) if firstElementVal.Kind() == reflect.Slice { inVal = firstElementVal } } } out := make([]string, inVal.Len()) for i := 0; i < inVal.Len(); i++ { out[i] = UserStrValue(inVal.Index(i)) } return out } func Index(index int) string { return fmt.Sprintf("index %d", index) } func Indent(str string, indent string) string { if str == "" { return indent } if str[len(str)-1:] == "\n" { out := "" for _, line := range strings.Split(str[:len(str)-1], "\n") { out += indent + line + "\n" } return out } out := "" for _, line := range strings.Split(strings.TrimRight(str, "\n"), "\n") { out += indent + line + "\n" } return out[:len(out)-1] } func TruncateEllipses(str string, maxLength int) string { ellipses := " ..." if len(str) > maxLength { str = str[:maxLength-len(ellipses)] str += ellipses } return str } ================================================ FILE: pkg/lib/strings/stringify_test.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package strings import ( "testing" "github.com/cortexlabs/cortex/pkg/lib/pointer" "github.com/stretchr/testify/require" ) type MyFloat float64 type MyString string type MyNested map[MyFloat][]MyString type Test struct { Str MyString Float float64 `json:"float"` Test2Ptr *Test2 Test3 Test3 } type Test2 struct { Bool bool `json:"bool"` Float *float64 Strs *[]string Test3Ptr *Test3 `yaml:"test3"` } type Test3 struct { Strs []string Map map[interface{}]interface{} `json:"map"` } type TestInterface interface { Test() } func (t *Test2) Test() {} func (t Test3) Test() {} func TestObj(t *testing.T) { var a interface{} require.Equal(t, "", Obj(a)) var b []string require.Equal(t, "", Obj(b)) var c *string require.Equal(t, "", Obj(c)) require.Equal(t, "true", Obj(true)) require.Equal(t, "2.2", Obj(float32(2.2))) require.Equal(t, "2.0", Obj(float32(2))) require.Equal(t, "2.0", Obj(float64(2))) require.Equal(t, "3", Obj(int(3))) require.Equal(t, "3", Obj(pointer.Int(3))) require.Equal(t, "-3", Obj(int8(-3))) require.Equal(t, "3", Obj(int16(3))) require.Equal(t, "-3", Obj(int32(-3))) require.Equal(t, "3", Obj(int64(3))) require.Equal(t, "4", Obj(int(4))) require.Equal(t, "4", Obj(int8(4))) require.Equal(t, "4", Obj(int16(4))) require.Equal(t, "4", Obj(int32(4))) require.Equal(t, "4", Obj(int64(4))) require.Equal(t, `""`, Obj("")) require.Equal(t, `"test"`, Obj("test")) require.Equal(t, `"test"`, Obj(pointer.String("test"))) var myFloat MyFloat = 2 require.Equal(t, "2.0", Obj(myFloat)) var myString MyString = "test" require.Equal(t, `"test"`, Obj(myString)) strSlice := []string{"a", "b", "c"} require.Equal(t, `["a", "b", "c"]`, ObjFlat(strSlice)) require.Equal(t, `["a", "b", "c"]`, ObjFlat(&strSlice)) intSlice := []int8{1, 2, 3} require.Equal(t, `[1, 2, 3]`, ObjFlat(intSlice)) mixedSlice := []interface{}{int(1), float64(2), "three", ""} require.Equal(t, `[1, 2.0, "three", ""]`, ObjFlat(mixedSlice)) nestedSlice := []interface{}{int(1), "three", strSlice, []interface{}{"a", []float64{1, 2.2}}} require.Equal(t, `[1, "three", ["a", "b", "c"], ["a", [1.0, 2.2]]]`, ObjFlat(nestedSlice)) strMap := map[string]string{"b": "y", "a": "x"} require.Equal(t, `{"a": "x", "b": "y"}`, ObjFlat(strMap)) require.Equal(t, `{"a": "x", "b": "y"}`, ObjFlat(&strMap)) mixedMap := map[interface{}]interface{}{"1": "a", true: strMap, int(3): strSlice} require.Equal(t, `{"1": "a", 3: ["a", "b", "c"], true: {"a": "x", "b": "y"}}`, ObjFlat(mixedMap)) myNested := MyNested{myFloat: []MyString{myString, myString}} require.Equal(t, `{2.0: ["test", "test"]}`, ObjFlat(myNested)) emptyMap := map[interface{}]interface{}{} require.Equal(t, `{}`, Obj(emptyMap)) require.Equal(t, `{}`, ObjFlat(emptyMap)) emptyCollectionsInMap := map[interface{}]interface{}{"empty_map": map[interface{}]interface{}{}, "a": "b", "empty_slice": []interface{}{}} require.Equal(t, `{"a": "b", "empty_map": {}, "empty_slice": []}`, ObjFlat(emptyCollectionsInMap)) require.Equal(t, `{ "a": "b", "empty_map": {}, "empty_slice": [] }`, Obj(emptyCollectionsInMap)) testStruct := Test{ Str: myString, Float: 2, Test2Ptr: &Test2{ Bool: false, Float: pointer.Float64(1.7), Strs: &strSlice, Test3Ptr: &Test3{ Strs: []string{"a", "b", "c"}, Map: map[interface{}]interface{}{"1": "a", true: strMap, int(3): intSlice}, }, }, Test3: Test3{ Strs: nil, Map: map[interface{}]interface{}{"1": nil, true: strMap, int(3): intSlice}, }, } testStructStr := `{"Str": "test", "float": 2.0, "Test2Ptr": {"bool": false, "Float": 1.7, "Strs": ["a", "b", "c"], "test3": {"Strs": ["a", "b", "c"], "map": {"1": "a", 3: [1, 2, 3], true: {"a": "x", "b": "y"}}}}, "Test3": {"Strs": , "map": {"1": , 3: [1, 2, 3], true: {"a": "x", "b": "y"}}}}` testStructStrMultiline := `{ "Str": "test", "float": 2.0, "Test2Ptr": { "bool": false, "Float": 1.7, "Strs": [ "a", "b", "c" ], "test3": { "Strs": [ "a", "b", "c" ], "map": { "1": "a", 3: [ 1, 2, 3 ], true: { "a": "x", "b": "y" } } } }, "Test3": { "Strs": , "map": { "1": , 3: [ 1, 2, 3 ], true: { "a": "x", "b": "y" } } } }` test2SubStrMultiline := `{ "bool": false, "Float": 1.7, "Strs": [ "a", "b", "c" ], "test3": { "Strs": [ "a", "b", "c" ], "map": { "1": "a", 3: [ 1, 2, 3 ], true: { "a": "x", "b": "y" } } } }` test2SubStr := `{"bool": false, "Float": 1.7, "Strs": ["a", "b", "c"], "test3": {"Strs": ["a", "b", "c"], "map": {"1": "a", 3: [1, 2, 3], true: {"a": "x", "b": "y"}}}}` test3SubStrMultiline := `{ "Strs": , "map": { "1": , 3: [ 1, 2, 3 ], true: { "a": "x", "b": "y" } } }` test3SubStr := `{"Strs": , "map": {"1": , 3: [1, 2, 3], true: {"a": "x", "b": "y"}}}` require.Equal(t, testStructStr, ObjFlat(testStruct)) require.Equal(t, testStructStr, ObjFlat(&testStruct)) ptr := &testStruct require.Equal(t, testStructStr, ObjFlat(&ptr)) var testInterface TestInterface testInterface = testStruct.Test2Ptr require.Equal(t, test2SubStr, ObjFlat(testInterface)) require.Equal(t, test2SubStr, ObjFlat(&testInterface)) testInterface = testStruct.Test3 require.Equal(t, test3SubStr, ObjFlat(testInterface)) require.Equal(t, test3SubStr, ObjFlat(&testInterface)) require.Equal(t, testStructStrMultiline, Obj(testStruct)) require.Equal(t, testStructStrMultiline, Obj(&testStruct)) ptr = &testStruct require.Equal(t, testStructStrMultiline, Obj(&ptr)) testInterface = testStruct.Test2Ptr require.Equal(t, test2SubStrMultiline, Obj(testInterface)) require.Equal(t, test2SubStrMultiline, Obj(&testInterface)) testInterface = testStruct.Test3 require.Equal(t, test3SubStrMultiline, Obj(testInterface)) require.Equal(t, test3SubStrMultiline, Obj(&testInterface)) } func TestRound(t *testing.T) { require.Equal(t, Round(1.111, 2, 0), "1.11") require.Equal(t, Round(1.111, 3, 0), "1.111") require.Equal(t, Round(1.111, 4, 0), "1.111") require.Equal(t, Round(1.555, 2, 0), "1.56") require.Equal(t, Round(1.555, 3, 0), "1.555") require.Equal(t, Round(1.555, 4, 0), "1.555") require.Equal(t, Round(1.100, 2, 0), "1.1") require.Equal(t, Round(1.111, 2, 2), "1.11") require.Equal(t, Round(1.111, 3, 3), "1.111") require.Equal(t, Round(1.111, 4, 4), "1.1110") require.Equal(t, Round(1.555, 2, 2), "1.56") require.Equal(t, Round(1.555, 3, 3), "1.555") require.Equal(t, Round(1.555, 4, 4), "1.5550") require.Equal(t, Round(1.100, 2, 2), "1.10") require.Equal(t, Round(30, 0, 0), "30") require.Equal(t, Round(2, 1, 1), "2.0") require.Equal(t, Round(1, 2, 2), "1.00") require.Equal(t, Round(20, 3, 3), "20.000") require.Equal(t, Round(1.5555, 3, 2), "1.556") require.Equal(t, Round(1.5, 3, 2), "1.50") require.Equal(t, Round(1, 3, 2), "1.00") require.Equal(t, Round(1.5555, 3, 1), "1.556") require.Equal(t, Round(1.5, 3, 1), "1.5") require.Equal(t, Round(1, 3, 1), "1.0") require.Equal(t, Round(1.5555, 3, 4), "1.5560") require.Equal(t, Round(1.5, 3, 4), "1.5000") require.Equal(t, Round(1, 3, 4), "1.0000") require.Equal(t, Round(30, 0, 0), "30") require.Equal(t, Round(2, 1, 0), "2") require.Equal(t, Round(1, 2, 0), "1") require.Equal(t, Round(20, 3, 0), "20") } ================================================ FILE: pkg/lib/structs/deepcopy.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package structs import ( "bytes" "encoding/gob" ) func DeepCopy(dst, src interface{}) error { var buf bytes.Buffer if err := gob.NewEncoder(&buf).Encode(src); err != nil { return err } return gob.NewDecoder(bytes.NewBuffer(buf.Bytes())).Decode(dst) } ================================================ FILE: pkg/lib/structs/deepcopy_test.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package structs import ( "testing" "github.com/cortexlabs/cortex/pkg/lib/pointer" "github.com/stretchr/testify/require" ) type SampleStruct struct { AField string BField *string CField *bool } func TestDeepCopy(t *testing.T) { t.Parallel() var a SampleStruct b := SampleStruct{ AField: "fox", BField: pointer.String("bull"), } err := DeepCopy(&a, &b) require.NoError(t, err) require.EqualValues(t, b.AField, "fox") require.EqualValues(t, b.AField, a.AField) require.True(t, a.BField != nil) require.True(t, b.BField != nil) require.True(t, a.BField != b.BField) require.True(t, a.CField == nil) require.True(t, b.CField == nil) require.EqualValues(t, *a.BField, "bull") require.EqualValues(t, *a.BField, *b.BField) } ================================================ FILE: pkg/lib/table/errors.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package table import ( "fmt" "github.com/cortexlabs/cortex/pkg/lib/errors" ) const ( ErrAtLeastOneColumn = "table.at_least_one_column" ErrHeaderWiderThanMaxWidth = "table.header_wider_than_max_width" ErrHeaderMinWidthGreaterThanMaxWidth = "table.header_min_width_greater_than_max_width" ErrWrongNumberOfColumns = "table.wrong_number_of_columns" ) func ErrorAtLeastOneColumn() error { return errors.WithStack(&errors.Error{ Kind: ErrAtLeastOneColumn, Message: "must have at least one column", }) } func ErrorHeaderWiderThanMaxWidth(headerTitle string, maxWidth int) error { return errors.WithStack(&errors.Error{ Kind: ErrHeaderWiderThanMaxWidth, Message: fmt.Sprintf("header %s is wider than max width (%d)", headerTitle, maxWidth), }) } func ErrorHeaderMinWidthGreaterThanMaxWidth(headerTitle string, minWidth int, maxWidth int) error { return errors.WithStack(&errors.Error{ Kind: ErrHeaderMinWidthGreaterThanMaxWidth, Message: fmt.Sprintf("header %s has min width > max width (%d > %d)", headerTitle, minWidth, maxWidth), }) } func ErrorWrongNumberOfColumns(rowNumber int, actualCols int, expectedCols int) error { return errors.WithStack(&errors.Error{ Kind: ErrWrongNumberOfColumns, Message: fmt.Sprintf("row %d does not have the expected number of columns (got %d, expected %d)", rowNumber, actualCols, expectedCols), }) } ================================================ FILE: pkg/lib/table/key_value.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package table import ( "fmt" "strings" "github.com/cortexlabs/cortex/pkg/lib/console" "github.com/cortexlabs/cortex/pkg/lib/pointer" s "github.com/cortexlabs/cortex/pkg/lib/strings" ) type KeyValuePairOpts struct { Delimiter *string // default: ":" NumSpaces *int // default: 1 RightJustify *bool // default: false BoldFirstLine *bool // default: false BoldKeys *bool // default: false } type KeyValuePairs struct { kvs []kv } type kv struct { k interface{} v interface{} } func (kvs *KeyValuePairs) Add(key interface{}, value interface{}) { kvs.kvs = append(kvs.kvs, kv{k: key, v: value}) } func (kvs *KeyValuePairs) AddAll(kvs2 KeyValuePairs) { for _, pair := range kvs2.kvs { kvs.Add(pair.k, pair.v) } } func (kvs KeyValuePairs) String(options ...*KeyValuePairOpts) string { opts := mergeOptions(options...) var maxLen int for _, pair := range kvs.kvs { keyLen := len(s.ObjFlatNoQuotes(pair.k)) if keyLen > maxLen { maxLen = keyLen } } var b strings.Builder for i, pair := range kvs.kvs { keyStr := s.ObjFlatNoQuotes(pair.k) keyLen := len(keyStr) if *opts.BoldKeys { keyStr = console.Bold(keyStr) } valStr := s.ObjFlatNoQuotes(pair.v) var str string if *opts.RightJustify { alignmentSpaces := strings.Repeat(" ", maxLen-keyLen) delimiterSpaces := strings.Repeat(" ", *opts.NumSpaces) str = alignmentSpaces + keyStr + *opts.Delimiter + delimiterSpaces + valStr + "\n" } else { spaces := strings.Repeat(" ", maxLen-keyLen+*opts.NumSpaces) str = keyStr + *opts.Delimiter + spaces + valStr + "\n" } if *opts.BoldFirstLine && i == 0 { str = console.Bold(str) } b.WriteString(str) } return b.String() } func (kvs KeyValuePairs) Print(options ...*KeyValuePairOpts) { fmt.Print(kvs.String(options...)) } func mergeOptions(options ...*KeyValuePairOpts) KeyValuePairOpts { mergedOpts := KeyValuePairOpts{} for _, opt := range options { if opt != nil && opt.Delimiter != nil { mergedOpts.Delimiter = opt.Delimiter } if opt != nil && opt.NumSpaces != nil { mergedOpts.NumSpaces = opt.NumSpaces } if opt != nil && opt.RightJustify != nil { mergedOpts.RightJustify = opt.RightJustify } if opt != nil && opt.BoldFirstLine != nil { mergedOpts.BoldFirstLine = opt.BoldFirstLine } if opt != nil && opt.BoldKeys != nil { mergedOpts.BoldKeys = opt.BoldKeys } } if mergedOpts.Delimiter == nil { mergedOpts.Delimiter = pointer.String(":") } if mergedOpts.NumSpaces == nil { mergedOpts.NumSpaces = pointer.Int(1) } if mergedOpts.RightJustify == nil { mergedOpts.RightJustify = pointer.Bool(false) } if mergedOpts.BoldFirstLine == nil { mergedOpts.BoldFirstLine = pointer.Bool(false) } if mergedOpts.BoldKeys == nil { mergedOpts.BoldKeys = pointer.Bool(false) } return mergedOpts } ================================================ FILE: pkg/lib/table/table.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package table import ( "fmt" "sort" "strings" "github.com/cortexlabs/cortex/pkg/lib/console" "github.com/cortexlabs/cortex/pkg/lib/errors" libmath "github.com/cortexlabs/cortex/pkg/lib/math" "github.com/cortexlabs/cortex/pkg/lib/pointer" s "github.com/cortexlabs/cortex/pkg/lib/strings" ) type Table struct { Headers []Header Rows [][]interface{} Spacing int // Spacing between rows. If 0 is provided, it defaults to 3. } type Header struct { Title string MaxWidth int // Max width of the text (not including spacing). Items that are longer will be truncated to less than MaxWidth to fit the ellipses. If 0 is provided, it defaults to no max. MinWidth int // Min width of the text (not including spacing) Hidden bool } func (t *Table) FindHeaderByTitle(title string) *Header { for i, header := range t.Headers { if header.Title == title { return &t.Headers[i] } } return nil } type Opts struct { Sort *bool // default is true BoldHeader *bool // default is true } func mergeTableOptions(options ...*Opts) Opts { mergedOpts := Opts{} for _, opt := range options { if opt != nil { if opt.Sort != nil { mergedOpts.Sort = opt.Sort } if opt.BoldHeader != nil { mergedOpts.BoldHeader = opt.BoldHeader } } } if mergedOpts.Sort == nil { mergedOpts.Sort = pointer.Bool(true) } if mergedOpts.BoldHeader == nil { mergedOpts.BoldHeader = pointer.Bool(true) } return mergedOpts } func validate(t Table) error { numCols := len(t.Headers) if numCols < 1 { return ErrorAtLeastOneColumn() } for _, header := range t.Headers { if header.MaxWidth != 0 && len(header.Title) > header.MaxWidth { return ErrorHeaderWiderThanMaxWidth(header.Title, header.MaxWidth) } if header.MinWidth > header.MaxWidth { return ErrorHeaderMinWidthGreaterThanMaxWidth(header.Title, header.MinWidth, header.MaxWidth) } } for i, row := range t.Rows { if len(row) != numCols { return ErrorWrongNumberOfColumns(i, len(row), numCols) } } return nil } // Prints the error message as a string (if there is an error) func (t *Table) MustPrint(opts ...*Opts) { fmt.Print(t.MustFormat(opts...)) } // Return the error message as a string func (t *Table) MustFormat(opts ...*Opts) string { str, err := t.Format(opts...) if err != nil { return "error: " + errors.Message(err) } return str } func (t *Table) Format(opts ...*Opts) (string, error) { mergedOpts := mergeTableOptions(opts...) if err := validate(*t); err != nil { return "", err } if t.Spacing <= 0 { t.Spacing = 3 } colWidths := make([]int, len(t.Headers)) for colNum, header := range t.Headers { colWidths[colNum] = len(header.Title) } rows := make([][]string, len(t.Rows)) for rowNum, row := range t.Rows { rows[rowNum] = make([]string, len(row)) for colNum, val := range row { strVal := s.ObjFlatNoQuotes(val) rows[rowNum][colNum] = strVal if len(strVal) > colWidths[colNum] { colWidths[colNum] = len(strVal) } } } maxColWidths := make([]int, len(t.Headers)) for colNum, colWidth := range colWidths { if t.Headers[colNum].MaxWidth <= 0 { maxColWidths[colNum] = colWidth } else { maxColWidths[colNum] = libmath.MinInt(colWidth, t.Headers[colNum].MaxWidth) } if maxColWidths[colNum] < t.Headers[colNum].MinWidth { maxColWidths[colNum] = t.Headers[colNum].MinWidth } } lastColIndex := len(t.Headers) - 1 var headerStr string for colNum, header := range t.Headers { if header.Hidden { continue } if *mergedOpts.BoldHeader { headerStr += console.Bold(header.Title) } else { headerStr += header.Title } if colNum != lastColIndex { headerStr += strings.Repeat(" ", maxColWidths[colNum]+t.Spacing-len(header.Title)) } } headerStr = s.TrimTrailingWhitespace(headerStr) ellipses := "..." rowStrs := make([]string, len(rows)) for rowNum, row := range rows { var rowStr string for colNum, val := range row { if t.Headers[colNum].Hidden { continue } if len(val) > maxColWidths[colNum] { val = val[0:maxColWidths[colNum]] // Ensure at least one space after ellipses for len(val)+len(ellipses) > maxColWidths[colNum]+t.Spacing-1 { val = val[0 : len(val)-1] } val += ellipses } rowStr += val if colNum != lastColIndex { rowStr += strings.Repeat(" ", maxColWidths[colNum]+t.Spacing-len(val)) } } rowStrs[rowNum] = s.TrimTrailingWhitespace(rowStr) } if *mergedOpts.Sort { sort.Strings(rowStrs) } return headerStr + "\n" + strings.Join(rowStrs, "\n") + "\n", nil } ================================================ FILE: pkg/lib/telemetry/error_cache.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package telemetry import ( "sync" "time" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/sets/strset" ) const ( _initialCoolDownPeriod = 5 * time.Second // The initial interval to wait before sending duplicate errors _maxCoolDownPeriod = 24 * time.Hour // The longest interval to wait before sending duplicate errors _coolDownFactor = 2 // Factor by which to increase cool down period after each error report _cacheEvictionPeriod = 24 * time.Hour // Duration of not seeing an error after which seeing it again resets the cooldown period _cacheCleanupInterval = 10 * time.Minute // Minimum time to wait before checking error cache for errors to evict ) var _errorCache = struct { m map[string]*errorStatus lastCleanup time.Time sync.RWMutex }{m: make(map[string]*errorStatus)} type errorStatus struct { LastReportTime time.Time LastSeenTime time.Time CoolDownPeriod time.Duration } func shouldBlock(err error, backoffMode BackoffMode) bool { if backoffMode == NoBackoff { return false } errMsg := errors.MessageFirstLine(err) now := time.Now() if backoffMode == BackoffAnyMessages { errMsg = "" } defer func() { go cleanupCache() }() _errorCache.Lock() defer _errorCache.Unlock() errStatus, ok := _errorCache.m[errMsg] if !ok || time.Since(errStatus.LastSeenTime) > _cacheEvictionPeriod { _errorCache.m[errMsg] = &errorStatus{ LastReportTime: now, LastSeenTime: now, CoolDownPeriod: _initialCoolDownPeriod, } return false } if time.Since(errStatus.LastReportTime) > errStatus.CoolDownPeriod { errStatus.LastSeenTime = now errStatus.LastReportTime = now errStatus.CoolDownPeriod = time.Duration(float64(errStatus.CoolDownPeriod.Nanoseconds())*_coolDownFactor) * time.Nanosecond if errStatus.CoolDownPeriod > _maxCoolDownPeriod { errStatus.CoolDownPeriod = _maxCoolDownPeriod } return false } errStatus.LastSeenTime = now return true } func cleanupCache() { staleErrorMessages := findStaleCachedErrors() if len(staleErrorMessages) == 0 { return } _errorCache.Lock() defer _errorCache.Unlock() for errMsg := range staleErrorMessages { delete(_errorCache.m, errMsg) } } func findStaleCachedErrors() strset.Set { _errorCache.RLock() defer _errorCache.RUnlock() if time.Since(_errorCache.lastCleanup) < _cacheCleanupInterval { return nil } _errorCache.lastCleanup = time.Now() staleErrorMessages := strset.New() for errMsg, errStatus := range _errorCache.m { if time.Since(errStatus.LastSeenTime) > _cacheEvictionPeriod { staleErrorMessages.Add(errMsg) } } return staleErrorMessages } ================================================ FILE: pkg/lib/telemetry/errors.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package telemetry import ( "github.com/cortexlabs/cortex/pkg/lib/errors" ) const ( ErrUserIDNotSpecified = "telemetry.user_id_not_specified" ErrSentryFlushTimeoutExceeded = "telemetry.sentry_flush_timeout_exceeded" ) func ErrorUserIDNotSpecified() error { return errors.WithStack(&errors.Error{ Kind: ErrUserIDNotSpecified, Message: "user ID must be specified to enable telemetry", }) } func ErrorSentryFlushTimeoutExceeded() error { return errors.WithStack(&errors.Error{ Kind: ErrSentryFlushTimeoutExceeded, Message: "sentry flush timout exceeded", }) } ================================================ FILE: pkg/lib/telemetry/telemetry.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package telemetry import ( "os" "reflect" "strings" "time" "github.com/cortexlabs/cortex/pkg/consts" "github.com/cortexlabs/cortex/pkg/lib/cast" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/maps" "github.com/cortexlabs/cortex/pkg/lib/parallel" s "github.com/cortexlabs/cortex/pkg/lib/strings" "github.com/getsentry/sentry-go" "gopkg.in/segmentio/analytics-go.v3" ) var _sentryDSN = "https://5cea3d2d67194d028f7191fcc6ebca14@sentry.io/1825326" var _segmentWriteKey = "BNhXifMk9EyhPICF2zAFpWYPCf4CRpV1" var _segment analytics.Client var _config *Config type Config struct { Enabled bool UserID string Properties map[string]string Environment string LogErrors bool BackoffMode BackoffMode } type BackoffMode int const ( NoBackoff BackoffMode = iota BackoffDuplicateMessages BackoffAnyMessages ) type silentSegmentLogger struct{} func (logger silentSegmentLogger) Logf(_ string, _ ...interface{}) { return } func (logger silentSegmentLogger) Errorf(_ string, _ ...interface{}) { return } type silentSentryLogger struct{} func (logger silentSentryLogger) Write(p []byte) (n int, err error) { return len(p), nil } func getSentryDSN() string { if envVar := os.Getenv("CORTEX_TELEMETRY_SENTRY_DSN"); envVar != "" { return envVar } return _sentryDSN } func Init(telemetryConfig Config) error { if !telemetryConfig.Enabled { _config = nil return nil } if telemetryConfig.UserID == "" { return ErrorUserIDNotSpecified() } err := sentry.Init(sentry.ClientOptions{ Dsn: getSentryDSN(), Release: consts.CortexVersion, Environment: telemetryConfig.Environment, }) if err != nil { _config = nil return err } var segmentLogger analytics.Logger if !telemetryConfig.LogErrors { sentry.Logger.SetOutput(silentSentryLogger{}) segmentLogger = silentSegmentLogger{} } writeKey := _segmentWriteKey if envVar := os.Getenv("CORTEX_TELEMETRY_SEGMENT_WRITE_KEY"); envVar != "" { writeKey = envVar } _segment, err = analytics.NewWithConfig(writeKey, analytics.Config{ BatchSize: 1, Logger: segmentLogger, DefaultContext: &analytics.Context{ App: analytics.AppInfo{ Version: consts.CortexVersion, }, Device: analytics.DeviceInfo{ Type: telemetryConfig.Environment, }, }, }) if err != nil { _config = nil return err } _config = &telemetryConfig return nil } func Event(name string, properties ...map[string]interface{}) { integrations := map[string]interface{}{ "All": true, "Slack": false, } eventHelper(name, maps.MergeStrInterfaceMaps(properties...), integrations) } func EventNotify(name string, properties ...map[string]interface{}) { integrations := map[string]interface{}{ "All": true, } eventHelper(name, maps.MergeStrInterfaceMaps(properties...), integrations) } func eventHelper(name string, properties map[string]interface{}, integrations map[string]interface{}) { if _config == nil || !_config.Enabled || strings.ToLower(os.Getenv("CORTEX_TELEMETRY_DISABLE")) == "true" { return } mergedProperties := maps.MergeStrInterfaceMaps(properties, cast.StrMapToStrInterfaceMap(_config.Properties)) err := _segment.Enqueue(analytics.Track{ Event: name, UserId: _config.UserID, Properties: mergedProperties, Integrations: integrations, }) if err != nil { Error(err) } } func Error(err error, tags ...map[string]string) { if err == nil || _config == nil || errors.IsNoTelemetry(err) { return } if shouldBlock(err, _config.BackoffMode) { return } mergedTags := maps.MergeStrMapsString(tags...) sentry.WithScope(func(scope *sentry.Scope) { e := EventFromException(err) scope.SetUser(sentry.User{ID: _config.UserID}) scope.SetTags(maps.MergeStrMapsString(_config.Properties, mergedTags)) scope.SetTags(map[string]string{"error_type": e.Exception[0].Type}) sentry.CaptureEvent(e) go sentry.Flush(10 * time.Second) }) } func EventFromException(exception error) *sentry.Event { stacktrace := sentry.ExtractStacktrace(exception) if stacktrace == nil { stacktrace = sentry.NewStacktrace() } errTypeString := reflect.TypeOf(errors.CauseOrSelf(exception)).String() errKind := errors.GetKind(exception) if errKind != "" && errKind != errors.ErrNotCortexError { errTypeString = errKind } event := sentry.NewEvent() event.Level = sentry.LevelError value := errors.Message(exception) if metadata := errors.GetMetadata(exception); metadata != nil { value = value + "\n\n########## metadata ##########\n" + s.ObjStripped(metadata) } event.Exception = []sentry.Exception{{ Value: value, Type: errTypeString, Stacktrace: stacktrace, }} return event } func RecordOperatorID(clientID string, operatorID string) { if _config == nil || !_config.Enabled || strings.ToLower(os.Getenv("CORTEX_TELEMETRY_DISABLE")) == "true" { return } _ = _segment.Enqueue(analytics.Identify{ UserId: clientID, Traits: analytics.NewTraits(). Set("operator_id", operatorID), }) } func closeSentry() error { if !sentry.Flush(5 * time.Second) { return ErrorSentryFlushTimeoutExceeded() } return nil } func closeSegment() error { if _segment == nil { return nil } return _segment.Close() } func Close() { parallel.Run(closeSegment, closeSentry) _config = nil } ================================================ FILE: pkg/lib/time/time.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package time import ( "fmt" "strconv" "strings" "time" ) func MicrosecsStr(t time.Time) string { nanos := fmt.Sprintf("%09d", t.Nanosecond()) return nanos[0:6] } func MillisecsStr(t time.Time) string { nanos := fmt.Sprintf("%09d", t.Nanosecond()) return nanos[0:3] } func Timestamp(t time.Time) string { microseconds := MicrosecsStr(t) return fmt.Sprintf("%d-%02d-%02d-%02d-%02d-%02d-%s", t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second(), microseconds) } func PtrsEqual(t1 *time.Time, t2 *time.Time) bool { if t1 == nil && t2 == nil { return true } if t1 == nil || t2 == nil { return false } return t1.Equal(*t2) } func CopyPtr(t *time.Time) *time.Time { if t == nil { return nil } tCopy := *t return &tCopy } func DifferenceStr(t1 *time.Time, t2 *time.Time) string { var duration time.Duration if t1 == nil && t2 == nil { return "-" } else if t1 == nil { return "infinity" } else if t2 == nil { duration = time.Since(*t1) } else { duration = (*t2).Sub(*t1) } durationSecs := int(duration.Seconds()) if durationSecs < 60 { return strconv.Itoa(durationSecs) + "s" } else if durationSecs < 3600 { return strconv.Itoa(durationSecs/60) + "m" + strconv.Itoa(durationSecs-durationSecs/60*60) + "s" } else if durationSecs < 48*3600 { return strconv.Itoa(durationSecs/3600) + "h" + strconv.Itoa((durationSecs-durationSecs/3600*3600)/60) + "m" } else { return strconv.Itoa(durationSecs/(24*3600)) + "d" + strconv.Itoa((durationSecs-durationSecs/(24*3600)*(24*3600))/3600) + "h" } } func SinceStr(t *time.Time) string { if t == nil { return "-" } now := time.Now() return DifferenceStr(t, &now) } func LocalTimestamp(t *time.Time) string { if t == nil { return "-" } return (*t).Local().Format("2006-01-02 15:04:05 MST") } func LocalTimestampHuman(t *time.Time) string { if t == nil { return "-" } return (*t).Local().Format("Monday, January 2, 2006 at 3:04pm MST") } func LocalHourNow() string { return time.Now().Local().Format("3:04:05pm MST") } func MillisToTime(epochMillis int64) time.Time { seconds := epochMillis / 1000 millis := epochMillis % 1000 return time.Unix(seconds, millis*int64(time.Millisecond)) } func ToMillis(t time.Time) int64 { return t.UnixNano() / int64(time.Millisecond) } type Timer struct { names []string start time.Time last time.Time } func StartTimer(names ...string) Timer { return Timer{ names: names, start: time.Now(), } } func (t *Timer) Print(messages ...string) { now := time.Now() separator := "" if len(t.names)+len(messages) > 0 { separator = ": " } totalTime := fmt.Sprintf("%s total", now.Sub(t.start)) stepTime := "" if !t.last.IsZero() { stepTime = fmt.Sprintf("%s step, ", now.Sub(t.last)) } fmt.Println(strings.Join(append(t.names, messages...), ": ") + separator + stepTime + totalTime) t.last = now } func MustParseDuration(str string) time.Duration { d, err := time.ParseDuration(str) if err != nil { panic(err) } return d } func MaxDuration(duration time.Duration, durations ...time.Duration) time.Duration { max := duration for _, d := range durations { if d > max { max = d } } return max } func GetCurrentUTCDate() time.Time { timestamp := time.Now().UTC() year, month, day := timestamp.Date() return time.Date(year, month, day, 0, 0, 0, 0, time.UTC) } ================================================ FILE: pkg/lib/urls/errors.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package urls import ( "fmt" "github.com/cortexlabs/cortex/pkg/lib/errors" s "github.com/cortexlabs/cortex/pkg/lib/strings" ) const ( ErrInvalidURL = "urls.invalid_url" ErrDNS1035 = "urls.dns1035" ErrDNS1123 = "urls.dns1123" ErrEndpoint = "urls.endpoint" ErrEndpointEmptyPath = "urls.endpoint_empty_path" ErrEndpointDoubleSlash = "urls.endpoint_double_slash" ) func ErrorInvalidURL(provided string) error { return errors.WithStack(&errors.Error{ Kind: ErrInvalidURL, Message: fmt.Sprintf("%s is not a valid URL", s.UserStr(provided)), }) } func ErrorDNS1035(provided string) error { return errors.WithStack(&errors.Error{ Kind: ErrDNS1035, Message: fmt.Sprintf("%s must contain only lower case letters, numbers, and dashes, start with a letter, and cannot end with a dash", s.UserStr(provided)), }) } func ErrorDNS1123(provided string) error { return errors.WithStack(&errors.Error{ Kind: ErrDNS1123, Message: fmt.Sprintf("%s must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character", s.UserStr(provided)), }) } func ErrorEndpoint(provided string) error { return errors.WithStack(&errors.Error{ Kind: ErrEndpoint, Message: fmt.Sprintf("%s must consist of lower case alphanumeric characters, '/', '-', '_', or '.'", s.UserStr(provided)), }) } func ErrorEndpointEmptyPath() error { return errors.WithStack(&errors.Error{ Kind: ErrEndpointEmptyPath, Message: fmt.Sprintf("%s is not allowed (a path must be specified)", s.UserStr("/")), }) } func ErrorEndpointDoubleSlash(provided string) error { return errors.WithStack(&errors.Error{ Kind: ErrEndpointDoubleSlash, Message: fmt.Sprintf("%s cannot contain adjacent slashes", s.UserStr(provided)), }) } ================================================ FILE: pkg/lib/urls/urls.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package urls import ( "net/url" "regexp" "strings" s "github.com/cortexlabs/cortex/pkg/lib/strings" ) var ( _dns1035Regex = regexp.MustCompile(`^[a-z]([-a-z0-9]*[a-z0-9])?$`) _dns1123Regex = regexp.MustCompile(`^[a-z0-9]([-a-z0-9]*[a-z0-9])?$`) _endpointRegex = regexp.MustCompile(`^[a-zA-Z0-9_\-\./]*$`) _urlQParamRegex = regexp.MustCompile(`(https?://.*)\?[^:\s]*`) ) func Parse(rawurl string) (*url.URL, error) { u, err := url.Parse(rawurl) if err != nil { return nil, ErrorInvalidURL(rawurl) } return u, nil } func Join(str string, strs ...string) string { fullPath := str for _, str := range strs { fullPath = s.EnsureSuffix(fullPath, "/") fullPath = fullPath + strings.TrimPrefix(str, "/") } return fullPath } func CheckDNS1035(str string) error { if !_dns1035Regex.MatchString(str) { return ErrorDNS1035(str) } return nil } func CheckDNS1123(str string) error { if !_dns1123Regex.MatchString(str) { return ErrorDNS1123(str) } return nil } func ValidateEndpointAllowEmptyPath(str string) (string, error) { if !_endpointRegex.MatchString(str) { return "", ErrorEndpoint(str) } if strings.Contains(str, "//") { return "", ErrorEndpointDoubleSlash(str) } return CanonicalizeEndpoint(str), nil } func ValidateEndpoint(str string) (string, error) { path, err := ValidateEndpointAllowEmptyPath(str) if err != nil { return "", err } if path == "/" { return "", ErrorEndpointEmptyPath() } return path, nil } func CanonicalizeEndpoint(str string) string { if str == "" || str == "/" { return "/" } return strings.TrimSuffix(s.EnsurePrefix(str, "/"), "/") } func CanonicalizeEndpointWithTrailingSlash(str string) string { if str == "" || str == "/" { return "/" } return s.EnsureSuffix(s.EnsurePrefix(str, "/"), "/") } func TrimQueryParamsURL(u url.URL) string { u.RawQuery = "" return u.String() } func TrimQueryParamsStr(str string) string { return _urlQParamRegex.ReplaceAllString(str, "$1") } ================================================ FILE: pkg/operator/endpoints/delete.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package endpoints import ( "net/http" "github.com/cortexlabs/cortex/pkg/operator/resources" "github.com/gorilla/mux" ) func Delete(w http.ResponseWriter, r *http.Request) { apiName := mux.Vars(r)["apiName"] keepCache := getOptionalBoolQParam("keepCache", false, r) response, err := resources.DeleteAPI(apiName, keepCache) if err != nil { respondError(w, r, err) return } respondJSON(w, r, response) } ================================================ FILE: pkg/operator/endpoints/deploy.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package endpoints import ( "net/http" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/files" "github.com/cortexlabs/cortex/pkg/operator/resources" ) func Deploy(w http.ResponseWriter, r *http.Request) { force := getOptionalBoolQParam("force", false, r) configFileName, err := getRequiredQueryParam("configFileName", r) if err != nil { respondError(w, r, errors.WithStack(err)) return } configBytes, err := files.ReadReqFile(r, "config") if err != nil { respondError(w, r, errors.WithStack(err)) return } else if len(configBytes) == 0 { respondError(w, r, ErrorFormFileMustBeProvided("config")) return } response, err := resources.Deploy(configFileName, configBytes, force) if err != nil { respondError(w, r, err) return } respondJSON(w, r, response) } ================================================ FILE: pkg/operator/endpoints/describe.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package endpoints import ( "net/http" "github.com/cortexlabs/cortex/pkg/operator/resources" "github.com/gorilla/mux" ) func DescribeAPI(w http.ResponseWriter, r *http.Request) { apiName := mux.Vars(r)["apiName"] response, err := resources.DescribeAPI(apiName) if err != nil { respondError(w, r, err) return } respondJSON(w, r, response) } ================================================ FILE: pkg/operator/endpoints/errors.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package endpoints import ( "fmt" "github.com/cortexlabs/cortex/pkg/lib/errors" s "github.com/cortexlabs/cortex/pkg/lib/strings" "github.com/cortexlabs/cortex/pkg/operator/operator" ) const ( ErrAPIVersionMismatch = "endpoints.api_version_mismatch" ErrHeaderMissing = "endpoints.header_missing" ErrHeaderMalformed = "endpoints.header_malformed" ErrAuthAPIError = "endpoints.auth_api_error" ErrFormFileMustBeProvided = "endpoints.form_file_must_be_provided" ErrAuthInvalid = "endpoints.auth_invalid" ErrAuthOtherAccount = "endpoints.auth_other_account" ErrQueryParamRequired = "endpoints.query_param_required" ErrPathParamRequired = "endpoints.path_param_required" ErrAnyQueryParamRequired = "endpoints.any_query_param_required" ErrAnyPathParamRequired = "endpoints.any_path_param_required" ErrLogsJobIDRequired = "endpoints.logs_job_id_required" ) func ErrorAPIVersionMismatch(operatorVersion string, clientVersion string) error { return errors.WithStack(&errors.Error{ Kind: ErrAPIVersionMismatch, Message: fmt.Sprintf("your CLI version (%s) doesn't match your Cortex operator version (%s); please update your cluster by following the instructions at https://docs.cortexlabs.com, or update your CLI (pip install cortex==%s)", clientVersion, operatorVersion, operatorVersion), }) } func ErrorHeaderMissing(header string) error { return errors.WithStack(&errors.Error{ Kind: ErrHeaderMissing, Message: fmt.Sprintf("missing %s header", header), }) } func ErrorAuthHeaderMissing(header, host, url string) error { return errors.WithStack(&errors.Error{ Kind: ErrHeaderMissing, Message: fmt.Sprintf("missing %s header", header), Metadata: map[string]string{ "host": host, "url": url, }, }) } func ErrorHeaderMalformed(header string) error { return errors.WithStack(&errors.Error{ Kind: ErrHeaderMalformed, Message: fmt.Sprintf("malformed %s header", header), }) } func ErrorAuthAPIError() error { return errors.WithStack(&errors.Error{ Kind: ErrAuthAPIError, Message: "the operator is unable to verify user's credentials using AWS STS; run `aws sts get-caller-identity` to view the credentials being used by the cortex client", }) } func ErrorAuthInvalid() error { return errors.WithStack(&errors.Error{ Kind: ErrAuthInvalid, Message: "invalid AWS credentials; run `aws sts get-caller-identity` to view the credentials being used by the cortex client", }) } func ErrorAuthOtherAccount() error { return errors.WithStack(&errors.Error{ Kind: ErrAuthOtherAccount, Message: "the AWS account associated with your CLI's AWS credentials differs from the AWS account associated with your cluster's AWS credentials; run `aws sts get-caller-identity` to view the credentials being used by the cortex client", }) } func ErrorFormFileMustBeProvided(fileName string) error { return errors.WithStack(&errors.Error{ Kind: ErrFormFileMustBeProvided, Message: fmt.Sprintf("request form file %s must be provided", s.UserStr(fileName)), }) } func ErrorQueryParamRequired(param string) error { return errors.WithStack(&errors.Error{ Kind: ErrQueryParamRequired, Message: fmt.Sprintf("query param required: %s", param), }) } func ErrorPathParamRequired(param string) error { return errors.WithStack(&errors.Error{ Kind: ErrPathParamRequired, Message: fmt.Sprintf("path param required: %s", param), }) } func ErrorAnyQueryParamRequired(param string, params ...string) error { allParams := append([]string{param}, params...) return errors.WithStack(&errors.Error{ Kind: ErrAnyQueryParamRequired, Message: fmt.Sprintf("query params required: %s", s.UserStrsOr(allParams)), }) } func ErrorAnyPathParamRequired(param string, params ...string) error { allParams := append([]string{param}, params...) return errors.WithStack(&errors.Error{ Kind: ErrAnyPathParamRequired, Message: fmt.Sprintf("path params required: %s", s.UserStrsOr(allParams)), }) } func ErrorLogsJobIDRequired(resource operator.DeployedResource) error { return errors.WithStack(&errors.Error{ Kind: ErrLogsJobIDRequired, Message: fmt.Sprintf("job id is required for %s; you can get a list of latest job ids with `cortex get %s` and use `cortex logs %s JOB_ID` to get the logs", resource.UserString(), resource.Name, resource.Name), }) } ================================================ FILE: pkg/operator/endpoints/get.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package endpoints import ( "net/http" "github.com/cortexlabs/cortex/pkg/operator/resources" "github.com/gorilla/mux" ) func GetAPIs(w http.ResponseWriter, r *http.Request) { response, err := resources.GetAPIs() if err != nil { respondError(w, r, err) return } respondJSON(w, r, response) } func GetAPI(w http.ResponseWriter, r *http.Request) { apiName := mux.Vars(r)["apiName"] response, err := resources.GetAPI(apiName) if err != nil { respondError(w, r, err) return } respondJSON(w, r, response) } func GetAPIByID(w http.ResponseWriter, r *http.Request) { apiName := mux.Vars(r)["apiName"] apiID := mux.Vars(r)["apiID"] response, err := resources.GetAPIByID(apiName, apiID) if err != nil { respondError(w, r, err) return } respondJSON(w, r, response) } ================================================ FILE: pkg/operator/endpoints/get_batch_job.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package endpoints import ( "net/http" "github.com/cortexlabs/cortex/pkg/operator/resources" "github.com/cortexlabs/cortex/pkg/operator/resources/job/batchapi" "github.com/cortexlabs/cortex/pkg/types/spec" "github.com/cortexlabs/cortex/pkg/types/userconfig" "github.com/gorilla/mux" ) func GetBatchJob(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) apiName := vars["apiName"] jobID, err := getRequiredQueryParam("jobID", r) if err != nil { respondError(w, r, err) return } deployedResource, err := resources.GetDeployedResourceByName(apiName) if err != nil { respondError(w, r, err) return } if deployedResource.Kind != userconfig.BatchAPIKind { respondError(w, r, resources.ErrorOperationIsOnlySupportedForKind(*deployedResource, userconfig.BatchAPIKind)) return } jobKey := spec.JobKey{ APIName: apiName, ID: jobID, Kind: userconfig.BatchAPIKind, } jobResponse, err := batchapi.GetJob(jobKey) if err != nil { respondError(w, r, err) return } respondJSON(w, r, jobResponse) } ================================================ FILE: pkg/operator/endpoints/get_task_job.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package endpoints import ( "net/http" "net/url" "github.com/cortexlabs/cortex/pkg/operator/operator" "github.com/cortexlabs/cortex/pkg/operator/resources" "github.com/cortexlabs/cortex/pkg/operator/resources/job/taskapi" "github.com/cortexlabs/cortex/pkg/operator/schema" "github.com/cortexlabs/cortex/pkg/types/spec" "github.com/cortexlabs/cortex/pkg/types/userconfig" "github.com/gorilla/mux" ) func GetTaskJob(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) apiName := vars["apiName"] jobID, err := getRequiredQueryParam("jobID", r) if err != nil { respondError(w, r, err) return } deployedResource, err := resources.GetDeployedResourceByName(apiName) if err != nil { respondError(w, r, err) return } if deployedResource.Kind != userconfig.TaskAPIKind { respondError(w, r, resources.ErrorOperationIsOnlySupportedForKind(*deployedResource, userconfig.TaskAPIKind)) return } jobKey := spec.JobKey{ APIName: apiName, ID: jobID, Kind: userconfig.TaskAPIKind, } jobStatus, err := taskapi.GetJobStatus(jobKey) if err != nil { respondError(w, r, err) return } apiSpec, err := operator.DownloadAPISpec(jobStatus.APIName, jobStatus.APIID) if err != nil { respondError(w, r, err) return } endpoint, err := operator.APIEndpoint(apiSpec) if err != nil { respondError(w, r, err) return } parsedURL, err := url.Parse(endpoint) if err != nil { respondError(w, r, err) } q := parsedURL.Query() q.Add("jobID", jobKey.ID) parsedURL.RawQuery = q.Encode() response := schema.TaskJobResponse{ JobStatus: *jobStatus, APISpec: *apiSpec, Endpoint: parsedURL.String(), } respondJSON(w, r, response) } ================================================ FILE: pkg/operator/endpoints/info.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package endpoints import ( "net/http" "sort" "github.com/cortexlabs/cortex/pkg/config" "github.com/cortexlabs/cortex/pkg/lib/aws" "github.com/cortexlabs/cortex/pkg/lib/k8s" "github.com/cortexlabs/cortex/pkg/operator/schema" "github.com/cortexlabs/cortex/pkg/types/clusterconfig" "github.com/cortexlabs/cortex/pkg/types/userconfig" kcore "k8s.io/api/core/v1" ) func Info(w http.ResponseWriter, r *http.Request) { workerNodeInfos, numPendingReplicas, err := getWorkerNodeInfos() if err != nil { respondError(w, r, err) return } operatorNodeInfos, err := getOperatorNodeInfos() if err != nil { respondError(w, r, err) return } fullClusterConfig := clusterconfig.InternalConfig{ Config: *config.ClusterConfig, OperatorMetadata: *config.OperatorMetadata, } response := schema.InfoResponse{ ClusterConfig: fullClusterConfig, WorkerNodeInfos: workerNodeInfos, OperatorNodeInfos: operatorNodeInfos, NumPendingReplicas: numPendingReplicas, } respondJSON(w, r, response) } func getWorkerNodeInfos() ([]schema.WorkerNodeInfo, int, error) { pods, err := config.K8sAllNamspaces.ListPods(nil) if err != nil { return nil, 0, err } nodes, err := config.K8sAllNamspaces.ListNodesByLabel("workload", "true") if err != nil { return nil, 0, err } nodeInfoMap := make(map[string]*schema.WorkerNodeInfo, len(nodes)) // node name -> info spotPriceCache := make(map[string]float64) // instance type -> spot price for i := range nodes { node := nodes[i] instanceType := node.Labels["node.kubernetes.io/instance-type"] nodeGroupName := node.Labels["alpha.eksctl.io/nodegroup-name"] isSpot := node.Labels["node-lifecycle"] == "spot" price := aws.InstanceMetadatas[config.ClusterConfig.Region][instanceType].Price if isSpot { if spotPrice, ok := spotPriceCache[instanceType]; ok { price = spotPrice } else { spotPrice, err := config.AWS.SpotInstancePrice(instanceType) if err == nil && spotPrice != 0 { price = spotPrice spotPriceCache[instanceType] = spotPrice } else { spotPriceCache[instanceType] = price // the request failed, so no need to try again in the future } } } nodeInfoMap[node.Name] = &schema.WorkerNodeInfo{ NodeInfo: schema.NodeInfo{ NodeGroupName: nodeGroupName, InstanceType: instanceType, IsSpot: isSpot, Price: price, }, Name: node.Name, NumReplicas: 0, // will be added to below ComputeUserCapacity: nodeComputeAllocatable(&node), // will be subtracted from below ComputeAvailable: nodeComputeAllocatable(&node), // will be subtracted from below ComputeUserRequested: userconfig.ZeroCompute(), // will be added to below } } var numPendingReplicas int for i := range pods { pod := pods[i] if pod.Status.Phase == kcore.PodSucceeded || pod.Status.Phase == kcore.PodFailed { // note: pending pods can be scheduled on nodes (image pull in progress) continue } _, isAPIPod := pod.Labels["apiName"] batchPodType, isBatchPod := pod.Labels["cortex.dev/batch"] if pod.Spec.NodeName == "" && isAPIPod { numPendingReplicas++ continue } node, ok := nodeInfoMap[pod.Spec.NodeName] if !ok { continue } if isAPIPod { if isBatchPod && batchPodType == "enqueuer" { node.NumEnqueuerReplicas++ } else { node.NumReplicas++ } } cpu, mem, gpu, inf := k8s.TotalPodCompute(&pod.Spec) node.ComputeAvailable.CPU.SubQty(cpu) node.ComputeAvailable.Mem.SubQty(mem) node.ComputeAvailable.GPU -= gpu node.ComputeAvailable.Inf -= inf if isAPIPod { node.ComputeUserRequested.CPU.AddQty(cpu) node.ComputeUserRequested.Mem.AddQty(mem) node.ComputeUserRequested.GPU += gpu node.ComputeUserRequested.Inf += inf } else { node.ComputeUserCapacity.CPU.SubQty(cpu) node.ComputeUserCapacity.Mem.SubQty(mem) node.ComputeUserCapacity.GPU -= gpu node.ComputeUserCapacity.Inf -= inf } } nodeNames := make([]string, 0, len(nodeInfoMap)) for nodeName := range nodeInfoMap { nodeNames = append(nodeNames, nodeName) } sort.Strings(nodeNames) nodeInfos := make([]schema.WorkerNodeInfo, len(nodeNames)) for i, nodeName := range nodeNames { nodeInfos[i] = *nodeInfoMap[nodeName] } return nodeInfos, numPendingReplicas, nil } func nodeComputeAllocatable(node *kcore.Node) userconfig.Compute { gpuQty := node.Status.Allocatable["nvidia.com/gpu"] infQty := node.Status.Allocatable["aws.amazon.com/neuron"] return userconfig.Compute{ CPU: k8s.WrapQuantity(*node.Status.Allocatable.Cpu()), Mem: k8s.WrapQuantity(*node.Status.Allocatable.Memory()), GPU: gpuQty.Value(), Inf: infQty.Value(), } } func getOperatorNodeInfos() ([]schema.NodeInfo, error) { nodes, err := config.K8sAllNamspaces.ListNodesByLabel("operator", "true") if err != nil { return nil, err } nodeInfoMap := make(map[string]*schema.NodeInfo, len(nodes)) // node name -> info for i := range nodes { node := nodes[i] instanceType := node.Labels["node.kubernetes.io/instance-type"] nodeGroupName := node.Labels["alpha.eksctl.io/nodegroup-name"] price := aws.InstanceMetadatas[config.ClusterConfig.Region][instanceType].Price nodeInfoMap[node.Name] = &schema.NodeInfo{ NodeGroupName: nodeGroupName, InstanceType: instanceType, Price: price, } } nodeNames := make([]string, 0, len(nodeInfoMap)) for nodeName := range nodeInfoMap { nodeNames = append(nodeNames, nodeName) } sort.Strings(nodeNames) nodeInfos := make([]schema.NodeInfo, len(nodeNames)) for i, nodeName := range nodeNames { nodeInfos[i] = *nodeInfoMap[nodeName] } return nodeInfos, nil } ================================================ FILE: pkg/operator/endpoints/logs.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package endpoints import ( "net/http" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/operator/operator" "github.com/cortexlabs/cortex/pkg/operator/resources" "github.com/cortexlabs/cortex/pkg/operator/resources/asyncapi" "github.com/cortexlabs/cortex/pkg/operator/resources/realtimeapi" "github.com/cortexlabs/cortex/pkg/operator/schema" "github.com/cortexlabs/cortex/pkg/types/userconfig" "github.com/gorilla/mux" "github.com/gorilla/websocket" ) func ReadLogs(w http.ResponseWriter, r *http.Request) { apiName := mux.Vars(r)["apiName"] jobID := getOptionalQParam("jobID", r) if jobID != "" { ReadJobLogs(w, r) return } deployedResource, err := resources.GetDeployedResourceByName(apiName) if err != nil { respondError(w, r, err) return } if deployedResource.Kind == userconfig.BatchAPIKind || deployedResource.Kind == userconfig.TaskAPIKind { respondError(w, r, ErrorLogsJobIDRequired(*deployedResource)) return } else if deployedResource.Kind != userconfig.RealtimeAPIKind && deployedResource.Kind != userconfig.AsyncAPIKind { respondError(w, r, resources.ErrorOperationIsOnlySupportedForKind(*deployedResource, userconfig.RealtimeAPIKind)) return } deploymentID := deployedResource.VirtualService.Labels["deploymentID"] podID := deployedResource.VirtualService.Labels["podID"] upgrader := websocket.Upgrader{} socket, err := upgrader.Upgrade(w, r, nil) if err != nil { respondError(w, r, err) return } defer socket.Close() labels := map[string]string{"apiName": apiName, "deploymentID": deploymentID, "podID": podID} operator.StreamLogsFromRandomPod(labels, socket) } func GetLogURL(w http.ResponseWriter, r *http.Request) { apiName := mux.Vars(r)["apiName"] jobID := getOptionalQParam("jobID", r) if jobID != "" { GetJobLogURL(w, r) return } deployedResource, err := resources.GetDeployedResourceByName(apiName) if err != nil { respondError(w, r, err) return } if deployedResource.Kind == userconfig.BatchAPIKind || deployedResource.Kind == userconfig.TaskAPIKind { respondError(w, r, ErrorLogsJobIDRequired(*deployedResource)) return } switch deployedResource.Kind { case userconfig.AsyncAPIKind: apiResponse, err := asyncapi.GetAPIByName(deployedResource) if err != nil { respondError(w, r, err) return } if apiResponse[0].Spec == nil { respondError(w, r, errors.ErrorUnexpected("unable to get api spec", apiName)) } logURL, err := operator.APILogURL(*apiResponse[0].Spec) if err != nil { respondError(w, r, err) return } respondJSON(w, r, schema.LogResponse{ LogURL: logURL, }) case userconfig.RealtimeAPIKind: apiResponse, err := realtimeapi.GetAPIByName(deployedResource) if err != nil { respondError(w, r, err) return } if apiResponse[0].Spec == nil { respondError(w, r, errors.ErrorUnexpected("unable to get api spec", apiName)) } logURL, err := operator.APILogURL(*apiResponse[0].Spec) if err != nil { respondError(w, r, err) return } respondJSON(w, r, schema.LogResponse{ LogURL: logURL, }) default: respondError(w, r, resources.ErrorOperationIsOnlySupportedForKind(*deployedResource, userconfig.RealtimeAPIKind, userconfig.AsyncAPIKind)) } } ================================================ FILE: pkg/operator/endpoints/logs_job.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package endpoints import ( "net/http" "github.com/cortexlabs/cortex/pkg/operator/operator" "github.com/cortexlabs/cortex/pkg/operator/resources" "github.com/cortexlabs/cortex/pkg/operator/resources/job/batchapi" "github.com/cortexlabs/cortex/pkg/operator/resources/job/taskapi" "github.com/cortexlabs/cortex/pkg/operator/schema" "github.com/cortexlabs/cortex/pkg/types/spec" "github.com/cortexlabs/cortex/pkg/types/userconfig" "github.com/gorilla/mux" "github.com/gorilla/websocket" ) func ReadJobLogs(w http.ResponseWriter, r *http.Request) { apiName := mux.Vars(r)["apiName"] jobID, err := getRequiredQueryParam("jobID", r) if err != nil { respondError(w, r, err) return } deployedResource, err := resources.GetDeployedResourceByName(apiName) if err != nil { respondError(w, r, err) return } if deployedResource.Kind != userconfig.BatchAPIKind && deployedResource.Kind != userconfig.TaskAPIKind { respondError(w, r, resources.ErrorOperationIsOnlySupportedForKind(*deployedResource, userconfig.BatchAPIKind, userconfig.TaskAPIKind)) return } upgrader := websocket.Upgrader{} socket, err := upgrader.Upgrade(w, r, nil) if err != nil { respondError(w, r, err) return } defer socket.Close() labels := map[string]string{"apiName": apiName, "jobID": jobID} if deployedResource.Kind == userconfig.BatchAPIKind { labels["cortex.dev/batch"] = "worker" } operator.StreamLogsFromRandomPod(labels, socket) } func GetJobLogURL(w http.ResponseWriter, r *http.Request) { apiName := mux.Vars(r)["apiName"] jobID, err := getRequiredQueryParam("jobID", r) if err != nil { respondError(w, r, err) return } deployedResource, err := resources.GetDeployedResourceByName(apiName) if err != nil { respondError(w, r, err) return } switch deployedResource.Kind { case userconfig.BatchAPIKind: jobResponse, err := batchapi.GetJob(spec.JobKey{ ID: jobID, APIName: apiName, Kind: userconfig.BatchAPIKind, }) if err != nil { respondError(w, r, err) return } logURL, err := operator.BatchJobLogURL(apiName, jobResponse.JobStatus) if err != nil { respondError(w, r, err) return } respondJSON(w, r, schema.LogResponse{ LogURL: logURL, }) case userconfig.TaskAPIKind: jobStatus, err := taskapi.GetJobStatus(spec.JobKey{ ID: jobID, APIName: apiName, Kind: userconfig.TaskAPIKind, }) if err != nil { respondError(w, r, err) return } logURL, err := operator.TaskJobLogURL(apiName, *jobStatus) if err != nil { respondError(w, r, err) return } respondJSON(w, r, schema.LogResponse{ LogURL: logURL, }) default: respondError(w, r, resources.ErrorOperationIsOnlySupportedForKind(*deployedResource, userconfig.BatchAPIKind, userconfig.TaskAPIKind)) } } ================================================ FILE: pkg/operator/endpoints/middleware.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package endpoints import ( "context" "net/http" "github.com/cortexlabs/cortex/pkg/config" "github.com/cortexlabs/cortex/pkg/consts" "github.com/cortexlabs/cortex/pkg/lib/aws" "github.com/cortexlabs/cortex/pkg/lib/sets/strset" "github.com/cortexlabs/cortex/pkg/lib/telemetry" ) var _cachedClientIDs = strset.New() type ctxKey int const ( ctxKeyUnknown ctxKey = iota ctxKeyClient ) func PanicMiddleware(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { defer recoverAndRespond(w, r) next.ServeHTTP(w, r) }) } func ClientIDMiddleware(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if clientID := r.URL.Query().Get("clientID"); clientID != "" { // Add clientID to context ctx := context.WithValue(r.Context(), ctxKeyClient, clientID) r = r.WithContext(ctx) if !_cachedClientIDs.Has(clientID) { telemetry.RecordOperatorID(clientID, config.OperatorMetadata.OperatorID) _cachedClientIDs.Add(clientID) } } next.ServeHTTP(w, r) }) } func AWSAuthMiddleware(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { authHeader := r.Header.Get(consts.AuthHeader) if authHeader == "" { respondError(w, r, ErrorAuthHeaderMissing(consts.AuthHeader, r.Host, r.RequestURI)) return } accountID, err := aws.ExecuteIdentityRequestFromHeader(authHeader) if err != nil { respondError(w, r, err) return } operatorAccountID, _, err := config.AWS.GetCachedAccountID() if err != nil { respondError(w, r, ErrorAuthAPIError()) return } if accountID != operatorAccountID { respondErrorCode(w, r, http.StatusForbidden, ErrorAuthOtherAccount()) return } next.ServeHTTP(w, r) }) } func APIVersionCheckMiddleware(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == "/info" { next.ServeHTTP(w, r) return } clientVersion := r.Header.Get("CortexAPIVersion") if clientVersion == "" { respondError(w, r, ErrorHeaderMissing("CortexAPIVersion")) return } if clientVersion != consts.CortexVersion { respondError(w, r, ErrorAPIVersionMismatch(consts.CortexVersion, clientVersion)) return } next.ServeHTTP(w, r) }) } ================================================ FILE: pkg/operator/endpoints/params.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package endpoints import ( "net/http" s "github.com/cortexlabs/cortex/pkg/lib/strings" "github.com/gorilla/mux" ) func getRequiredPathParam(paramName string, r *http.Request) (string, error) { param := mux.Vars(r)[paramName] if param == "" { return "", ErrorPathParamRequired(paramName) } return param, nil } func getRequiredQueryParam(paramName string, r *http.Request) (string, error) { param := r.URL.Query().Get(paramName) if param == "" { return "", ErrorQueryParamRequired(paramName) } return param, nil } func getOptionalQParam(paramName string, r *http.Request) string { return r.URL.Query().Get(paramName) } func getOptionalBoolQParam(paramName string, defaultVal bool, r *http.Request) bool { param := r.URL.Query().Get(paramName) paramBool, ok := s.ParseBool(param) if ok { return paramBool } return defaultVal } ================================================ FILE: pkg/operator/endpoints/refresh.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package endpoints import ( "net/http" "github.com/cortexlabs/cortex/pkg/operator/resources" "github.com/cortexlabs/cortex/pkg/operator/schema" "github.com/gorilla/mux" ) func Refresh(w http.ResponseWriter, r *http.Request) { apiName := mux.Vars(r)["apiName"] force := getOptionalBoolQParam("force", false, r) msg, err := resources.RefreshAPI(apiName, force) if err != nil { respondError(w, r, err) return } response := schema.RefreshResponse{ Message: msg, } respondJSON(w, r, response) } ================================================ FILE: pkg/operator/endpoints/respond.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package endpoints import ( "encoding/json" "net/http" "github.com/cortexlabs/cortex/pkg/lib/errors" libjson "github.com/cortexlabs/cortex/pkg/lib/json" "github.com/cortexlabs/cortex/pkg/lib/logging" "github.com/cortexlabs/cortex/pkg/lib/telemetry" "github.com/cortexlabs/cortex/pkg/operator/schema" ) var operatorLogger = logging.GetLogger() func respondJSON(w http.ResponseWriter, r *http.Request, response interface{}) { jsonBytes, err := libjson.Marshal(response) if err != nil { respondError(w, r, errors.Wrap(err, "failed to encode response")) return } w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) w.Write(jsonBytes) } func respondError(w http.ResponseWriter, r *http.Request, err error, strs ...string) { respondErrorCode(w, r, http.StatusBadRequest, err, strs...) } func respondErrorCode(w http.ResponseWriter, r *http.Request, code int, err error, strs ...string) { err = errors.Wrap(err, strs...) if !errors.IsNoTelemetry(err) { errTags := map[string]string{} if clientID := r.Context().Value(ctxKeyClient); clientID != nil { if clientIDStr, ok := clientID.(string); ok { errTags["client_id"] = clientIDStr } } telemetry.Error(err, errTags) } if !errors.IsNoPrint(err) { operatorLogger.Error(err) } w.Header().Set("Content-Type", "application/json") w.WriteHeader(code) response := schema.ErrorResponse{ Kind: errors.GetKind(err), Message: errors.Message(err), } json.NewEncoder(w).Encode(response) } func recoverAndRespond(w http.ResponseWriter, r *http.Request, strs ...string) { if errInterface := recover(); errInterface != nil { err := errors.CastRecoverError(errInterface, strs...) operatorLogger.Error(err) telemetry.Error(err) respondError(w, r, err) } } ================================================ FILE: pkg/operator/endpoints/stop_batch_job.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package endpoints import ( "fmt" "net/http" "github.com/cortexlabs/cortex/pkg/operator/resources/job/batchapi" "github.com/cortexlabs/cortex/pkg/operator/schema" "github.com/cortexlabs/cortex/pkg/types/spec" "github.com/cortexlabs/cortex/pkg/types/userconfig" "github.com/gorilla/mux" ) func StopBatchJob(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) apiName := vars["apiName"] jobID, err := getRequiredQueryParam("jobID", r) if err != nil { respondError(w, r, err) return } err = batchapi.StopJob(spec.JobKey{ APIName: apiName, ID: jobID, Kind: userconfig.BatchAPIKind, }) if err != nil { respondError(w, r, err) return } respondJSON(w, r, schema.DeleteResponse{ Message: fmt.Sprintf("stopped job %s", jobID), }) } ================================================ FILE: pkg/operator/endpoints/stop_task_job.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package endpoints import ( "fmt" "net/http" "github.com/cortexlabs/cortex/pkg/operator/resources/job/taskapi" "github.com/cortexlabs/cortex/pkg/operator/schema" "github.com/cortexlabs/cortex/pkg/types/spec" "github.com/cortexlabs/cortex/pkg/types/userconfig" "github.com/gorilla/mux" ) func StopTaskJob(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) apiName := vars["apiName"] jobID, err := getRequiredQueryParam("jobID", r) if err != nil { respondError(w, r, err) return } err = taskapi.StopJob(spec.JobKey{ APIName: apiName, ID: jobID, Kind: userconfig.TaskAPIKind, }) if err != nil { respondError(w, r, err) return } respondJSON(w, r, schema.DeleteResponse{ Message: fmt.Sprintf("stopped job %s", jobID), }) } ================================================ FILE: pkg/operator/endpoints/submit_batch.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package endpoints import ( "encoding/json" "fmt" "io" "io/ioutil" "net/http" "github.com/cortexlabs/cortex/pkg/consts" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/operator/resources" "github.com/cortexlabs/cortex/pkg/operator/resources/job/batchapi" "github.com/cortexlabs/cortex/pkg/operator/schema" "github.com/cortexlabs/cortex/pkg/types/userconfig" "github.com/gorilla/mux" ) func SubmitBatchJob(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) apiName := vars["apiName"] dryRun := getOptionalBoolQParam("dryRun", false, r) deployedResource, err := resources.GetDeployedResourceByName(apiName) if err != nil { respondError(w, r, err) return } if deployedResource.Kind != userconfig.BatchAPIKind { respondError(w, r, resources.ErrorOperationIsOnlySupportedForKind(*deployedResource, userconfig.BatchAPIKind)) return } // max payload size, same as API Gateway rw := http.MaxBytesReader(w, r.Body, 10<<20) bodyBytes, err := ioutil.ReadAll(rw) if err != nil { respondError(w, r, err) return } submission := schema.BatchJobSubmission{} err = json.Unmarshal(bodyBytes, &submission) if err != nil { respondError(w, r, errors.Append(err, fmt.Sprintf("\n\njob submission schema can be found at https://docs.cortexlabs.com/v/%s/", consts.CortexVersionMinor))) return } if dryRun { // plain text response for dry run because it is typically consumed by people w.Header().Set("Content-type", "text/plain") fileNames, err := batchapi.DryRun(&submission) if err != nil { w.WriteHeader(http.StatusBadRequest) _, _ = io.WriteString(w, "\n"+err.Error()+"\n") return } for _, fileName := range fileNames { _, err := io.WriteString(w, fileName+"\n") if err != nil { w.WriteHeader(http.StatusBadRequest) _, _ = io.WriteString(w, "\n"+err.Error()+"\n") return } } _, _ = io.WriteString(w, "validations passed") return } jobSpec, err := batchapi.SubmitJob(apiName, &submission) if err != nil { respondError(w, r, err) return } respondJSON(w, r, jobSpec) } ================================================ FILE: pkg/operator/endpoints/submit_task.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package endpoints import ( "encoding/json" "fmt" "io/ioutil" "net/http" "github.com/cortexlabs/cortex/pkg/consts" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/operator/resources" "github.com/cortexlabs/cortex/pkg/operator/resources/job/taskapi" "github.com/cortexlabs/cortex/pkg/operator/schema" "github.com/cortexlabs/cortex/pkg/types/spec" "github.com/cortexlabs/cortex/pkg/types/userconfig" "github.com/gorilla/mux" ) func SubmitTaskJob(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) apiName := vars["apiName"] deployedResource, err := resources.GetDeployedResourceByName(apiName) if err != nil { respondError(w, r, err) return } if deployedResource.Kind != userconfig.TaskAPIKind { respondError(w, r, resources.ErrorOperationIsOnlySupportedForKind(*deployedResource, userconfig.TaskAPIKind)) return } // max payload size, same as API Gateway rw := http.MaxBytesReader(w, r.Body, 10<<20) bodyBytes, err := ioutil.ReadAll(rw) if err != nil { respondError(w, r, err) return } submission := schema.TaskJobSubmission{ RuntimeTaskJobConfig: spec.RuntimeTaskJobConfig{Workers: 1}, } err = json.Unmarshal(bodyBytes, &submission) if err != nil { respondError(w, r, errors.Append(err, fmt.Sprintf("\n\ntask job submission schema can be found at https://docs.cortexlabs.com/v/%s/", consts.CortexVersionMinor)), ) return } jobSpec, err := taskapi.SubmitJob(apiName, &submission) if err != nil { respondError(w, r, err) return } respondJSON(w, r, jobSpec) } ================================================ FILE: pkg/operator/endpoints/verify_cortex.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package endpoints import ( "net/http" "github.com/cortexlabs/cortex/pkg/operator/schema" ) func VerifyCortex(w http.ResponseWriter, r *http.Request) { respondJSON(w, r, schema.VerifyCortexResponse{}) } ================================================ FILE: pkg/operator/lib/exit/exit.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package exit import ( "os" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/logging" "github.com/cortexlabs/cortex/pkg/lib/telemetry" ) var operatorLogger = logging.GetLogger() func ErrorNoTelemetry(err error, wrapStrs ...string) { for _, str := range wrapStrs { err = errors.Wrap(err, str) } if err != nil && !errors.IsNoPrint(err) { operatorLogger.Error(err) } telemetry.Close() os.Exit(1) } func Error(err error, wrapStrs ...string) { for _, str := range wrapStrs { err = errors.Wrap(err, str) } if err != nil && !errors.IsNoTelemetry(err) { telemetry.Error(err) } if err != nil && !errors.IsNoPrint(err) { operatorLogger.Error(err) } telemetry.Close() os.Exit(1) } ================================================ FILE: pkg/operator/lib/routines/routines.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package routines import ( "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/logging" "github.com/cortexlabs/cortex/pkg/lib/telemetry" ) var operatorLogger = logging.GetLogger() func RunWithPanicHandler(f func()) { go func() { defer func() { if r := recover(); r != nil { err := errors.CastRecoverError(r) operatorLogger.Error(err) telemetry.Error(err) } }() f() }() } ================================================ FILE: pkg/operator/operator/cron.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package operator import ( "context" "github.com/cortexlabs/cortex/pkg/config" "github.com/cortexlabs/cortex/pkg/consts" "github.com/cortexlabs/cortex/pkg/lib/aws" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/k8s" "github.com/cortexlabs/cortex/pkg/lib/logging" libmath "github.com/cortexlabs/cortex/pkg/lib/math" "github.com/cortexlabs/cortex/pkg/lib/sets/strset" "github.com/cortexlabs/cortex/pkg/lib/telemetry" "github.com/cortexlabs/cortex/pkg/types/clusterconfig" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" v1 "k8s.io/api/core/v1" kmeta "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" ) var operatorLogger = logging.GetLogger() var previousListOfEvictedPods = strset.New() func DeleteEvictedPods() error { failedPods, err := config.K8s.ListPods(&kmeta.ListOptions{ FieldSelector: "status.phase=Failed", }) if err != nil { return err } var errs []error currentEvictedPods := strset.New() for _, pod := range failedPods { if pod.Status.Reason != k8s.ReasonEvicted { continue } if previousListOfEvictedPods.Has(pod.Name) { _, err := config.K8s.DeletePod(pod.Name) if err != nil { errs = append(errs, err) } continue } currentEvictedPods.Add(pod.Name) } previousListOfEvictedPods = currentEvictedPods if errors.HasError(errs) { return errors.FirstError(errs...) } return nil } type instanceInfo struct { InstanceType string `json:"instance_type" yaml:"instance_type"` IsSpot bool `json:"is_spot" yaml:"is_spot"` Price float64 `json:"price" yaml:"price"` OnDemandPrice float64 `json:"on_demand_price" yaml:"on_demand_price"` Count int32 `json:"count" yaml:"count"` Memory int64 `json:"memory" yaml:"memory"` CPU float64 `json:"cpu" yaml:"cpu"` GPU int64 `json:"gpu" yaml:"gpu"` Inf int64 `json:"inf" yaml:"inf"` } func ClusterTelemetry() error { properties, err := clusterTelemetryProperties() if err != nil { return err } telemetry.Event("operator.cron", properties, config.ClusterConfig.CoreConfig.TelemetryEvent(), ) return nil } func clusterTelemetryProperties() (map[string]interface{}, error) { ctx := context.Background() var nodeList v1.NodeList err := config.K8s.List(ctx, &nodeList) if err != nil { return nil, err } nodes := nodeList.Items instanceInfos := make(map[string]*instanceInfo) var numOperatorInstances int var totalInstances int var totalInstancePrice float64 var totalInstancePriceIfOnDemand float64 spotPriceCache := make(map[string]float64) // instance type -> spot price for _, node := range nodes { if node.Labels["workload"] != "true" { if node.Labels["alpha.eksctl.io/nodegroup-name"] == "cx-operator" { numOperatorInstances++ } continue } instanceType := node.Labels["node.kubernetes.io/instance-type"] if instanceType == "" { instanceType = "unknown" } isSpot := false if node.Labels["node-lifecycle"] == "spot" { isSpot = true } totalInstances++ instanceInfosKey := instanceType + "_ondemand" if isSpot { instanceInfosKey = instanceType + "_spot" } if info, ok := instanceInfos[instanceInfosKey]; ok { info.Count++ continue } onDemandPrice := aws.InstanceMetadatas[config.ClusterConfig.Region][instanceType].Price price := onDemandPrice if isSpot { if spotPrice, ok := spotPriceCache[instanceType]; ok { price = spotPrice } else { spotPrice, err := config.AWS.SpotInstancePrice(instanceType) if err == nil && spotPrice != 0 { price = spotPrice spotPriceCache[instanceType] = spotPrice } else { spotPriceCache[instanceType] = price // the request failed, so no need to try again in the future } } } ngName := node.Labels["alpha.eksctl.io/nodegroup-name"] ebsPricePerVolume := getEBSPriceForNodeGroupInstance(config.ClusterConfig.NodeGroups, ngName) onDemandPrice += ebsPricePerVolume price += ebsPricePerVolume gpuQty := node.Status.Capacity["nvidia.com/gpu"] infQty := node.Status.Capacity["aws.amazon.com/neuron"] info := instanceInfo{ InstanceType: instanceType, IsSpot: isSpot, Price: price, OnDemandPrice: onDemandPrice, Count: 1, Memory: node.Status.Capacity.Memory().Value(), CPU: float64(node.Status.Capacity.Cpu().MilliValue()) / 1000, GPU: gpuQty.Value(), Inf: infQty.Value(), } instanceInfos[instanceInfosKey] = &info totalInstancePrice += info.Price totalInstancePriceIfOnDemand += info.OnDemandPrice } fixedPrice := cortexSystemPrice(numOperatorInstances, 1) return map[string]interface{}{ "region": config.ClusterConfig.Region, "instance_count": totalInstances, "instances": instanceInfos, "fixed_price": fixedPrice, "workload_price": totalInstancePrice, "workload_price_if_on_demand": totalInstancePriceIfOnDemand, "total_price": totalInstancePrice + fixedPrice, "total_price_if_on_demand": totalInstancePriceIfOnDemand + fixedPrice, }, nil } func getEBSPriceForNodeGroupInstance(ngs []*clusterconfig.NodeGroup, ngName string) float64 { var ebsPrice float64 for _, ng := range ngs { var ngNamePrefix string if ng.Spot { ngNamePrefix = "cx-ws-" } else { ngNamePrefix = "cx-wd-" } if ng.Name == ngNamePrefix+ngName { ebsPrice = aws.EBSMetadatas[config.ClusterConfig.Region][ng.InstanceVolumeType.String()].PriceGB * float64(ng.InstanceVolumeSize) / 30 / 24 if ng.InstanceVolumeType == clusterconfig.IO1VolumeType && ng.InstanceVolumeIOPS != nil { ebsPrice += aws.EBSMetadatas[config.ClusterConfig.Region][ng.InstanceVolumeType.String()].PriceIOPS * float64(*ng.InstanceVolumeIOPS) / 30 / 24 } if ng.InstanceVolumeType == clusterconfig.GP3VolumeType && ng.InstanceVolumeIOPS != nil && ng.InstanceVolumeThroughput != nil { ebsPrice += libmath.MaxFloat64(0, (aws.EBSMetadatas[config.ClusterConfig.Region][ng.InstanceVolumeType.String()].PriceIOPS-3000)*float64(*ng.InstanceVolumeIOPS)/30/24) ebsPrice += libmath.MaxFloat64(0, (aws.EBSMetadatas[config.ClusterConfig.Region][ng.InstanceVolumeType.String()].PriceThroughput-125)*float64(*ng.InstanceVolumeThroughput)/30/24) } break } } if ebsPrice == 0 && (ngName == "cx-operator" || ngName == "cx-prometheus") { return aws.EBSMetadatas[config.ClusterConfig.Region]["gp3"].PriceGB * 20 / 30 / 24 } return ebsPrice } func cortexSystemPrice(numOperatorInstances, numPrometheusInstances int) float64 { eksPrice := aws.EKSPrices[config.ClusterConfig.Region] metricsEBSPrice := aws.EBSMetadatas[config.ClusterConfig.Region]["gp2"].PriceGB * (40 + 2) / 30 / 24 nlbPrice := aws.NLBMetadatas[config.ClusterConfig.Region].Price natUnitPrice := aws.NATMetadatas[config.ClusterConfig.Region].Price var natTotalPrice float64 if config.ClusterConfig.NATGateway == clusterconfig.SingleNATGateway { natTotalPrice = natUnitPrice } else if config.ClusterConfig.NATGateway == clusterconfig.HighlyAvailableNATGateway { natTotalPrice = natUnitPrice * float64(len(config.ClusterConfig.AvailabilityZones)) } operatorInstancePrice := aws.InstanceMetadatas[config.ClusterConfig.Region]["t3.medium"].Price operatorEBSPrice := aws.EBSMetadatas[config.ClusterConfig.Region]["gp3"].PriceGB * 20 / 30 / 24 prometheusInstancePrice := aws.InstanceMetadatas[config.ClusterConfig.Region][config.ClusterConfig.PrometheusInstanceType].Price prometheusEBSPrice := aws.EBSMetadatas[config.ClusterConfig.Region]["gp3"].PriceGB * 20 / 30 / 24 fixedCosts := eksPrice + metricsEBSPrice + 2*nlbPrice + natTotalPrice + float64(numOperatorInstances)*(operatorInstancePrice+operatorEBSPrice) + float64(numPrometheusInstances)*(prometheusInstancePrice+prometheusEBSPrice) return fixedCosts } var clusterGauge = promauto.NewGaugeVec( prometheus.GaugeOpts{ Name: "cortex_cluster_cost", Help: "The cost breakdown of the cortex cluster", }, []string{"api", "kind", "component"}, ) func CostBreakdown() error { ctx := context.Background() var nodeList v1.NodeList err := config.K8s.List(ctx, &nodeList) if err != nil { return err } nodes := nodeList.Items var podList v1.PodList err = config.K8s.List(ctx, &podList, client.InNamespace(consts.DefaultNamespace), client.HasLabels{"apiName", "apiKind"}, ) if err != nil { return err } pods := podList.Items spotPriceCache := make(map[string]float64) // instance type -> spot price // Total cluster costs = cortex system + cortex workloads var totalClusterCosts float64 = cortexSystemPrice(0, 0) // Total cortex system costs = operator + prometheus node groups + workload daemonsets var totalCortexSystemCosts float64 = cortexSystemPrice(0, 0) // Total workload compute costs by api name totalWorkloadComputeCostsByAPIName := make(map[string]float64, 0) // Total workload compute costs by kind totalWorkloadComputeCostsByAPIKind := make(map[string]float64, 0) for _, node := range nodes { instanceType := node.Labels["node.kubernetes.io/instance-type"] if instanceType == "" { continue } workloadNode := false if node.Labels["workload"] == "true" { workloadNode = true } isSpot := false if node.Labels["node-lifecycle"] == "spot" { isSpot = true } var instanceComputePrice float64 if isSpot { if spotPrice, ok := spotPriceCache[instanceType]; ok { instanceComputePrice = spotPrice } else { spotPrice, err := config.AWS.SpotInstancePrice(instanceType) if err == nil && spotPrice != 0 { instanceComputePrice = spotPrice spotPriceCache[instanceType] = spotPrice } else { instanceComputePrice = aws.InstanceMetadatas[config.ClusterConfig.Region][instanceType].Price // the request failed, so no need to try again in the future spotPriceCache[instanceType] = instanceComputePrice } } } else { instanceComputePrice = aws.InstanceMetadatas[config.ClusterConfig.Region][instanceType].Price } ngName := node.Labels["alpha.eksctl.io/nodegroup-name"] instanceEBSPrice := getEBSPriceForNodeGroupInstance(config.ClusterConfig.NodeGroups, ngName) instancePrice := instanceComputePrice + instanceEBSPrice totalClusterCosts += instancePrice if !workloadNode { totalCortexSystemCosts += instancePrice continue } for _, pod := range pods { if pod.Spec.NodeName != node.Name { continue } maxPods := k8s.HowManyPodsFitOnNode(pod.Spec, node, consts.CortexCPUPodReserved, consts.CortexMemPodReserved) costPerPod := instancePrice / float64(maxPods) if apiName, ok := pod.Labels["apiName"]; ok { if _, okMap := totalWorkloadComputeCostsByAPIName[apiName]; okMap { totalWorkloadComputeCostsByAPIName[apiName] += costPerPod } else { totalWorkloadComputeCostsByAPIName[apiName] = costPerPod } } if apiKind, ok := pod.Labels["apiKind"]; ok { if _, okMap := totalWorkloadComputeCostsByAPIName[apiKind]; okMap { totalWorkloadComputeCostsByAPIKind[apiKind] += costPerPod } else { totalWorkloadComputeCostsByAPIKind[apiKind] = costPerPod } } } } totalWorkloadComputeCosts := totalClusterCosts - totalCortexSystemCosts var totalWorkloadComputeCostsFromAPIName float64 var totalWorkloadComputeCostsFromAPIKind float64 apiNameRenormalizationRatio := float64(1) apiKindRenormalizationRatio := float64(1) for apiName := range totalWorkloadComputeCostsByAPIName { totalWorkloadComputeCostsFromAPIName += totalWorkloadComputeCostsByAPIName[apiName] } if totalWorkloadComputeCostsFromAPIName > 0 { apiNameRenormalizationRatio = totalWorkloadComputeCosts / totalWorkloadComputeCostsFromAPIName } for apiKind := range totalWorkloadComputeCostsByAPIKind { totalWorkloadComputeCostsFromAPIKind += totalWorkloadComputeCostsByAPIKind[apiKind] } if totalWorkloadComputeCostsFromAPIKind > 0 { apiKindRenormalizationRatio = totalWorkloadComputeCosts / totalWorkloadComputeCostsFromAPIKind } clusterGauge.Reset() clusterGauge.WithLabelValues("false", "false", "cluster-costs").Set(totalClusterCosts) clusterGauge.WithLabelValues("false", "false", "cortex-system-costs").Set(totalCortexSystemCosts) for apiName := range totalWorkloadComputeCostsByAPIName { clusterGauge.WithLabelValues("true", "false", apiName).Set(totalWorkloadComputeCostsByAPIName[apiName] * apiNameRenormalizationRatio) } for apiKind := range totalWorkloadComputeCostsByAPIKind { clusterGauge.WithLabelValues("false", "true", apiKind).Set(totalWorkloadComputeCostsByAPIKind[apiKind] * apiKindRenormalizationRatio) } return nil } func ErrorHandler(cronName string) func(error) { return func(err error) { err = errors.Wrap(err, cronName+" cron failed") telemetry.Error(err) operatorLogger.Error(err) } } ================================================ FILE: pkg/operator/operator/deployed_resource.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package operator import ( "github.com/cortexlabs/cortex/pkg/types/userconfig" istioclientnetworking "istio.io/client-go/pkg/apis/networking/v1beta1" ) type DeployedResource struct { userconfig.Resource VirtualService *istioclientnetworking.VirtualService } func (deployedResourced *DeployedResource) ID() string { return deployedResourced.VirtualService.Labels["apiID"] } ================================================ FILE: pkg/operator/operator/errors.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package operator import ( "fmt" "github.com/cortexlabs/cortex/pkg/lib/errors" s "github.com/cortexlabs/cortex/pkg/lib/strings" ) const ( ErrCortexInstallationBroken = "operator.cortex_installation_broken" ErrLoadBalancerInitializing = "operator.load_balancer_initializing" ErrInvalidOperatorLogLevel = "operator.invalid_operator_log_level" ) func ErrorCortexInstallationBroken() error { return errors.WithStack(&errors.Error{ Kind: ErrCortexInstallationBroken, Message: "cortex is out of date or not installed properly; spin down your cluster with `cortex cluster down` and create a new one with `cortex cluster up`", }) } func ErrorLoadBalancerInitializing() error { return errors.WithStack(&errors.Error{ Kind: ErrLoadBalancerInitializing, Message: "load balancer is still initializing", }) } func ErrorInvalidOperatorLogLevel(provided string, loglevels []string) error { return errors.WithStack(&errors.Error{ Kind: ErrLoadBalancerInitializing, Message: fmt.Sprintf("invalid operator log level %s; must be one of %s", provided, s.StrsOr(loglevels)), }) } ================================================ FILE: pkg/operator/operator/k8s.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package operator import ( "strings" "github.com/cortexlabs/cortex/pkg/config" "github.com/cortexlabs/cortex/pkg/lib/urls" "github.com/cortexlabs/cortex/pkg/types/spec" "github.com/cortexlabs/cortex/pkg/types/userconfig" ) // APILoadBalancerURL returns the http endpoint of the ingress load balancer for deployed APIs func APILoadBalancerURL() (string, error) { return getLoadBalancerURL("ingressgateway-apis") } // LoadBalancerURL returns the http endpoint of the ingress load balancer for the operator func LoadBalancerURL() (string, error) { return getLoadBalancerURL("ingressgateway-operator") } func getLoadBalancerURL(name string) (string, error) { service, err := config.K8sIstio.GetService(name) if err != nil { return "", err } if service == nil { return "", ErrorCortexInstallationBroken() } if len(service.Status.LoadBalancer.Ingress) == 0 { return "", ErrorLoadBalancerInitializing() } if service.Status.LoadBalancer.Ingress[0].Hostname != "" { return "http://" + service.Status.LoadBalancer.Ingress[0].Hostname, nil } return "http://" + service.Status.LoadBalancer.Ingress[0].IP, nil } func APIEndpoint(api *spec.API) (string, error) { var err error baseAPIEndpoint := "" baseAPIEndpoint, err = APILoadBalancerURL() if err != nil { return "", err } baseAPIEndpoint = strings.Replace(baseAPIEndpoint, "https://", "http://", 1) return urls.Join(baseAPIEndpoint, *api.Networking.Endpoint), nil } func APIEndpointFromResource(deployedResource *DeployedResource) (string, error) { apiEndpoint, err := userconfig.EndpointFromAnnotation(deployedResource.VirtualService) if err != nil { return "", err } baseAPIEndpoint := "" baseAPIEndpoint, err = APILoadBalancerURL() if err != nil { return "", err } baseAPIEndpoint = strings.Replace(baseAPIEndpoint, "https://", "http://", 1) return urls.Join(baseAPIEndpoint, apiEndpoint), nil } ================================================ FILE: pkg/operator/operator/logging.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package operator import ( "fmt" "os" "strings" "sync" "time" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/logging" "github.com/cortexlabs/cortex/pkg/types/spec" "github.com/cortexlabs/cortex/pkg/types/userconfig" "go.uber.org/zap" ) const ( _loggerTTL = time.Hour * 1 _evictionCronPeriod = time.Minute * 10 ) type cachedLogger struct { value *zap.SugaredLogger lastAccess time.Time } type loggerCache struct { m map[string]*cachedLogger sync.Mutex } var _loggerCache loggerCache func init() { _loggerCache = loggerCache{m: map[string]*cachedLogger{}} go func() { for range time.Tick(_evictionCronPeriod) { _loggerCache.Lock() for k, v := range _loggerCache.m { if time.Since(v.lastAccess) > _loggerTTL { delete(_loggerCache.m, k) } } _loggerCache.Unlock() } }() } func getFromCacheOrNil(key string) *zap.SugaredLogger { _loggerCache.Lock() defer _loggerCache.Unlock() item, ok := _loggerCache.m[key] if ok { item.lastAccess = time.Now() return item.value } return nil } func initializeLogger(key string, level userconfig.LogLevel, fields map[string]interface{}) (*zap.SugaredLogger, error) { loggerConfig := logging.DefaultZapConfig(level, fields) disableJSONLogging := strings.ToLower(os.Getenv("CORTEX_DISABLE_JSON_LOGGING")) if disableJSONLogging == "true" { loggerConfig.Encoding = "console" } logger, err := loggerConfig.Build() if err != nil { return nil, errors.WithStack(err) } sugarLogger := logger.Sugar() _loggerCache.Lock() defer _loggerCache.Unlock() _loggerCache.m[key] = &cachedLogger{ lastAccess: time.Now(), value: sugarLogger, } return sugarLogger, nil } func GetRealtimeAPILogger(apiName string, apiID string) (*zap.SugaredLogger, error) { loggerCacheKey := fmt.Sprintf("apiName=%s,apiID=%s", apiName, apiID) logger := getFromCacheOrNil(loggerCacheKey) if logger != nil { return logger, nil } apiSpec, err := DownloadAPISpec(apiName, apiID) if err != nil { return nil, err } return initializeLogger(loggerCacheKey, userconfig.InfoLogLevel, map[string]interface{}{ "apiName": apiSpec.Name, "apiKind": apiSpec.Kind.String(), "apiID": apiSpec.ID, }) } func GetRealtimeAPILoggerFromSpec(apiSpec *spec.API) (*zap.SugaredLogger, error) { loggerCacheKey := fmt.Sprintf("apiName=%s,apiID=%s", apiSpec.Name, apiSpec.ID) logger := getFromCacheOrNil(loggerCacheKey) if logger != nil { return logger, nil } return initializeLogger(loggerCacheKey, userconfig.InfoLogLevel, map[string]interface{}{ "apiName": apiSpec.Name, "apiKind": apiSpec.Kind.String(), "apiID": apiSpec.ID, }) } func GetJobLogger(jobKey spec.JobKey) (*zap.SugaredLogger, error) { loggerCacheKey := fmt.Sprintf("apiName=%s,jobID=%s", jobKey.APIName, jobKey.ID) logger := getFromCacheOrNil(loggerCacheKey) if logger != nil { return logger, nil } return initializeLogger(loggerCacheKey, userconfig.InfoLogLevel, map[string]interface{}{ "apiName": jobKey.APIName, "apiKind": jobKey.Kind.String(), "jobID": jobKey.ID, }) } func GetJobLoggerFromSpec(apiSpec *spec.API, jobKey spec.JobKey) (*zap.SugaredLogger, error) { loggerCacheKey := fmt.Sprintf("apiName=%s,jobID=%s", jobKey.APIName, jobKey.ID) logger := getFromCacheOrNil(loggerCacheKey) if logger != nil { return logger, nil } return initializeLogger(loggerCacheKey, userconfig.InfoLogLevel, map[string]interface{}{ "apiName": jobKey.APIName, "apiKind": jobKey.Kind.String(), "jobID": jobKey.ID, }) } ================================================ FILE: pkg/operator/operator/memory_capacity.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package operator import ( "github.com/cortexlabs/cortex/pkg/config" "github.com/cortexlabs/cortex/pkg/lib/aws" "github.com/cortexlabs/cortex/pkg/lib/k8s" "github.com/cortexlabs/cortex/pkg/lib/slices" kresource "k8s.io/apimachinery/pkg/api/resource" kmeta "k8s.io/apimachinery/pkg/apis/meta/v1" klabels "k8s.io/apimachinery/pkg/labels" ) const _memConfigMapName = "cortex-instance-memory" const _configKeyPrefix = "memory-capacity-" func getMemoryCapacityFromNodes(primaryInstances []string) (map[string]*kresource.Quantity, error) { opts := kmeta.ListOptions{ LabelSelector: klabels.SelectorFromSet(map[string]string{ "workload": "true", }).String(), } nodes, err := config.K8s.ListNodes(&opts) if err != nil { return nil, err } minMemMap := map[string]*kresource.Quantity{} for _, primaryInstance := range primaryInstances { minMemMap[primaryInstance] = nil } for _, node := range nodes { isPrimaryInstance := false var primaryInstanceType string for k, v := range node.Labels { if k == "node.kubernetes.io/instance-type" && slices.HasString(primaryInstances, v) { isPrimaryInstance = true primaryInstanceType = v break } } if !isPrimaryInstance { continue } curMem := node.Status.Capacity.Memory() if curMem == nil || curMem.IsZero() { continue } if minMemMap[primaryInstanceType] == nil || minMemMap[primaryInstanceType].Cmp(*curMem) > 0 { minMemMap[primaryInstanceType] = curMem } } return minMemMap, nil } func getMemoryCapacityFromConfigMap() (map[string]*kresource.Quantity, error) { configMapData, _, err := config.K8s.GetConfigMapData(_memConfigMapName) if err != nil { return nil, err } if len(configMapData) == 0 { return nil, nil } memoryCapacitiesMap := map[string]*kresource.Quantity{} for k := range configMapData { memoryUserStr := configMapData[k] mem, err := kresource.ParseQuantity(memoryUserStr) if err != nil { return nil, err } instanceType := k[len(_configKeyPrefix):] if mem.IsZero() { memoryCapacitiesMap[instanceType] = nil } else { memoryCapacitiesMap[instanceType] = &mem } } return memoryCapacitiesMap, nil } func UpdateMemoryCapacityConfigMap() (map[string]kresource.Quantity, error) { primaryInstances := []string{} minMemMap := map[string]kresource.Quantity{} for _, ng := range config.ClusterConfig.NodeGroups { instanceMetadata := aws.InstanceMetadatas[config.ClusterConfig.Region][ng.InstanceType] minMemMap[ng.InstanceType] = instanceMetadata.Memory primaryInstances = append(primaryInstances, ng.InstanceType) } nodeMemCapacityMap, err := getMemoryCapacityFromNodes(primaryInstances) if err != nil { return nil, err } previousMinMemMap, err := getMemoryCapacityFromConfigMap() if err != nil { return nil, err } configMapData := map[string]string{} for _, primaryInstance := range primaryInstances { minMem := minMemMap[primaryInstance] if nodeMemCapacityMap[primaryInstance] != nil && minMem.Cmp(*nodeMemCapacityMap[primaryInstance]) > 0 { minMem = *nodeMemCapacityMap[primaryInstance] } if previousMinMemMap[primaryInstance] != nil && minMem.Cmp(*previousMinMemMap[primaryInstance]) > 0 { minMem = *previousMinMemMap[primaryInstance] } if previousMinMemMap[primaryInstance] == nil || minMem.Cmp(*previousMinMemMap[primaryInstance]) < 0 { configMapData[_configKeyPrefix+primaryInstance] = minMem.String() } else { configMapData[_configKeyPrefix+primaryInstance] = previousMinMemMap[primaryInstance].String() } minMemMap[primaryInstance] = minMem } configMap := k8s.ConfigMap(&k8s.ConfigMapSpec{ Name: _memConfigMapName, Data: configMapData, }) _, err = config.K8s.ApplyConfigMap(configMap) if err != nil { return nil, err } return minMemMap, nil } ================================================ FILE: pkg/operator/operator/storage.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package operator import ( "github.com/cortexlabs/cortex/pkg/config" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/parallel" "github.com/cortexlabs/cortex/pkg/types/spec" ) func DownloadAPISpec(apiName string, apiID string) (*spec.API, error) { bucketKey := spec.Key(apiName, apiID, config.ClusterConfig.ClusterUID) var api spec.API if err := config.AWS.ReadJSONFromS3(&api, config.ClusterConfig.Bucket, bucketKey); err != nil { return nil, err } return &api, nil } func DownloadAPISpecs(apiNames []string, apiIDs []string) ([]spec.API, error) { apis := make([]spec.API, len(apiNames)) fns := make([]func() error, len(apiNames)) for i := range apiNames { localIdx := i fns[i] = func() error { api, err := DownloadAPISpec(apiNames[localIdx], apiIDs[localIdx]) if err != nil { return err } apis[localIdx] = *api return nil } } if len(fns) > 0 { err := parallel.RunFirstErr(fns[0], fns[1:]...) if err != nil { return nil, err } } return apis, nil } func DownloadBatchJobSpec(jobKey spec.JobKey) (*spec.BatchJob, error) { jobSpec := spec.BatchJob{} if err := config.AWS.ReadJSONFromS3(&jobSpec, config.ClusterConfig.Bucket, jobKey.SpecFilePath(config.ClusterConfig.ClusterUID)); err != nil { return nil, errors.Wrap(err, "unable to download job specification", jobKey.UserString()) } return &jobSpec, nil } func DownloadTaskJobSpec(jobKey spec.JobKey) (*spec.TaskJob, error) { jobSpec := spec.TaskJob{} if err := config.AWS.ReadJSONFromS3(&jobSpec, config.ClusterConfig.Bucket, jobKey.SpecFilePath(config.ClusterConfig.ClusterUID)); err != nil { return nil, errors.Wrap(err, "unable to download job specification", jobKey.UserString()) } return &jobSpec, nil } ================================================ FILE: pkg/operator/operator/workload_logging.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package operator import ( "bufio" "bytes" "encoding/json" "fmt" "io" "os/exec" "strings" "text/template" "time" "github.com/cortexlabs/cortex/pkg/config" awslib "github.com/cortexlabs/cortex/pkg/lib/aws" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/k8s" "github.com/cortexlabs/cortex/pkg/lib/telemetry" "github.com/cortexlabs/cortex/pkg/operator/lib/routines" "github.com/cortexlabs/cortex/pkg/types/spec" "github.com/cortexlabs/cortex/pkg/types/status" "github.com/gorilla/websocket" ) const ( _socketWriteDeadlineWait = 10 * time.Second _socketCloseGracePeriod = 10 * time.Second _socketMaxMessageSize = 8192 _readBufferSize = 4096 _pendingPodCheckInterval = 1 * time.Second _pollPeriod = 250 * time.Millisecond ) func timeString(t time.Time) string { return fmt.Sprintf("%sT%02d*3a%02d*3a%02d", t.Format("2006-01-02"), t.Hour(), t.Minute(), t.Second()) } var _apiLogURLTemplate *template.Template = template.Must(template.New("api_log_url_template").Parse(strings.TrimSpace(` https://console.{{.Partition}}.com/cloudwatch/home?region={{.Region}}#logsV2:logs-insights$3FqueryDetail$3D$257E$2528end$257E0$257Estart$257E-3600$257EtimeType$257E$2527RELATIVE$257Eunit$257E$2527seconds$257EeditorString$257E$2527fields*20*40timestamp*2c*20message*0a*7c*20filter*20cortex.labels.apiName*3d*22{{.APIName}}*22*0a*7c*20sort*20*40timestamp*20asc*0a$257Esource$257E$2528$257E$2527{{.LogGroup}}$2529$2529 `))) var _completedJobLogURLTemplate *template.Template = template.Must(template.New("completed_job_log_url_template").Parse(strings.TrimSpace(` https://console.{{.Partition}}.com/cloudwatch/home?region={{.Region}}#logsV2:logs-insights$3FqueryDetail$3D$257E$2528end$257E$2527{{.EndTime}}$257Estart$257E$2527{{.StartTime}}$257EtimeType$257E$2527ABSOLUTE$257Etz$257E$2527Local$257EeditorString$257E$2527fields*20*40timestamp*2c*20message*0a*7c*20filter*20cortex.labels.apiName*3d*22{{.APIName}}*22*20and*20cortex.labels.jobID*3d*22{{.JobID}}*22*0a*7c*20sort*20*40timestamp*20asc*0a$257Esource$257E$2528$257E$2527{{.LogGroup}}$2529$2529 `))) var _inProgressJobLogsURLTemplate *template.Template = template.Must(template.New("in_progress_job_log_url_template").Parse(strings.TrimSpace(` https://console.{{.Partition}}.com/cloudwatch/home?region={{.Region}}#logsV2:logs-insights$3FqueryDetail$3D$257E$2528end$257E0$257Estart$257E-3600$257EtimeType$257E$2527RELATIVE$257Eunit$257E$2527seconds$257EeditorString$257E$2527fields*20*40timestamp*2c*20message*0a*7c*20filter*20cortex.labels.apiName*3d*22{{.APIName}}*22*20and*20cortex.labels.jobID*3d*22{{.JobID}}*22*0a*7c*20sort*20*40timestamp*20asc*0a$257Esource$257E$2528$257E$2527{{.LogGroup}}$2529$2529 `))) type apiLogURLTemplateArgs struct { Partition string Region string LogGroup string APIName string } type completedJobLogURLTemplateArgs struct { Partition string Region string StartTime string EndTime string LogGroup string APIName string JobID string } type inProgressJobLogURLTemplateArgs struct { Partition string Region string LogGroup string APIName string JobID string } func completedBatchJobLogsURL(args completedJobLogURLTemplateArgs) (string, error) { buf := &bytes.Buffer{} err := _completedJobLogURLTemplate.Execute(buf, args) if err != nil { return "", err } return strings.TrimSpace(buf.String()), nil } func inProgressBatchJobLogsURL(args inProgressJobLogURLTemplateArgs) (string, error) { buf := &bytes.Buffer{} err := _inProgressJobLogsURLTemplate.Execute(buf, args) if err != nil { return "", err } return strings.TrimSpace(buf.String()), nil } func APILogURL(api spec.API) (string, error) { partition := "aws.amazon" region := config.ClusterConfig.Region if awslib.PartitionFromRegion(region) == "aws-us-gov" { partition = "amazonaws-us-gov" } logGroup := config.ClusterConfig.ClusterName args := apiLogURLTemplateArgs{ Partition: partition, Region: region, LogGroup: logGroup, APIName: api.Name, } buf := &bytes.Buffer{} err := _apiLogURLTemplate.Execute(buf, args) if err != nil { return "", err } return strings.TrimSpace(buf.String()), nil } func BatchJobLogURL(apiName string, jobStatus status.BatchJobStatus) (string, error) { partition := "aws.amazon" region := config.ClusterConfig.Region if awslib.PartitionFromRegion(region) == "aws-us-gov" { partition = "amazonaws-us-gov" } logGroup := config.ClusterConfig.ClusterName if jobStatus.EndTime != nil { endTime := *jobStatus.EndTime endTime = endTime.Add(60 * time.Second) return completedBatchJobLogsURL(completedJobLogURLTemplateArgs{ Partition: partition, Region: region, StartTime: timeString(jobStatus.StartTime), EndTime: timeString(endTime), LogGroup: logGroup, APIName: apiName, JobID: jobStatus.ID, }) } return inProgressBatchJobLogsURL(inProgressJobLogURLTemplateArgs{ Partition: partition, Region: region, LogGroup: logGroup, APIName: apiName, JobID: jobStatus.ID, }) } func TaskJobLogURL(apiName string, jobStatus status.TaskJobStatus) (string, error) { partition := "aws.amazon" region := config.ClusterConfig.Region if awslib.PartitionFromRegion(region) == "aws-us-gov" { partition = "amazonaws-us-gov" } logGroup := config.ClusterConfig.ClusterName if jobStatus.EndTime != nil { endTime := *jobStatus.EndTime endTime = endTime.Add(60 * time.Second) return completedBatchJobLogsURL(completedJobLogURLTemplateArgs{ Partition: partition, Region: region, StartTime: timeString(jobStatus.StartTime), EndTime: timeString(endTime), LogGroup: logGroup, APIName: apiName, JobID: jobStatus.ID, }) } return inProgressBatchJobLogsURL(inProgressJobLogURLTemplateArgs{ Partition: partition, Region: region, LogGroup: logGroup, APIName: apiName, JobID: jobStatus.ID, }) } func waitForPodToBeNotPending(podName string, cancelListener chan struct{}, socket *websocket.Conn) bool { wrotePending := false timer := time.NewTimer(0) for true { select { case <-cancelListener: return false case <-timer.C: pod, err := config.K8s.GetPod(podName) if err != nil { writeAndCloseSocket(socket, fmt.Sprintf("error encountered while attempting to stream logs from pod %s\n%s", podName, err.Error())) return false } if pod == nil { writeAndCloseSocket(socket, "unable to find pod") return false } podStatus := k8s.GetPodStatus(pod) if podStatus == k8s.PodStatusPending { if !wrotePending { writeString(socket, "waiting for pod to initialize ...\n") } wrotePending = true timer.Reset(_pendingPodCheckInterval) continue } return true } } return false } type jsonMessage struct { Message string `json:"message"` ExcInfo string `json:"exc_info"` } func startKubectlProcess(podName string, cancelListener chan struct{}, socket *websocket.Conn) { shouldContinue := waitForPodToBeNotPending(podName, cancelListener, socket) if !shouldContinue { return } cmd := exec.Command("/usr/local/bin/kubectl", "-n="+config.K8s.Namespace, "logs", "--all-containers", podName, "--follow") cleanup := func() { // trigger a wait on the child process and while the process is being waited on, // send the kill signal to allow cleanup to happen correctly and prevent zombie processes time.AfterFunc(1*time.Second, func() { cmd.Process.Kill() }) cmd.Process.Wait() } defer cleanup() logStream, err := cmd.StdoutPipe() if err != nil { telemetry.Error(errors.ErrorUnexpected(err.Error())) operatorLogger.Error(err) } cmd.Start() routines.RunWithPanicHandler(func() { pumpStdout(socket, logStream) }) <-cancelListener } func pumpStdout(socket *websocket.Conn, reader io.Reader) { // it seems like if the buffer is maxed out with no ending token, the scanner just exits. // increase the buffer used by the scanner to accommodate larger log lines (a common issue when printing progress) p := make([]byte, 1024*1024) scanner := bufio.NewScanner(reader) scanner.Buffer(p, 1024*1024) for scanner.Scan() { logBytes := scanner.Bytes() var message jsonMessage err := json.Unmarshal(logBytes, &message) if err != nil { writeString(socket, string(logBytes)+"\n") } else { writeString(socket, message.Message+"\n") if message.ExcInfo != "" { writeString(socket, message.ExcInfo+"\n") } } } closeSocket(socket) } func StreamLogsFromRandomPod(podSearchLabels map[string]string, socket *websocket.Conn) { pods, err := config.K8s.ListPodsByLabels(podSearchLabels) if err != nil { writeAndCloseSocket(socket, err.Error()) return } if len(pods) == 0 { writeAndCloseSocket(socket, "there are currently no pods running for this workload; please visit your logging dashboard for historical logs\n") return } cancelListener := make(chan struct{}) defer close(cancelListener) routines.RunWithPanicHandler(func() { startKubectlProcess(pods[0].Name, cancelListener, socket) }) pumpStdin(socket) cancelListener <- struct{}{} } func pumpStdin(socket *websocket.Conn) { socket.SetReadLimit(_socketMaxMessageSize) for { _, _, err := socket.ReadMessage() if err != nil { break } } } func writeString(socket *websocket.Conn, message string) { socket.WriteMessage(websocket.TextMessage, []byte(message)) } func writeAndCloseSocket(socket *websocket.Conn, message string) { writeString(socket, message) closeSocket(socket) } func closeSocket(socket *websocket.Conn) { socket.SetWriteDeadline(time.Now().Add(_socketWriteDeadlineWait)) socket.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, "")) time.Sleep(_socketCloseGracePeriod) } ================================================ FILE: pkg/operator/resources/asyncapi/api.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package asyncapi import ( "fmt" "path/filepath" "sort" "time" "github.com/cortexlabs/cortex/pkg/config" "github.com/cortexlabs/cortex/pkg/lib/cron" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/k8s" "github.com/cortexlabs/cortex/pkg/lib/parallel" "github.com/cortexlabs/cortex/pkg/lib/pointer" "github.com/cortexlabs/cortex/pkg/operator/lib/routines" "github.com/cortexlabs/cortex/pkg/operator/operator" "github.com/cortexlabs/cortex/pkg/operator/schema" "github.com/cortexlabs/cortex/pkg/types/spec" "github.com/cortexlabs/cortex/pkg/types/status" "github.com/cortexlabs/cortex/pkg/types/userconfig" "github.com/cortexlabs/cortex/pkg/workloads" istioclientnetworking "istio.io/client-go/pkg/apis/networking/v1beta1" kapps "k8s.io/api/apps/v1" kcore "k8s.io/api/core/v1" ) const ( _tickPeriodMetrics = 10 * time.Second _asyncDashboardUID = "asyncapi" ) var ( _metricsCrons = make(map[string]cron.Cron) ) type resources struct { apiDeployment *kapps.Deployment apiConfigMap *kcore.ConfigMap apiVirtualService *istioclientnetworking.VirtualService } func generateDeploymentID() string { return k8s.RandomName()[:10] } func UpdateAPI(apiConfig userconfig.API, force bool) (*spec.API, string, error) { prevK8sResources, err := getK8sResources(apiConfig.Name) if err != nil { return nil, "", err } initialDeploymentTime := time.Now().UnixNano() deploymentID := generateDeploymentID() if prevK8sResources.apiVirtualService != nil && prevK8sResources.apiVirtualService.Labels["initialDeploymentTime"] != "" { var err error initialDeploymentTime, err = k8s.ParseInt64Label(prevK8sResources.apiVirtualService, "initialDeploymentTime") if err != nil { return nil, "", err } deploymentID = prevK8sResources.apiVirtualService.Labels["deploymentID"] } api := spec.GetAPISpec(&apiConfig, initialDeploymentTime, deploymentID, config.ClusterConfig.ClusterUID) // resource creation if prevK8sResources.apiVirtualService == nil { if err := config.AWS.UploadJSONToS3(api, config.ClusterConfig.Bucket, api.Key); err != nil { return nil, "", errors.Wrap(err, "upload api spec") } tags := map[string]string{ "apiName": apiConfig.Name, } queueURL, err := createFIFOQueue(apiConfig.Name, initialDeploymentTime, tags) if err != nil { return nil, "", err } if err = applyK8sResources(*api, prevK8sResources, queueURL); err != nil { routines.RunWithPanicHandler(func() { _ = parallel.RunFirstErr( func() error { return deleteQueueByURL(queueURL) }, func() error { return deleteK8sResources(api.Name) }, ) }) return nil, "", err } return api, fmt.Sprintf("creating %s", api.Resource.UserString()), nil } // resource update if prevK8sResources.apiVirtualService.Labels["specID"] != api.SpecID { isUpdating, err := isAPIUpdating(prevK8sResources.apiDeployment) if err != nil { return nil, "", err } if isUpdating && !force { return nil, "", ErrorAPIUpdating(api.Name) } if err := config.AWS.UploadJSONToS3(api, config.ClusterConfig.Bucket, api.Key); err != nil { return nil, "", errors.Wrap(err, "upload api spec") } initialDeploymentTime, err := k8s.ParseInt64Label(prevK8sResources.apiVirtualService, "initialDeploymentTime") if err != nil { return nil, "", err } queueURL, err := getQueueURL(api.Name, initialDeploymentTime) if err != nil { return nil, "", err } if err = applyK8sResources(*api, prevK8sResources, queueURL); err != nil { return nil, "", err } return api, fmt.Sprintf("updating %s", api.Resource.UserString()), nil } // nothing changed isUpdating, err := isAPIUpdating(prevK8sResources.apiDeployment) if err != nil { return nil, "", err } if isUpdating { return api, fmt.Sprintf("%s is already updating", api.Resource.UserString()), nil } return api, fmt.Sprintf("%s is up to date", api.Resource.UserString()), nil } func RefreshAPI(apiName string, force bool) (string, error) { prevK8sResources, err := getK8sResources(apiName) if err != nil { return "", err } else if prevK8sResources.apiVirtualService == nil || prevK8sResources.apiDeployment == nil { return "", errors.ErrorUnexpected("unable to find deployment", apiName) } isUpdating, err := isAPIUpdating(prevK8sResources.apiDeployment) if err != nil { return "", err } if isUpdating && !force { return "", ErrorAPIUpdating(apiName) } apiID, err := k8s.GetLabel(prevK8sResources.apiVirtualService, "apiID") if err != nil { return "", err } api, err := operator.DownloadAPISpec(apiName, apiID) if err != nil { return "", err } initialDeploymentTime, err := k8s.ParseInt64Label(prevK8sResources.apiVirtualService, "initialDeploymentTime") if err != nil { return "", err } api = spec.GetAPISpec(api.API, initialDeploymentTime, generateDeploymentID(), config.ClusterConfig.ClusterUID) if err := config.AWS.UploadJSONToS3(api, config.ClusterConfig.Bucket, api.Key); err != nil { return "", errors.Wrap(err, "upload api spec") } queueURL, err := getQueueURL(api.Name, initialDeploymentTime) if err != nil { return "", err } if err = applyK8sResources(*api, prevK8sResources, queueURL); err != nil { return "", err } return fmt.Sprintf("updating %s", api.Resource.UserString()), nil } func DeleteAPI(apiName string, keepCache bool) error { err := parallel.RunFirstErr( func() error { vs, err := config.K8s.GetVirtualService(workloads.K8sName(apiName)) if err != nil { return err } if vs != nil { initialDeploymentTime, err := k8s.ParseInt64Label(vs, "initialDeploymentTime") if err != nil { return err } queueURL, err := getQueueURL(apiName, initialDeploymentTime) if err != nil { return err } // best effort deletion _ = deleteQueueByURL(queueURL) } return nil }, func() error { return deleteK8sResources(apiName) }, func() error { if keepCache { return nil } // best effort deletion, swallow errors because there could be weird error messages _ = deleteBucketResources(apiName) return nil }, ) if err != nil { return err } return nil } func GetAllAPIs(deployments []kapps.Deployment) ([]schema.APIResponse, error) { asyncAPIs := make([]schema.APIResponse, 0) mappedAsyncAPIs := make(map[string]schema.APIResponse, 0) apiNames := make([]string, 0) for i := range deployments { apiName := deployments[i].Labels["apiName"] apiNames = append(apiNames, apiName) metadata, err := spec.MetadataFromDeployment(&deployments[i]) if err != nil { return nil, errors.Wrap(err, fmt.Sprintf("api %s", apiName)) } mappedAsyncAPIs[apiName] = schema.APIResponse{ Status: status.FromDeployment(&deployments[i]), Metadata: metadata, } } sort.Strings(apiNames) for _, apiName := range apiNames { asyncAPIs = append(asyncAPIs, mappedAsyncAPIs[apiName]) } return asyncAPIs, nil } func GetAPIByName(deployedResource *operator.DeployedResource) ([]schema.APIResponse, error) { apiDeployment, err := config.K8s.GetDeployment(workloads.K8sName(deployedResource.Name)) if err != nil { return nil, err } if apiDeployment == nil { return nil, errors.ErrorUnexpected("unable to find api deployment", deployedResource.Name) } apiStatus := status.FromDeployment(apiDeployment) apiMetadata, err := spec.MetadataFromDeployment(apiDeployment) if err != nil { return nil, errors.ErrorUnexpected("unable to obtain metadata", deployedResource.Name) } api, err := operator.DownloadAPISpec(apiMetadata.Name, apiMetadata.APIID) if err != nil { return nil, err } apiEndpoint, err := operator.APIEndpoint(api) if err != nil { return nil, err } dashboardURL := pointer.String(getDashboardURL(api.Name)) return []schema.APIResponse{ { Spec: api, Metadata: apiMetadata, Status: apiStatus, Endpoint: &apiEndpoint, DashboardURL: dashboardURL, }, }, nil } func DescribeAPIByName(deployedResource *operator.DeployedResource) ([]schema.APIResponse, error) { var apiDeployment *kapps.Deployment apiDeployment, err := config.K8s.GetDeployment(workloads.K8sName(deployedResource.Name)) if err != nil { return nil, err } if apiDeployment == nil { return nil, errors.ErrorUnexpected("unable to find api deployment", deployedResource.Name) } apiStatus := status.FromDeployment(apiDeployment) apiMetadata, err := spec.MetadataFromDeployment(apiDeployment) if err != nil { return nil, errors.ErrorUnexpected("unable to obtain metadata", deployedResource.Name) } apiPods, err := config.K8s.ListPodsByLabels(map[string]string{ "apiName": apiDeployment.Labels["apiName"], }) if err != nil { return nil, err } apiStatus.ReplicaCounts = GetReplicaCounts(apiDeployment, apiPods) apiEndpoint, err := operator.APIEndpointFromResource(deployedResource) if err != nil { return nil, err } dashboardURL := pointer.String(getDashboardURL(deployedResource.Name)) return []schema.APIResponse{ { Metadata: apiMetadata, Status: apiStatus, Endpoint: &apiEndpoint, DashboardURL: dashboardURL, }, }, nil } func UpdateAPIMetricsCron(apiDeployment *kapps.Deployment) error { apiName := apiDeployment.Labels["apiName"] if prevMetricsCron, ok := _metricsCrons[apiName]; ok { prevMetricsCron.Cancel() } initialDeploymentTime, err := k8s.ParseInt64Label(apiDeployment, "initialDeploymentTime") if err != nil { return err } queueURL, err := getQueueURL(apiName, initialDeploymentTime) if err != nil { return err } metricsCron := updateQueueLengthMetricsFn(apiName, queueURL) _metricsCrons[apiName] = cron.Run(metricsCron, operator.ErrorHandler(apiName+" metrics"), _tickPeriodMetrics) return nil } func getK8sResources(apiName string) (resources, error) { var deployment *kapps.Deployment var apiConfigMap *kcore.ConfigMap var apiVirtualService *istioclientnetworking.VirtualService apiK8sName := workloads.K8sName(apiName) err := parallel.RunFirstErr( func() error { var err error deployment, err = config.K8s.GetDeployment(apiK8sName) return err }, func() error { var err error apiConfigMap, err = config.K8s.GetConfigMap(apiK8sName) return err }, func() error { var err error apiVirtualService, err = config.K8s.GetVirtualService(apiK8sName) return err }, ) return resources{ apiDeployment: deployment, apiConfigMap: apiConfigMap, apiVirtualService: apiVirtualService, }, err } func applyK8sResources(api spec.API, prevK8sResources resources, queueURL string) error { apiDeployment := deploymentSpec(api, prevK8sResources.apiDeployment, queueURL) apiConfigMap, err := configMapSpec(api) if err != nil { return err } apiVirtualService := apiVirtualServiceSpec(api, queueURL) return parallel.RunFirstErr( func() error { if err := applyK8sConfigMap(prevK8sResources.apiConfigMap, &apiConfigMap); err != nil { return err } if err := applyK8sDeployment(prevK8sResources.apiDeployment, &apiDeployment); err != nil { return err } if err := UpdateAPIMetricsCron(&apiDeployment); err != nil { return err } return nil }, func() error { return applyK8sVirtualService(prevK8sResources.apiVirtualService, &apiVirtualService) }, ) } func applyK8sConfigMap(prevConfigMap *kcore.ConfigMap, newConfigMap *kcore.ConfigMap) error { if prevConfigMap == nil { _, err := config.K8s.CreateConfigMap(newConfigMap) if err != nil { return err } } else { _, err := config.K8s.UpdateConfigMap(newConfigMap) if err != nil { return err } } return nil } func applyK8sDeployment(prevDeployment *kapps.Deployment, newDeployment *kapps.Deployment) error { if prevDeployment == nil { _, err := config.K8s.CreateDeployment(newDeployment) if err != nil { return err } } else if prevDeployment.Status.ReadyReplicas == 0 { // Delete deployment if it never became ready _, _ = config.K8s.DeleteDeployment(prevDeployment.Name) _, err := config.K8s.CreateDeployment(newDeployment) if err != nil { return err } } else { _, err := config.K8s.UpdateDeployment(newDeployment) if err != nil { return err } } return nil } func applyK8sVirtualService(prevVirtualService *istioclientnetworking.VirtualService, newVirtualService *istioclientnetworking.VirtualService) error { if prevVirtualService == nil { _, err := config.K8s.CreateVirtualService(newVirtualService) return err } _, err := config.K8s.UpdateVirtualService(prevVirtualService, newVirtualService) return err } func deleteBucketResources(apiName string) error { prefix := filepath.Join(config.ClusterConfig.ClusterUID, "apis", apiName) return config.AWS.DeleteS3Dir(config.ClusterConfig.Bucket, prefix, true) } func deleteK8sResources(apiName string) error { apiK8sName := workloads.K8sName(apiName) err := parallel.RunFirstErr( func() error { if metricsCron, ok := _metricsCrons[apiName]; ok { metricsCron.Cancel() delete(_metricsCrons, apiName) } _, err := config.K8s.DeleteDeployment(apiK8sName) return err }, func() error { _, err := config.K8s.DeleteConfigMap(apiK8sName) return err }, func() error { _, err := config.K8s.DeleteVirtualService(apiK8sName) return err }, ) return err } // returns true if min_replicas are not ready and no updated replicas have errored func isAPIUpdating(deployment *kapps.Deployment) (bool, error) { pods, err := config.K8s.ListPodsByLabel("apiName", deployment.Labels["apiName"]) if err != nil { return false, err } replicaCounts := GetReplicaCounts(deployment, pods) autoscalingSpec, err := userconfig.AutoscalingFromAnnotations(deployment) if err != nil { return false, err } if replicaCounts.Ready < autoscalingSpec.MinReplicas && replicaCounts.TotalFailed() == 0 { return true, nil } return false, nil } func isPodSpecLatest(deployment *kapps.Deployment, pod *kcore.Pod) bool { return deployment.Spec.Template.Labels["podID"] == pod.Labels["podID"] && deployment.Spec.Template.Labels["deploymentID"] == pod.Labels["deploymentID"] } func getDashboardURL(apiName string) string { loadBalancerURL, err := operator.LoadBalancerURL() if err != nil { return "" } dashboardURL := fmt.Sprintf( "%s/dashboard/d/%s/asyncapi?orgId=1&refresh=30s&var-api_name=%s", loadBalancerURL, _asyncDashboardUID, apiName, ) return dashboardURL } ================================================ FILE: pkg/operator/resources/asyncapi/errors.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package asyncapi import ( "fmt" "github.com/cortexlabs/cortex/pkg/lib/errors" ) const ( ErrAPIUpdating = "asyncapi.api_updating" ) func ErrorAPIUpdating(apiName string) error { return errors.WithStack(&errors.Error{ Kind: ErrAPIUpdating, Message: fmt.Sprintf("%s is updating (override with --force)", apiName), }) } ================================================ FILE: pkg/operator/resources/asyncapi/k8s_specs.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package asyncapi import ( "github.com/cortexlabs/cortex/pkg/consts" "github.com/cortexlabs/cortex/pkg/lib/k8s" "github.com/cortexlabs/cortex/pkg/lib/pointer" s "github.com/cortexlabs/cortex/pkg/lib/strings" "github.com/cortexlabs/cortex/pkg/types/spec" "github.com/cortexlabs/cortex/pkg/workloads" istionetworking "istio.io/api/networking/v1beta1" "istio.io/client-go/pkg/apis/networking/v1beta1" kapps "k8s.io/api/apps/v1" kcore "k8s.io/api/core/v1" ) var _terminationGracePeriodSeconds int64 = 60 // seconds func apiVirtualServiceSpec(api spec.API, queueURL string) v1beta1.VirtualService { return *k8s.VirtualService(&k8s.VirtualServiceSpec{ Name: workloads.K8sName(api.Name), Gateways: []string{"apis-gateway"}, Destinations: []k8s.Destination{ { ServiceName: "async-gateway", Weight: 100, Port: uint32(consts.ProxyPortInt32), Headers: &istionetworking.Headers{ Request: &istionetworking.Headers_HeaderOperations{ Set: map[string]string{ consts.CortexAPINameHeader: api.Name, consts.CortexQueueURLHeader: queueURL, }, }, }, }, }, PrefixPath: api.Networking.Endpoint, Rewrite: pointer.String("/"), Annotations: api.ToK8sAnnotations(), Labels: map[string]string{ "apiName": api.Name, "apiKind": api.Kind.String(), "apiID": api.ID, "specID": api.SpecID, "initialDeploymentTime": s.Int64(api.InitialDeploymentTime), "deploymentID": api.DeploymentID, "podID": api.PodID, "cortex.dev/api": "true", }, }) } func configMapSpec(api spec.API) (kcore.ConfigMap, error) { configMapConfig := workloads.ConfigMapConfig{ Probes: workloads.GetReadinessProbesFromContainers(api.Pod.Containers), } configMapData, err := configMapConfig.GenerateConfigMapData() if err != nil { return kcore.ConfigMap{}, err } return *k8s.ConfigMap(&k8s.ConfigMapSpec{ Name: workloads.K8sName(api.Name), Data: configMapData, Labels: map[string]string{ "apiName": api.Name, "apiKind": api.Kind.String(), "cortex.dev/api": "true", }, }), nil } func deploymentSpec(api spec.API, prevDeployment *kapps.Deployment, queueURL string) kapps.Deployment { var ( containers []kcore.Container volumes []kcore.Volume ) containers, volumes = workloads.AsyncContainers(api, queueURL) return *k8s.Deployment(&k8s.DeploymentSpec{ Name: workloads.K8sName(api.Name), Replicas: getRequestedReplicasFromDeployment(api, prevDeployment), MaxSurge: pointer.String(api.UpdateStrategy.MaxSurge), MaxUnavailable: pointer.String(api.UpdateStrategy.MaxUnavailable), Labels: map[string]string{ "apiName": api.Name, "apiKind": api.Kind.String(), "apiID": api.ID, "specID": api.SpecID, "initialDeploymentTime": s.Int64(api.InitialDeploymentTime), "deploymentID": api.DeploymentID, "podID": api.PodID, "cortex.dev/api": "true", }, Annotations: api.ToK8sAnnotations(), Selector: map[string]string{ "apiName": api.Name, "apiKind": api.Kind.String(), }, PodSpec: k8s.PodSpec{ Labels: map[string]string{ "apiName": api.Name, "apiKind": api.Kind.String(), "apiID": api.ID, "initialDeploymentTime": s.Int64(api.InitialDeploymentTime), "deploymentID": api.DeploymentID, "podID": api.PodID, "cortex.dev/api": "true", }, K8sPodSpec: kcore.PodSpec{ RestartPolicy: "Always", TerminationGracePeriodSeconds: pointer.Int64(_terminationGracePeriodSeconds), Containers: containers, NodeSelector: workloads.NodeSelectors(), Tolerations: workloads.GenerateResourceTolerations(), Affinity: workloads.GenerateNodeAffinities(api.NodeGroups), Volumes: volumes, ServiceAccountName: workloads.ServiceAccountName, }, }, }) } func getRequestedReplicasFromDeployment(api spec.API, deployment *kapps.Deployment) int32 { requestedReplicas := api.Autoscaling.InitReplicas if deployment != nil && deployment.Spec.Replicas != nil { requestedReplicas = *deployment.Spec.Replicas } if requestedReplicas < api.Autoscaling.MinReplicas { requestedReplicas = api.Autoscaling.MinReplicas } if requestedReplicas > api.Autoscaling.MaxReplicas { requestedReplicas = api.Autoscaling.MaxReplicas } return requestedReplicas } ================================================ FILE: pkg/operator/resources/asyncapi/queue.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package asyncapi import ( "fmt" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/sqs" "github.com/cortexlabs/cortex/pkg/config" "github.com/cortexlabs/cortex/pkg/lib/errors" s "github.com/cortexlabs/cortex/pkg/lib/strings" "github.com/cortexlabs/cortex/pkg/types/clusterconfig" ) func createFIFOQueue(apiName string, initialDeploymentTime int64, tags map[string]string) (string, error) { for key, value := range config.ClusterConfig.Tags { tags[key] = value } queueName := apiQueueName(apiName, initialDeploymentTime) attributes := map[string]string{ sqs.QueueAttributeNameFifoQueue: "true", sqs.QueueAttributeNameVisibilityTimeout: "60", } output, err := config.AWS.SQS().CreateQueue( &sqs.CreateQueueInput{ Attributes: aws.StringMap(attributes), QueueName: aws.String(queueName), Tags: aws.StringMap(tags), }, ) if err != nil { return "", errors.Wrap(err, "failed to create sqs queue", queueName) } return *output.QueueUrl, nil } func apiQueueName(apiName string, initialDeploymentTime int64) string { // initialDeploymentTime is incorporated so that the queue name changes when doing a deploy after a delete // (if the queue name doesn't change, the user would have to wait 60 seconds before recreating the queue) initialDeploymentTimeStr := s.Int64(initialDeploymentTime) initialDeploymentTimeID := initialDeploymentTimeStr[len(initialDeploymentTimeStr)-10:] return config.ClusterConfig.SQSNamePrefix() + apiName + clusterconfig.SQSQueueDelimiter + initialDeploymentTimeID + ".fifo" } func deleteQueueByURL(queueURL string) error { _, err := config.AWS.SQS().DeleteQueue(&sqs.DeleteQueueInput{ QueueUrl: aws.String(queueURL), }) if err != nil { return errors.Wrap(err, "failed to delete queue", queueURL) } return err } func getQueueURL(apiName string, initialDeploymentTime int64) (string, error) { operatorAccountID, _, err := config.AWS.GetCachedAccountID() if err != nil { return "", errors.Wrap(err, "failed to construct queue url", "unable to get account id") } return fmt.Sprintf( "https://sqs.%s.amazonaws.com/%s/%s", config.AWS.Region, operatorAccountID, apiQueueName(apiName, initialDeploymentTime), ), nil } ================================================ FILE: pkg/operator/resources/asyncapi/queue_metrics.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package asyncapi import ( "context" "strconv" "time" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/sqs" "github.com/cortexlabs/cortex/pkg/config" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/types/userconfig" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" ) const ( _sqsQueryTimeoutSeconds = 10 ) var activeGauge = promauto.NewGaugeVec( prometheus.GaugeOpts{ Name: "cortex_async_active", Help: "The number of messages that are actively being processed by an AsyncAPI", ConstLabels: map[string]string{"api_kind": userconfig.AsyncAPIKind.String()}, }, []string{"api_name"}, ) var queuedGauge = promauto.NewGaugeVec( prometheus.GaugeOpts{ Name: "cortex_async_queued", Help: "The number queued messages for an AsyncAPI", ConstLabels: map[string]string{"api_kind": userconfig.AsyncAPIKind.String()}, }, []string{"api_name"}, ) var inFlightGauge = promauto.NewGaugeVec( prometheus.GaugeOpts{ Name: "cortex_async_in_flight", Help: "The number of in-flight messages for an AsyncAPI (including active and queued)", ConstLabels: map[string]string{"api_kind": userconfig.AsyncAPIKind.String()}, }, []string{"api_name"}, ) func updateQueueLengthMetricsFn(apiName, queueURL string) func() error { return func() error { sqsClient := config.AWS.SQS() ctx, cancel := context.WithTimeout(context.Background(), _sqsQueryTimeoutSeconds*time.Second) defer cancel() input := &sqs.GetQueueAttributesInput{ AttributeNames: []*string{ aws.String("ApproximateNumberOfMessages"), aws.String("ApproximateNumberOfMessagesNotVisible"), }, QueueUrl: aws.String(queueURL), } output, err := sqsClient.GetQueueAttributesWithContext(ctx, input) if err != nil { return errors.WithStack(err) } visibleMessagesStr := output.Attributes["ApproximateNumberOfMessages"] invisibleMessagesStr := output.Attributes["ApproximateNumberOfMessagesNotVisible"] visibleMessages, err := strconv.ParseFloat(*visibleMessagesStr, 64) if err != nil { return errors.WithStack(err) } invisibleMessages, err := strconv.ParseFloat(*invisibleMessagesStr, 64) if err != nil { return errors.WithStack(err) } activeGauge.WithLabelValues(apiName).Set(invisibleMessages) queuedGauge.WithLabelValues(apiName).Set(visibleMessages) inFlightGauge.WithLabelValues(apiName).Set(invisibleMessages + visibleMessages) return nil } } ================================================ FILE: pkg/operator/resources/asyncapi/status.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package asyncapi import ( "github.com/cortexlabs/cortex/pkg/lib/k8s" "github.com/cortexlabs/cortex/pkg/types/status" kapps "k8s.io/api/apps/v1" kcore "k8s.io/api/core/v1" ) func GetReplicaCounts(deployment *kapps.Deployment, pods []kcore.Pod) *status.ReplicaCounts { counts := status.ReplicaCounts{} counts.Requested = *deployment.Spec.Replicas for i := range pods { pod := pods[i] if pod.Labels["apiName"] != deployment.Labels["apiName"] { continue } addPodToReplicaCounts(&pod, deployment, &counts) } return &counts } func addPodToReplicaCounts(pod *kcore.Pod, deployment *kapps.Deployment, counts *status.ReplicaCounts) { latest := false if isPodSpecLatest(deployment, pod) { latest = true } isPodReady := k8s.IsPodReady(pod) if latest && isPodReady { counts.Ready++ return } else if !latest && isPodReady { counts.ReadyOutOfDate++ return } podStatus := k8s.GetPodStatus(pod) if podStatus == k8s.PodStatusTerminating { counts.Terminating++ return } if !latest { return } switch podStatus { case k8s.PodStatusPending: counts.Pending++ case k8s.PodStatusStalled: counts.Stalled++ case k8s.PodStatusCreating: counts.Creating++ case k8s.PodStatusReady: counts.Ready++ case k8s.PodStatusNotReady: counts.NotReady++ case k8s.PodStatusErrImagePull: counts.ErrImagePull++ case k8s.PodStatusFailed: counts.Failed++ case k8s.PodStatusKilled: counts.Killed++ case k8s.PodStatusKilledOOM: counts.KilledOOM++ case k8s.PodStatusUnknown: counts.Unknown++ } } ================================================ FILE: pkg/operator/resources/errors.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package resources import ( "fmt" "strings" "github.com/cortexlabs/cortex/pkg/config" "github.com/cortexlabs/cortex/pkg/consts" "github.com/cortexlabs/cortex/pkg/lib/console" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/k8s" "github.com/cortexlabs/cortex/pkg/lib/slices" s "github.com/cortexlabs/cortex/pkg/lib/strings" "github.com/cortexlabs/cortex/pkg/lib/table" "github.com/cortexlabs/cortex/pkg/operator/operator" "github.com/cortexlabs/cortex/pkg/types/userconfig" "github.com/cortexlabs/cortex/pkg/workloads" kresource "k8s.io/apimachinery/pkg/api/resource" ) const ( ErrOperationIsOnlySupportedForKind = "resources.operation_is_only_supported_for_kind" ErrAPINotDeployed = "resources.api_not_deployed" ErrAPIIDNotFound = "resources.api_id_not_found" ErrCannotChangeTypeOfDeployedAPI = "resources.cannot_change_kind_of_deployed_api" ErrNoAvailableNodeComputeLimit = "resources.no_available_node_compute_limit" ErrJobIDRequired = "resources.job_id_required" ErrRealtimeAPIUsedByTrafficSplitter = "resources.realtime_api_used_by_traffic_splitter" ErrAPIsNotDeployed = "resources.apis_not_deployed" ErrInvalidNodeGroupSelector = "resources.invalid_node_group_selector" ErrNoNodeGroups = "resources.no_node_groups" ) func ErrorOperationIsOnlySupportedForKind(resource operator.DeployedResource, supportedKind userconfig.Kind, supportedKinds ...userconfig.Kind) error { supportedKindsSlice := append(make([]string, 0, 1+len(supportedKinds)), supportedKind.String()) for _, kind := range supportedKinds { supportedKindsSlice = append(supportedKindsSlice, kind.String()) } msg := fmt.Sprintf("%s %s", s.StrsOr(supportedKindsSlice), s.PluralS(userconfig.KindKey, len(supportedKindsSlice))) return errors.WithStack(&errors.Error{ Kind: ErrOperationIsOnlySupportedForKind, Message: fmt.Sprintf("this operation is only allowed for %s and is not supported for %s of kind %s", msg, resource.Name, resource.Kind), }) } func ErrorAPINotDeployed(apiName string) error { return errors.WithStack(&errors.Error{ Kind: ErrAPINotDeployed, Message: fmt.Sprintf("%s is not deployed", apiName), }) } func ErrorAPIIDNotFound(apiName string, apiID string) error { return errors.WithStack(&errors.Error{ Kind: ErrAPIIDNotFound, Message: fmt.Sprintf("%s with id %s has never been deployed", apiName, apiID), }) } func ErrorCannotChangeKindOfDeployedAPI(name string, newKind, prevKind userconfig.Kind) error { return errors.WithStack(&errors.Error{ Kind: ErrCannotChangeTypeOfDeployedAPI, Message: fmt.Sprintf("cannot change the kind of %s to %s because it has already been deployed with kind %s; please delete it with `cortex delete %s` and redeploy after updating the api configuration appropriately", name, newKind.String(), prevKind.String(), name), }) } func ErrorNoAvailableNodeComputeLimit(api *userconfig.API, compute userconfig.Compute, maxMemMap map[string]kresource.Quantity) error { msg := "no instance types in your cluster are large enough to satisfy the requested resources for your pod\n\n" msg += console.Bold("requested pod resources\n") msg += podResourceRequestsTable(api, compute) msg += "\n" + s.TrimTrailingNewLines(nodeGroupResourcesTable(api, compute, maxMemMap)) return errors.WithStack(&errors.Error{ Kind: ErrNoAvailableNodeComputeLimit, Message: msg, }) } func ErrorAPIUsedByTrafficSplitter(trafficSplitters []string) error { return errors.WithStack(&errors.Error{ Kind: ErrRealtimeAPIUsedByTrafficSplitter, Message: fmt.Sprintf("cannot delete api because it is used by the following %s: %s", s.PluralS("TrafficSplitter", len(trafficSplitters)), s.StrsSentence(trafficSplitters, "")), }) } func ErrorAPIsNotDeployed(notDeployedAPIs []string) error { message := fmt.Sprintf("apis %s were either not found or are not RealtimeAPIs", s.StrsAnd(notDeployedAPIs)) if len(notDeployedAPIs) == 1 { message = fmt.Sprintf("api %s was either not found or is not a RealtimeAPI", notDeployedAPIs[0]) } return errors.WithStack(&errors.Error{ Kind: ErrAPIsNotDeployed, Message: message, }) } func ErrorInvalidNodeGroupSelector(selected string, availableNodeGroups []string) error { return errors.WithStack(&errors.Error{ Kind: ErrInvalidNodeGroupSelector, Message: fmt.Sprintf("node group \"%s\" doesn't exist; remove the node group selector to let Cortex determine automatically where to place the API, or specify a valid node group name (%s)", selected, s.StrsOr(availableNodeGroups)), }) } func ErrorNoNodeGroups() error { return errors.WithStack(&errors.Error{ Kind: ErrNoNodeGroups, Message: fmt.Sprintf("your api cannot be deployed because your cluster doesn't have any node groups; create a node group with `cortex cluster configure CLUSTER_CONFIG_FILE`"), }) } func podResourceRequestsTable(api *userconfig.API, compute userconfig.Compute) string { sidecarCPUNote := "" sidecarMemNote := "" if api.Kind == userconfig.RealtimeAPIKind { sidecarCPUNote = fmt.Sprintf(" (including %s for the %s sidecar container)", consts.CortexProxyCPU.String(), workloads.ProxyContainerName) sidecarMemNote = fmt.Sprintf(" (including %s for the %s sidecar container)", k8s.ToMiCeilStr(consts.CortexProxyMem), workloads.ProxyContainerName) } else if api.Kind == userconfig.AsyncAPIKind || api.Kind == userconfig.BatchAPIKind { sidecarCPUNote = fmt.Sprintf(" (including %s for the %s sidecar container)", consts.CortexDequeuerCPU.String(), workloads.DequeuerContainerName) sidecarMemNote = fmt.Sprintf(" (including %s for the %s sidecar container)", k8s.ToMiCeilStr(consts.CortexDequeuerMem), workloads.DequeuerContainerName) } var items table.KeyValuePairs if compute.CPU != nil { items.Add("CPU", compute.CPU.String()+sidecarCPUNote) } if compute.Mem != nil { items.Add("memory", compute.Mem.ToMiCeilStr()+sidecarMemNote) } if compute.GPU > 0 { items.Add("GPU", compute.GPU) } if compute.Inf > 0 { items.Add("Inf", compute.Inf) } return items.String() } func nodeGroupResourcesTable(api *userconfig.API, compute userconfig.Compute, maxMemMap map[string]kresource.Quantity) string { var skippedNodeGroups []string var nodeGroupResourceRows [][]interface{} showGPU := false showInf := false if compute.GPU > 0 { showGPU = true } if compute.Inf > 0 { showInf = true } for _, ng := range config.ClusterConfig.NodeGroups { nodeCPU, nodeMem, nodeGPU, nodeInf := getNodeCapacity(ng.InstanceType, maxMemMap) if nodeGPU > 0 { showGPU = true } if nodeInf > 0 { showInf = true } if api.NodeGroups != nil && !slices.HasString(api.NodeGroups, ng.Name) { skippedNodeGroups = append(skippedNodeGroups, ng.Name) } else { nodeGroupResourceRows = append(nodeGroupResourceRows, []interface{}{ng.Name, ng.InstanceType, nodeCPU, k8s.ToMiFloorStr(nodeMem), nodeGPU, nodeInf}) } } nodeGroupResourceRowsTable := table.Table{ Headers: []table.Header{ {Title: "node group"}, {Title: "instance type"}, {Title: "CPU"}, {Title: "memory"}, {Title: "GPU", Hidden: !showGPU}, {Title: "Inf", Hidden: !showInf}, }, Rows: nodeGroupResourceRows, } out := nodeGroupResourceRowsTable.MustFormat() if len(skippedNodeGroups) > 0 { out += fmt.Sprintf("\nthe following %s skipped (based on the api configuration's %s field): %s", s.PluralCustom("node group was", "node groups were", len(skippedNodeGroups)), userconfig.NodeGroupsKey, strings.Join(skippedNodeGroups, ", ")) } return out } ================================================ FILE: pkg/operator/resources/job/batchapi/api.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package batchapi import ( "context" "fmt" "path/filepath" "time" "github.com/cortexlabs/cortex/pkg/config" batch "github.com/cortexlabs/cortex/pkg/crds/apis/batch/v1alpha1" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/k8s" "github.com/cortexlabs/cortex/pkg/lib/parallel" "github.com/cortexlabs/cortex/pkg/lib/pointer" "github.com/cortexlabs/cortex/pkg/lib/sets/strset" "github.com/cortexlabs/cortex/pkg/operator/lib/routines" "github.com/cortexlabs/cortex/pkg/operator/operator" "github.com/cortexlabs/cortex/pkg/operator/resources/job" "github.com/cortexlabs/cortex/pkg/operator/schema" "github.com/cortexlabs/cortex/pkg/types/spec" "github.com/cortexlabs/cortex/pkg/types/status" "github.com/cortexlabs/cortex/pkg/types/userconfig" "github.com/cortexlabs/cortex/pkg/workloads" istioclientnetworking "istio.io/client-go/pkg/apis/networking/v1beta1" "sigs.k8s.io/controller-runtime/pkg/client" ) const _batchDashboardUID = "batchapi" func UpdateAPI(apiConfig *userconfig.API) (*spec.API, string, error) { prevVirtualService, err := config.K8s.GetVirtualService(workloads.K8sName(apiConfig.Name)) if err != nil { return nil, "", err } initialDeploymentTime := time.Now().UnixNano() if prevVirtualService != nil && prevVirtualService.Labels["initialDeploymentTime"] != "" { var err error initialDeploymentTime, err = k8s.ParseInt64Label(prevVirtualService, "initialDeploymentTime") if err != nil { return nil, "", err } } api := spec.GetAPISpec(apiConfig, initialDeploymentTime, "", config.ClusterConfig.ClusterUID) // Deployment ID not needed for BatchAPI spec if prevVirtualService == nil { if err := config.AWS.UploadJSONToS3(api, config.ClusterConfig.Bucket, api.Key); err != nil { return nil, "", errors.Wrap(err, "upload api spec") } err = applyK8sResources(api, prevVirtualService) if err != nil { routines.RunWithPanicHandler(func() { _ = deleteK8sResources(api.Name) }) return nil, "", err } return api, fmt.Sprintf("created %s", api.Resource.UserString()), nil } if prevVirtualService.Labels["specID"] != api.SpecID { if err := config.AWS.UploadJSONToS3(api, config.ClusterConfig.Bucket, api.Key); err != nil { return nil, "", errors.Wrap(err, "upload api spec") } err = applyK8sResources(api, prevVirtualService) if err != nil { return nil, "", err } return api, fmt.Sprintf("updated %s", api.Resource.UserString()), nil } return api, fmt.Sprintf("%s is up to date", api.Resource.UserString()), nil } func DeleteAPI(apiName string, keepCache bool) error { // best effort deletion, so don't handle error yet err := parallel.RunFirstErr( func() error { return deleteK8sResources(apiName) }, func() error { if keepCache { return nil } return deleteS3Resources(apiName) }, ) if err != nil { return err } return nil } func deleteS3Resources(apiName string) error { return parallel.RunFirstErr( func() error { prefix := filepath.Join(config.ClusterConfig.ClusterUID, "apis", apiName) return config.AWS.DeleteS3Dir(config.ClusterConfig.Bucket, prefix, true) }, func() error { prefix := spec.JobAPIPrefix(config.ClusterConfig.ClusterUID, userconfig.BatchAPIKind, apiName) routines.RunWithPanicHandler(func() { _ = config.AWS.DeleteS3Dir(config.ClusterConfig.Bucket, prefix, true) // deleting job files may take a while }) return nil }, ) } // GetAllAPIs returns all batch apis, for each API returning the most recently submitted job and all running jobs func GetAllAPIs(virtualServices []istioclientnetworking.VirtualService, batchJobList []batch.BatchJob) ([]schema.APIResponse, error) { batchAPIsMap := map[string]*schema.APIResponse{} jobIDToBatchJobMap := map[string]*batch.BatchJob{} apiNameToBatchJobsMap := map[string][]*batch.BatchJob{} for i, batchJob := range batchJobList { jobIDToBatchJobMap[batchJob.Name] = &batchJobList[i] apiNameToBatchJobsMap[batchJob.Spec.APIName] = append(apiNameToBatchJobsMap[batchJob.Spec.APIName], &batchJobList[i]) } for i := range virtualServices { apiName := virtualServices[i].Labels["apiName"] metadata, err := spec.MetadataFromVirtualService(&virtualServices[i]) if err != nil { return nil, errors.Wrap(err, fmt.Sprintf("api %s", apiName)) } var jobStatuses []status.BatchJobStatus batchJobs := apiNameToBatchJobsMap[metadata.Name] if len(batchJobs) == 0 { jobStates, err := job.GetMostRecentlySubmittedJobStates(metadata.Name, 1, userconfig.BatchAPIKind) if err != nil { return nil, err } if len(jobStates) > 0 { jobStatus, err := getJobStatusFromJobState(jobStates[0]) if err != nil { return nil, err } jobStatuses = append(jobStatuses, *jobStatus) } } else { for i := range batchJobs { batchJob := batchJobs[i] jobStatus, err := getJobStatusFromBatchJob(*batchJob) if err != nil { return nil, err } jobStatuses = append(jobStatuses, *jobStatus) } } batchAPIsMap[metadata.Name] = &schema.APIResponse{ Metadata: metadata, BatchJobStatuses: jobStatuses, } } batchAPIList := make([]schema.APIResponse, 0, len(batchAPIsMap)) for _, batchAPI := range batchAPIsMap { batchAPIList = append(batchAPIList, *batchAPI) } return batchAPIList, nil } func GetAPIByName(deployedResource *operator.DeployedResource) ([]schema.APIResponse, error) { metadata, err := spec.MetadataFromVirtualService(deployedResource.VirtualService) if err != nil { return nil, err } api, err := operator.DownloadAPISpec(deployedResource.Name, metadata.APIID) if err != nil { return nil, err } ctx := context.Background() batchJobList := batch.BatchJobList{} if err = config.K8s.List( ctx, &batchJobList, client.InNamespace(config.K8s.Namespace), client.MatchingLabels{"apiName": deployedResource.Name}, ); err != nil { return nil, err } endpoint, err := operator.APIEndpoint(api) if err != nil { return nil, err } var jobStatuses []status.BatchJobStatus jobIDSet := strset.New() for _, batchJob := range batchJobList.Items { jobStatus, err := getJobStatusFromBatchJob(batchJob) if err != nil { return nil, err } jobStatuses = append(jobStatuses, *jobStatus) jobIDSet.Add(batchJob.Name) } if len(jobStatuses) < 10 { jobStates, err := job.GetMostRecentlySubmittedJobStates(deployedResource.Name, 10+len(jobStatuses), userconfig.BatchAPIKind) if err != nil { return nil, err } for _, jobState := range jobStates { if jobIDSet.Has(jobState.ID) { continue } jobIDSet.Add(jobState.ID) jobStatus, err := getJobStatusFromJobState(jobState) if err != nil { return nil, err } if jobStatus != nil { jobStatuses = append(jobStatuses, *jobStatus) if len(jobStatuses) == 10 { break } } } } dashboardURL := pointer.String(getDashboardURL(api.Name)) return []schema.APIResponse{ { Spec: api, Metadata: metadata, BatchJobStatuses: jobStatuses, Endpoint: &endpoint, DashboardURL: dashboardURL, }, }, nil } func getDashboardURL(apiName string) string { loadBalancerURL, err := operator.LoadBalancerURL() if err != nil { return "" } dashboardURL := fmt.Sprintf( "%s/dashboard/d/%s/batchapi?orgId=1&refresh=30s&var-api_name=%s", loadBalancerURL, _batchDashboardUID, apiName, ) return dashboardURL } ================================================ FILE: pkg/operator/resources/job/batchapi/errors.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package batchapi import ( "fmt" "github.com/cortexlabs/cortex/pkg/lib/errors" ) const ( ErrNoS3FilesFound = "batchapi.no_s3_files_found" ErrBatchItemSizeExceedsLimit = "batchapi.item_size_exceeds_limit" ) func ErrorNoS3FilesFound() error { return errors.WithStack(&errors.Error{ Kind: ErrNoS3FilesFound, Message: "no s3 files match search criteria", }) } func ErrorItemSizeExceedsLimit(index int, size int, limit int) error { return errors.WithStack(&errors.Error{ Kind: ErrBatchItemSizeExceedsLimit, Message: fmt.Sprintf("item %d has size %d bytes which exceeds the limit (%d bytes)", index, size, limit), }) } ================================================ FILE: pkg/operator/resources/job/batchapi/job.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package batchapi import ( "context" "time" "github.com/cortexlabs/cortex/pkg/config" batch "github.com/cortexlabs/cortex/pkg/crds/apis/batch/v1alpha1" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/pointer" "github.com/cortexlabs/cortex/pkg/operator/operator" "github.com/cortexlabs/cortex/pkg/operator/schema" "github.com/cortexlabs/cortex/pkg/types/spec" "github.com/cortexlabs/cortex/pkg/types/userconfig" "github.com/cortexlabs/cortex/pkg/workloads" "github.com/cortexlabs/yaml" kmeta "k8s.io/apimachinery/pkg/apis/meta/v1" ) const _batchJobTTL = 40 * time.Second // Double the duration of the statsd pod monitor func DryRun(submission *schema.BatchJobSubmission) ([]string, error) { err := validateJobSubmission(submission) if err != nil { return nil, err } if submission.FilePathLister != nil { s3Files, err := listFilesDryRun(&submission.FilePathLister.S3Lister) if err != nil { return nil, errors.Wrap(err, schema.FilePathListerKey) } return s3Files, nil } if submission.DelimitedFiles != nil { s3Files, err := listFilesDryRun(&submission.DelimitedFiles.S3Lister) if err != nil { return nil, errors.Wrap(err, schema.DelimitedFilesKey) } return s3Files, nil } return nil, nil } func SubmitJob(apiName string, submission *schema.BatchJobSubmission) (*spec.BatchJob, error) { err := validateJobSubmission(submission) if err != nil { return nil, err } virtualService, err := config.K8s.GetVirtualService(workloads.K8sName(apiName)) if err != nil { return nil, err } apiID := virtualService.Labels["apiID"] jobID := spec.MonotonicallyDecreasingID() apiSpec, err := operator.DownloadAPISpec(apiName, apiID) if err != nil { return nil, err } jobSpec := spec.BatchJob{ RuntimeBatchJobConfig: submission.RuntimeBatchJobConfig, JobKey: spec.JobKey{ APIName: apiName, ID: jobID, Kind: userconfig.BatchAPIKind, }, APIID: apiSpec.ID, StartTime: time.Now(), } err = uploadJobSpec(&jobSpec) if err != nil { return nil, err } // upload job payload for enqueuer payloadKey := spec.JobPayloadKey(config.ClusterConfig.ClusterUID, userconfig.BatchAPIKind, apiName, jobID) if err = config.AWS.UploadJSONToS3(submission, config.ClusterConfig.Bucket, payloadKey); err != nil { return nil, err } var jobConfig *string if submission.Config != nil { jobConfigBytes, err := yaml.Marshal(submission.Config) if err != nil { return nil, err } jobConfig = pointer.String(string(jobConfigBytes)) } var timeout *kmeta.Duration if submission.Timeout != nil { timeout = &kmeta.Duration{Duration: time.Duration(*submission.Timeout) * time.Second} } var deadLetterQueue *batch.DeadLetterQueueSpec if submission.SQSDeadLetterQueue != nil { deadLetterQueue = &batch.DeadLetterQueueSpec{ ARN: submission.SQSDeadLetterQueue.ARN, MaxReceiveCount: int32(submission.SQSDeadLetterQueue.MaxReceiveCount), } } batchJob := batch.BatchJob{ ObjectMeta: kmeta.ObjectMeta{ Name: jobID, Namespace: config.K8s.Namespace, Labels: map[string]string{ "apiName": apiName, "apiID": apiID, "specID": virtualService.Labels["specID"], "apiKind": userconfig.BatchAPIKind.String(), "cortex.dev/api": "true", }, }, Spec: batch.BatchJobSpec{ APIName: apiName, APIID: apiID, Workers: int32(submission.Workers), Config: jobConfig, Timeout: timeout, DeadLetterQueue: deadLetterQueue, TTL: &kmeta.Duration{Duration: _batchJobTTL}, NodeGroups: apiSpec.NodeGroups, Probes: workloads.GetReadinessProbesFromContainers(apiSpec.Pod.Containers), }, } ctx := context.Background() if err = config.K8s.Create(ctx, &batchJob); err != nil { return nil, err } return &jobSpec, nil } func StopJob(jobKey spec.JobKey) error { return config.K8s.Delete(context.Background(), &batch.BatchJob{ ObjectMeta: kmeta.ObjectMeta{Name: jobKey.ID, Namespace: config.K8s.Namespace}, }) } func uploadJobSpec(jobSpec *spec.BatchJob) error { err := config.AWS.UploadJSONToS3(jobSpec, config.ClusterConfig.Bucket, jobSpec.SpecFilePath(config.ClusterConfig.ClusterUID)) if err != nil { return err } return nil } ================================================ FILE: pkg/operator/resources/job/batchapi/job_status.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package batchapi import ( "context" "net/url" "time" "github.com/cortexlabs/cortex/pkg/config" batch "github.com/cortexlabs/cortex/pkg/crds/apis/batch/v1alpha1" "github.com/cortexlabs/cortex/pkg/lib/aws" "github.com/cortexlabs/cortex/pkg/lib/pointer" "github.com/cortexlabs/cortex/pkg/lib/telemetry" "github.com/cortexlabs/cortex/pkg/operator/operator" "github.com/cortexlabs/cortex/pkg/operator/resources/job" "github.com/cortexlabs/cortex/pkg/operator/schema" "github.com/cortexlabs/cortex/pkg/types/metrics" "github.com/cortexlabs/cortex/pkg/types/spec" "github.com/cortexlabs/cortex/pkg/types/status" "github.com/cortexlabs/cortex/pkg/types/userconfig" "github.com/cortexlabs/yaml" kerrors "k8s.io/apimachinery/pkg/api/errors" "sigs.k8s.io/controller-runtime/pkg/client" ) func GetJob(jobKey spec.JobKey) (*schema.BatchJobResponse, error) { ctx := context.Background() var batchJob batch.BatchJob err := config.K8s.Get(ctx, client.ObjectKey{Name: jobKey.ID, Namespace: config.K8s.Namespace}, &batchJob) if err != nil && !kerrors.IsNotFound(err) { return nil, err } if kerrors.IsNotFound(err) { return getJobFromS3(jobKey) } return getJobFromCluster(batchJob) } func getJobFromS3(jobKey spec.JobKey) (*schema.BatchJobResponse, error) { jobState, err := job.GetJobState(jobKey) if err != nil { return nil, err } jobStatus, err := getJobStatusFromJobState(jobState) if err != nil { return nil, err } var jobMetrics *metrics.BatchMetrics if _, ok := jobState.LastUpdatedMap[spec.MetricsFileKey]; ok && jobState.Status.IsCompleted() { jobMetrics, err = readMetricsFromS3(jobState.JobKey) if err != nil { telemetry.Error(err) } } if jobMetrics == nil { // try to get metrics from prometheus if they aren't available in S3 because there might be a delay jobMetrics, err = batch.GetMetrics(config.Prometheus, jobStatus.JobKey, time.Now()) if err != nil { telemetry.Error(err) } } apiSpec, err := operator.DownloadAPISpec(jobStatus.APIName, jobStatus.APIID) if err != nil { return nil, err } endpoint, err := getJobEndpoint(apiSpec, jobKey) if err != nil { return nil, err } return &schema.BatchJobResponse{ APISpec: *apiSpec, JobStatus: *jobStatus, Metrics: jobMetrics, Endpoint: endpoint, }, nil } func getJobFromCluster(batchJob batch.BatchJob) (*schema.BatchJobResponse, error) { jobStatus, err := getJobStatusFromBatchJob(batchJob) if err != nil { return nil, err } jobMetrics, err := batch.GetMetrics(config.Prometheus, jobStatus.JobKey, time.Now()) if err != nil { telemetry.Error(err) } apiSpec, err := operator.DownloadAPISpec(jobStatus.APIName, jobStatus.APIID) if err != nil { return nil, err } endpoint, err := getJobEndpoint(apiSpec, jobStatus.JobKey) if err != nil { return nil, err } return &schema.BatchJobResponse{ APISpec: *apiSpec, JobStatus: *jobStatus, Metrics: jobMetrics, Endpoint: endpoint, }, nil } func getJobStatusFromBatchJob(batchJob batch.BatchJob) (*status.BatchJobStatus, error) { jobKey := spec.JobKey{ ID: batchJob.Name, APIName: batchJob.Spec.APIName, Kind: userconfig.BatchAPIKind, } var deadLetterQueue *spec.SQSDeadLetterQueue if batchJob.Spec.DeadLetterQueue != nil { deadLetterQueue = &spec.SQSDeadLetterQueue{ ARN: batchJob.Spec.DeadLetterQueue.ARN, MaxReceiveCount: int(batchJob.Spec.DeadLetterQueue.MaxReceiveCount), } } var jobConfig map[string]interface{} if batchJob.Spec.Config != nil { if err := yaml.Unmarshal([]byte(*batchJob.Spec.Config), &jobConfig); err != nil { return nil, err } } var timeout *int if batchJob.Spec.Timeout != nil { timeout = pointer.Int(int(batchJob.Spec.Timeout.Seconds())) } jobStatus := status.BatchJobStatus{ BatchJob: spec.BatchJob{ JobKey: jobKey, RuntimeBatchJobConfig: spec.RuntimeBatchJobConfig{ Workers: int(batchJob.Spec.Workers), SQSDeadLetterQueue: deadLetterQueue, Config: jobConfig, Timeout: timeout, }, APIID: batchJob.Spec.APIID, StartTime: batchJob.CreationTimestamp.Time, SQSUrl: batchJob.Status.QueueURL, TotalBatchCount: batchJob.Status.TotalBatchCount, }, WorkerCounts: batchJob.Status.WorkerCounts, Status: batchJob.Status.Status, } if batchJob.Status.EndTime != nil { jobStatus.EndTime = &batchJob.Status.EndTime.Time } queueMetrics, err := getQueueMetrics(jobKey) if aws.IsNonExistentQueueErr(err) { jobStatus.BatchesInQueue = 0 jobStatus.TotalBatchCount = 0 } else if err != nil { return nil, err } else { jobStatus.BatchesInQueue = queueMetrics.TotalUserMessages() if batchJob.Status.Status == status.JobEnqueuing { jobStatus.TotalBatchCount = queueMetrics.TotalUserMessages() } } jobStatus.WorkerCounts = batchJob.Status.WorkerCounts return &jobStatus, nil } func getJobStatusFromJobState(jobState *job.State) (*status.BatchJobStatus, error) { jobKey := jobState.JobKey jobSpec, err := operator.DownloadBatchJobSpec(jobKey) if err != nil { return nil, err } jobStatus := status.BatchJobStatus{ BatchJob: *jobSpec, EndTime: jobState.EndTime, Status: jobState.Status, } return &jobStatus, nil } func readMetricsFromS3(jobKey spec.JobKey) (*metrics.BatchMetrics, error) { s3Key := spec.JobMetricsKey(config.ClusterConfig.ClusterUID, userconfig.BatchAPIKind, jobKey.APIName, jobKey.ID) batchMetrics := metrics.BatchMetrics{} err := config.AWS.ReadJSONFromS3(&batchMetrics, config.ClusterConfig.Bucket, s3Key) if err != nil { return nil, err } return &batchMetrics, nil } func getJobEndpoint(apiSpec *spec.API, jobKey spec.JobKey) (string, error) { endpoint, err := operator.APIEndpoint(apiSpec) if err != nil { return "", err } parsedURL, err := url.Parse(endpoint) if err != nil { return "", err } q := parsedURL.Query() q.Add("jobID", jobKey.ID) parsedURL.RawQuery = q.Encode() return parsedURL.String(), nil } ================================================ FILE: pkg/operator/resources/job/batchapi/k8s_specs.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package batchapi import ( "context" "path" "github.com/cortexlabs/cortex/pkg/config" "github.com/cortexlabs/cortex/pkg/consts" batch "github.com/cortexlabs/cortex/pkg/crds/apis/batch/v1alpha1" "github.com/cortexlabs/cortex/pkg/lib/k8s" "github.com/cortexlabs/cortex/pkg/lib/parallel" "github.com/cortexlabs/cortex/pkg/lib/pointer" s "github.com/cortexlabs/cortex/pkg/lib/strings" "github.com/cortexlabs/cortex/pkg/types/spec" "github.com/cortexlabs/cortex/pkg/workloads" istioclientnetworking "istio.io/client-go/pkg/apis/networking/v1beta1" "sigs.k8s.io/controller-runtime/pkg/client" ) const _operatorService = "operator" func virtualServiceSpec(api *spec.API) *istioclientnetworking.VirtualService { return k8s.VirtualService(&k8s.VirtualServiceSpec{ Name: workloads.K8sName(api.Name), Gateways: []string{"apis-gateway"}, Destinations: []k8s.Destination{{ ServiceName: _operatorService, Weight: 100, Port: uint32(consts.ProxyPortInt32), }}, PrefixPath: api.Networking.Endpoint, Rewrite: pointer.String(path.Join("batch", api.Name)), Annotations: api.ToK8sAnnotations(), Labels: map[string]string{ "apiName": api.Name, "apiID": api.ID, "specID": api.SpecID, "podID": api.PodID, "initialDeploymentTime": s.Int64(api.InitialDeploymentTime), "apiKind": api.Kind.String(), "cortex.dev/api": "true", }, }) } func applyK8sResources(api *spec.API, prevVirtualService *istioclientnetworking.VirtualService) error { newVirtualService := virtualServiceSpec(api) if prevVirtualService == nil { _, err := config.K8s.CreateVirtualService(newVirtualService) return err } _, err := config.K8s.UpdateVirtualService(prevVirtualService, newVirtualService) return err } func deleteK8sResources(apiName string) error { return parallel.RunFirstErr( func() error { err := config.K8s.DeleteAllOf( context.Background(), &batch.BatchJob{}, client.InNamespace(config.K8s.Namespace), client.MatchingLabels{"apiName": apiName}, ) return client.IgnoreNotFound(err) }, func() error { _, err := config.K8s.DeleteVirtualService(workloads.K8sName(apiName)) return err }, ) } ================================================ FILE: pkg/operator/resources/job/batchapi/queue.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package batchapi import ( "fmt" "github.com/cortexlabs/cortex/pkg/config" "github.com/cortexlabs/cortex/pkg/lib/errors" s "github.com/cortexlabs/cortex/pkg/lib/strings" "github.com/cortexlabs/cortex/pkg/types/clusterconfig" "github.com/cortexlabs/cortex/pkg/types/metrics" "github.com/cortexlabs/cortex/pkg/types/spec" ) func apiQueueNamePrefix(apiName string) string { // _b__ return config.ClusterConfig.SQSNamePrefix() + "b" + clusterconfig.SQSQueueDelimiter + apiName + clusterconfig.SQSQueueDelimiter } // QueueName is cx__b__.fifo func getJobQueueName(jobKey spec.JobKey) string { return apiQueueNamePrefix(jobKey.APIName) + jobKey.ID + ".fifo" } func getJobQueueURL(jobKey spec.JobKey) (string, error) { operatorAccountID, _, err := config.AWS.GetCachedAccountID() if err != nil { return "", errors.Wrap(err, "failed to construct queue url", "unable to get account id") } return fmt.Sprintf("https://sqs.%s.amazonaws.com/%s/%s", config.AWS.Region, operatorAccountID, getJobQueueName(jobKey)), nil } func getQueueMetrics(jobKey spec.JobKey) (*metrics.QueueMetrics, error) { queueURL, err := getJobQueueURL(jobKey) if err != nil { return nil, err } return getQueueMetricsFromURL(queueURL) } func getQueueMetricsFromURL(queueURL string) (*metrics.QueueMetrics, error) { attributes, err := config.AWS.GetAllQueueAttributes(queueURL) if err != nil { return nil, errors.Wrap(err, "failed to get queue metrics") } qMetrics := metrics.QueueMetrics{} parsedInt, ok := s.ParseInt(attributes["ApproximateNumberOfMessages"]) if ok { qMetrics.Visible = parsedInt } parsedInt, ok = s.ParseInt(attributes["ApproximateNumberOfMessagesNotVisible"]) if ok { qMetrics.NotVisible = parsedInt } return &qMetrics, nil } ================================================ FILE: pkg/operator/resources/job/batchapi/s3_iterator.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package batchapi import ( "github.com/aws/aws-sdk-go/service/s3" "github.com/cortexlabs/cortex/pkg/config" "github.com/cortexlabs/cortex/pkg/lib/aws" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/operator/schema" "github.com/gobwas/glob" ) // Takes in a function(shouldSkip, bucketName, s3.Object) func s3IteratorFromLister(s3Lister schema.S3Lister, fn func(string, *s3.Object) (bool, error)) (int64, error) { includeGlobPatterns := make([]glob.Glob, 0, len(s3Lister.Includes)) for _, includePattern := range s3Lister.Includes { globExpression, err := glob.Compile(includePattern, '/') if err != nil { return 0, errors.Wrap(err, "failed to interpret glob pattern", includePattern) } includeGlobPatterns = append(includeGlobPatterns, globExpression) } excludeGlobPatterns := make([]glob.Glob, 0, len(s3Lister.Excludes)) for _, excludePattern := range s3Lister.Excludes { globExpression, err := glob.Compile(excludePattern, '/') if err != nil { return 0, errors.Wrap(err, "failed to interpret glob pattern", excludePattern) } excludeGlobPatterns = append(excludeGlobPatterns, globExpression) } var numResults int64 for _, s3Path := range s3Lister.S3Paths { bucket, key, err := aws.SplitS3Path(s3Path) if err != nil { return 0, err } awsClientForBucket, err := aws.NewFromClientS3Path(s3Path, config.AWS) if err != nil { return 0, err } err = awsClientForBucket.S3Iterator(bucket, key, false, nil, nil, func(s3Obj *s3.Object) (bool, error) { s3FilePath := aws.S3Path(bucket, *s3Obj.Key) shouldSkip := false if len(includeGlobPatterns) > 0 { shouldSkip = true for _, includeGlobPattern := range includeGlobPatterns { if includeGlobPattern.Match(s3FilePath) { shouldSkip = false break } } } for _, excludeGlobPattern := range excludeGlobPatterns { if excludeGlobPattern.Match(s3FilePath) { shouldSkip = true break } } if !shouldSkip { shouldContinue, err := fn(bucket, s3Obj) numResults++ if s3Lister.MaxResults != nil && numResults >= *s3Lister.MaxResults { shouldContinue = false } return shouldContinue, err } return true, nil }) if err != nil { return 0, err } if s3Lister.MaxResults != nil && numResults >= *s3Lister.MaxResults { return numResults, nil } } return numResults, nil } ================================================ FILE: pkg/operator/resources/job/batchapi/validations.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package batchapi import ( "fmt" "github.com/aws/aws-sdk-go/service/s3" "github.com/cortexlabs/cortex/pkg/consts" awslib "github.com/cortexlabs/cortex/pkg/lib/aws" cr "github.com/cortexlabs/cortex/pkg/lib/configreader" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/pointer" "github.com/cortexlabs/cortex/pkg/operator/resources/job" "github.com/cortexlabs/cortex/pkg/operator/schema" "github.com/gobwas/glob" ) const ( _messageSizeLimit = 250 * 1024 // normally its 256 * 1024 but reserve 6k for message attributes ) func validateJobSubmissionSchema(submission *schema.BatchJobSubmission) error { providedKeys := []string{} if submission.ItemList != nil { providedKeys = append(providedKeys, schema.ItemListKey) } if submission.FilePathLister != nil { providedKeys = append(providedKeys, schema.FilePathListerKey) } if submission.DelimitedFiles != nil { providedKeys = append(providedKeys, schema.DelimitedFilesKey) } if len(providedKeys) == 0 { return job.ErrorSpecifyExactlyOneKey(schema.ItemListKey, schema.FilePathListerKey, schema.DelimitedFilesKey) } if len(providedKeys) > 1 { return job.ErrorConflictingFields(providedKeys[0], providedKeys[1:]...) } if submission.ItemList != nil { if len(submission.ItemList.Items) == 0 { return errors.Wrap(cr.ErrorTooFewElements(1), schema.ItemsKey) } for i, batch := range submission.ItemList.Items { if len(batch) > _messageSizeLimit { return ErrorItemSizeExceedsLimit(i, len(batch), _messageSizeLimit) } } if submission.ItemList.BatchSize < 1 { return errors.Wrap(cr.ErrorMustBeGreaterThanOrEqualTo(submission.ItemList.BatchSize, 1), schema.ItemListKey, schema.BatchSizeKey) } } if submission.FilePathLister != nil { if submission.FilePathLister.BatchSize < 1 { return errors.Wrap(cr.ErrorMustBeGreaterThanOrEqualTo(submission.FilePathLister.BatchSize, 1), schema.FilePathListerKey, schema.BatchSizeKey) } } if submission.DelimitedFiles != nil { if submission.DelimitedFiles.BatchSize < 1 { return errors.Wrap(cr.ErrorMustBeGreaterThanOrEqualTo(submission.DelimitedFiles.BatchSize, 1), schema.DelimitedFilesKey, schema.BatchSizeKey) } } if submission.Workers <= 0 { return errors.Wrap(cr.ErrorMustBeGreaterThanOrEqualTo(submission.Workers, 1), schema.WorkersKey) } if submission.Timeout != nil && *submission.Timeout <= 0 { return errors.Wrap(cr.ErrorMustBeGreaterThanOrEqualTo(submission.Timeout, 1), schema.TimeoutKey) } if submission.SQSDeadLetterQueue != nil { if len(submission.SQSDeadLetterQueue.ARN) == 0 { return errors.Wrap(cr.ErrorCannotBeEmpty(), schema.SQSDeadLetterQueueKey, schema.ARNKey) } if submission.SQSDeadLetterQueue.MaxReceiveCount < 1 { return errors.Wrap(cr.ErrorMustBeGreaterThanOrEqualTo(submission.SQSDeadLetterQueue.MaxReceiveCount, 1), schema.SQSDeadLetterQueueKey, schema.MaxReceiveCountKey) } } return nil } func validateJobSubmission(submission *schema.BatchJobSubmission) error { err := validateJobSubmissionSchema(submission) if err != nil { return errors.Append(err, fmt.Sprintf("\n\njob submission schema can be found at https://docs.cortexlabs.com/v/%s/", consts.CortexVersionMinor)) } if submission.FilePathLister != nil { err := validateS3Lister(&submission.FilePathLister.S3Lister) if err != nil { return errors.Wrap(err, schema.FilePathListerKey) } } if submission.DelimitedFiles != nil { err := validateS3Lister(&submission.DelimitedFiles.S3Lister) if err != nil { return errors.Wrap(err, schema.DelimitedFilesKey) } } return nil } func validateS3Lister(s3Lister *schema.S3Lister) error { if len(s3Lister.S3Paths) == 0 { return errors.Wrap(cr.ErrorTooFewElements(1), schema.S3PathsKey) } for _, globPattern := range s3Lister.Includes { _, err := glob.Compile(globPattern, '/') if err != nil { return errors.Wrap(err, schema.IncludesKey, globPattern) } } for _, globPattern := range s3Lister.Excludes { _, err := glob.Compile(globPattern, '/') if err != nil { return errors.Wrap(err, schema.ExcludesKey, globPattern) } } for _, s3Path := range s3Lister.S3Paths { if !awslib.IsValidS3Path(s3Path) { return awslib.ErrorInvalidS3Path(s3Path) } } shortCircuitLister := schema.S3Lister{ S3Paths: s3Lister.S3Paths, Includes: s3Lister.Includes, Excludes: s3Lister.Excludes, MaxResults: pointer.Int64(1), } numResults, err := s3IteratorFromLister(shortCircuitLister, func(objPath string, s3Obj *s3.Object) (bool, error) { return false, nil }) if err != nil { return err } if numResults == 0 { return ErrorNoS3FilesFound() } return nil } func listFilesDryRun(s3Lister *schema.S3Lister) ([]string, error) { var s3Files []string for _, s3Path := range s3Lister.S3Paths { if !awslib.IsValidS3Path(s3Path) { return nil, awslib.ErrorInvalidS3Path(s3Path) } } _, err := s3IteratorFromLister(*s3Lister, func(bucket string, s3Obj *s3.Object) (bool, error) { s3Files = append(s3Files, awslib.S3Path(bucket, *s3Obj.Key)) return true, nil }) if err != nil { return nil, err } if len(s3Files) == 0 { return nil, ErrorNoS3FilesFound() } return s3Files, nil } ================================================ FILE: pkg/operator/resources/job/cache.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package job import ( "path" "strings" "github.com/cortexlabs/cortex/pkg/config" "github.com/cortexlabs/cortex/pkg/types/spec" "github.com/cortexlabs/cortex/pkg/types/userconfig" ) func ListAllInProgressJobKeysByAPI(kind userconfig.Kind, apiName string) ([]spec.JobKey, error) { return listAllInProgressJobKeysByAPI(kind, &apiName) } func ListAllInProgressJobKeys(kind userconfig.Kind) ([]spec.JobKey, error) { return listAllInProgressJobKeysByAPI(kind, nil) } func DeleteInProgressFile(jobKey spec.JobKey) error { err := config.AWS.DeleteS3File(config.ClusterConfig.Bucket, inProgressKey(jobKey)) if err != nil { return err } return nil } func DeleteAllInProgressFilesByAPI(kind userconfig.Kind, apiName string) error { err := config.AWS.DeleteS3Prefix(config.ClusterConfig.Bucket, allInProgressForAPIKey(kind, apiName), true) if err != nil { return err } return nil } func listAllInProgressJobKeysByAPI(kind userconfig.Kind, apiName *string) ([]spec.JobKey, error) { _, ok := _jobKinds[kind] if !ok { return nil, ErrorInvalidJobKind(kind) } var jobPath string if apiName != nil { jobPath = allInProgressForAPIKey(kind, *apiName) } else { jobPath = allInProgressKey(kind) } s3Objects, err := config.AWS.ListS3Dir(config.ClusterConfig.Bucket, jobPath, false, nil, nil) if err != nil { return nil, err } jobKeys := make([]spec.JobKey, 0, len(s3Objects)) for _, obj := range s3Objects { if obj != nil { jobKeys = append(jobKeys, jobKeyFromInProgressKey(*obj.Key)) } } return jobKeys, nil } func uploadInProgressFile(jobKey spec.JobKey) error { err := config.AWS.UploadStringToS3("", config.ClusterConfig.Bucket, inProgressKey(jobKey)) if err != nil { return err } return nil } // e.g. /jobs//in_progress func allInProgressKey(kind userconfig.Kind) string { return path.Join( config.ClusterConfig.ClusterUID, _jobsPrefix, kind.String(), _inProgressFilePrefix, ) } // e.g. /jobs//in_progress/ func allInProgressForAPIKey(kind userconfig.Kind, apiName string) string { return path.Join(allInProgressKey(kind), apiName) } // e.g. /jobs//in_progress// func inProgressKey(jobKey spec.JobKey) string { return path.Join(allInProgressForAPIKey(jobKey.Kind, jobKey.APIName), jobKey.ID) } func jobKeyFromInProgressKey(s3Key string) spec.JobKey { pathSplit := strings.Split(s3Key, "/") kind := pathSplit[len(pathSplit)-4] apiName := pathSplit[len(pathSplit)-2] jobID := pathSplit[len(pathSplit)-1] return spec.JobKey{APIName: apiName, ID: jobID, Kind: userconfig.KindFromString(kind)} } ================================================ FILE: pkg/operator/resources/job/consts.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package job import "github.com/cortexlabs/cortex/pkg/types/userconfig" const ( _jobsPrefix = "jobs" _inProgressFilePrefix = "in_progress" _enqueuingLivenessFile = "enqueuing_liveness" ) var _jobKinds = map[userconfig.Kind]bool{ userconfig.TaskAPIKind: true, userconfig.BatchAPIKind: true, } func LivenessFile() string { return _enqueuingLivenessFile } ================================================ FILE: pkg/operator/resources/job/errors.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package job import ( "fmt" "github.com/cortexlabs/cortex/pkg/lib/errors" s "github.com/cortexlabs/cortex/pkg/lib/strings" "github.com/cortexlabs/cortex/pkg/types/spec" "github.com/cortexlabs/cortex/pkg/types/userconfig" ) const ( ErrInvalidJobKind = "job.invalid_kind" ErrJobNotFound = "job.not_found" ErrJobIsNotInProgress = "job.job_is_not_in_progress" ErrJobHasAlreadyBeenStopped = "job.job_has_already_been_stopped" ErrConflictingFields = "job.conflicting_fields" ErrSpecifyExactlyOneKey = "job.specify_exactly_one_key" ) func ErrorInvalidJobKind(kind userconfig.Kind) error { return errors.WithStack(&errors.Error{ Kind: ErrInvalidJobKind, Message: fmt.Sprintf("invalid job kind %s", kind.String()), }) } func ErrorJobNotFound(jobKey spec.JobKey) error { return errors.WithStack(&errors.Error{ Kind: ErrJobNotFound, Message: fmt.Sprintf("unable to find %s job %s", jobKey.Kind.String(), jobKey.UserString()), }) } func ErrorJobIsNotInProgress(kind userconfig.Kind) error { return errors.WithStack(&errors.Error{ Kind: ErrJobIsNotInProgress, Message: fmt.Sprintf("cannot stop %s job because it is not in progress", kind.String()), }) } func ErrorJobHasAlreadyBeenStopped(kind userconfig.Kind) error { return errors.WithStack(&errors.Error{ Kind: ErrJobHasAlreadyBeenStopped, Message: fmt.Sprintf("%s job has already been stopped", kind.String()), }) } func ErrorConflictingFields(key string, keys ...string) error { allKeys := append([]string{key}, keys...) return errors.WithStack(&errors.Error{ Kind: ErrConflictingFields, Message: fmt.Sprintf("please specify either the %s field (but not more than one at the same time)", s.StrsOr(allKeys)), }) } func ErrorSpecifyExactlyOneKey(key string, keys ...string) error { allKeys := append([]string{key}, keys...) return errors.WithStack(&errors.Error{ Kind: ErrSpecifyExactlyOneKey, Message: fmt.Sprintf("specify exactly one of the following keys: %s", s.StrsOr(allKeys)), }) } ================================================ FILE: pkg/operator/resources/job/state.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package job import ( "path" "path/filepath" "time" "github.com/cortexlabs/cortex/pkg/config" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/pointer" "github.com/cortexlabs/cortex/pkg/lib/strings" "github.com/cortexlabs/cortex/pkg/types/spec" "github.com/cortexlabs/cortex/pkg/types/status" "github.com/cortexlabs/cortex/pkg/types/userconfig" ) const ( _averageFilesPerJobState = 10 ) type State struct { spec.JobKey Status status.JobCode LastUpdatedMap map[string]time.Time EndTime *time.Time } func (j State) GetLastUpdated() time.Time { lastUpdated := time.Time{} for _, fileLastUpdated := range j.LastUpdatedMap { if lastUpdated.After(fileLastUpdated) { lastUpdated = fileLastUpdated } } return lastUpdated } func (j State) GetFirstCreated() time.Time { firstCreated := time.Unix(1<<63-62135596801, 999999999) // Max time for _, fileLastUpdated := range j.LastUpdatedMap { if firstCreated.After(fileLastUpdated) { firstCreated = fileLastUpdated } } return firstCreated } // Doesn't assume only status files are present. The order below matters. func GetTaskStatusCode(lastUpdatedMap map[string]time.Time) status.JobCode { if _, ok := lastUpdatedMap[status.JobStopped.String()]; ok { return status.JobStopped } if _, ok := lastUpdatedMap[status.JobTimedOut.String()]; ok { return status.JobTimedOut } if _, ok := lastUpdatedMap[status.JobWorkerOOM.String()]; ok { return status.JobWorkerOOM } if _, ok := lastUpdatedMap[status.JobWorkerError.String()]; ok { return status.JobWorkerError } if _, ok := lastUpdatedMap[status.JobEnqueueFailed.String()]; ok { return status.JobEnqueueFailed } if _, ok := lastUpdatedMap[status.JobUnexpectedError.String()]; ok { return status.JobUnexpectedError } if _, ok := lastUpdatedMap[status.JobCompletedWithFailures.String()]; ok { return status.JobCompletedWithFailures } if _, ok := lastUpdatedMap[status.JobSucceeded.String()]; ok { return status.JobSucceeded } if _, ok := lastUpdatedMap[status.JobRunning.String()]; ok { return status.JobRunning } if _, ok := lastUpdatedMap[status.JobEnqueuing.String()]; ok { return status.JobEnqueuing } if _, ok := lastUpdatedMap[status.JobPending.String()]; ok { return status.JobPending } return status.JobUnknown } func GetBatchStatusCode(lastUpdatedMap map[string]time.Time) status.JobCode { if _, ok := lastUpdatedMap[status.JobTimedOut.String()]; ok { return status.JobTimedOut } if _, ok := lastUpdatedMap[status.JobWorkerOOM.String()]; ok { return status.JobWorkerOOM } if _, ok := lastUpdatedMap[status.JobWorkerError.String()]; ok { return status.JobWorkerError } if _, ok := lastUpdatedMap[status.JobEnqueueFailed.String()]; ok { return status.JobEnqueueFailed } if _, ok := lastUpdatedMap[status.JobUnexpectedError.String()]; ok { return status.JobUnexpectedError } if _, ok := lastUpdatedMap[status.JobCompletedWithFailures.String()]; ok { return status.JobCompletedWithFailures } if _, ok := lastUpdatedMap[status.JobSucceeded.String()]; ok { return status.JobSucceeded } if _, ok := lastUpdatedMap[status.JobStopped.String()]; ok { return status.JobStopped } if _, ok := lastUpdatedMap[status.JobRunning.String()]; ok { return status.JobRunning } if _, ok := lastUpdatedMap[status.JobEnqueuing.String()]; ok { return status.JobEnqueuing } if _, ok := lastUpdatedMap[status.JobPending.String()]; ok { return status.JobPending } return status.JobUnknown } func GetJobState(jobKey spec.JobKey) (*State, error) { s3Objects, err := config.AWS.ListS3Prefix(config.ClusterConfig.Bucket, jobKey.Prefix(config.ClusterConfig.ClusterUID), false, nil, nil) if err != nil { return nil, errors.Wrap(err, "failed to get job state", jobKey.UserString()) } if len(s3Objects) == 0 { return nil, errors.Wrap(ErrorJobNotFound(jobKey), "failed to get job state") } lastUpdatedMap := map[string]time.Time{} for _, object := range s3Objects { lastUpdatedMap[filepath.Base(*object.Key)] = *object.LastModified } jobState := getJobStateFromFiles(jobKey, lastUpdatedMap) return &jobState, nil } func getJobStateFromFiles(jobKey spec.JobKey, lastUpdatedFileMap map[string]time.Time) State { var statusCode status.JobCode switch jobKey.Kind { case userconfig.BatchAPIKind: statusCode = GetBatchStatusCode(lastUpdatedFileMap) case userconfig.TaskAPIKind: statusCode = GetTaskStatusCode(lastUpdatedFileMap) } var jobEndTime *time.Time if statusCode.IsCompleted() { if endTime, ok := lastUpdatedFileMap[statusCode.String()]; ok { jobEndTime = &endTime } } return State{ JobKey: jobKey, LastUpdatedMap: lastUpdatedFileMap, Status: statusCode, EndTime: jobEndTime, } } func GetMostRecentlySubmittedJobStates(apiName string, count int, kind userconfig.Kind) ([]*State, error) { // a single job state may include 5 files on average, overshoot the number of files needed apiPrefix := strings.EnsureSuffix(spec.JobAPIPrefix(config.ClusterConfig.ClusterUID, kind, apiName), "/") s3Objects, err := config.AWS.ListS3Prefix( config.ClusterConfig.Bucket, apiPrefix, false, pointer.Int64(int64(count*_averageFilesPerJobState)), nil, ) if err != nil { return nil, err } // job id -> file name -> last update timestamp lastUpdatedMaps := map[string]map[string]time.Time{} var jobIDOrder []string for _, object := range s3Objects { if object == nil { continue } fileName := filepath.Base(*object.Key) jobID := filepath.Base(filepath.Dir(*object.Key)) if _, ok := lastUpdatedMaps[jobID]; !ok { jobIDOrder = append(jobIDOrder, jobID) lastUpdatedMaps[jobID] = map[string]time.Time{fileName: *object.LastModified} } else { lastUpdatedMaps[jobID][fileName] = *object.LastModified } } jobStates := make([]*State, 0, count) jobStateCount := 0 for _, jobID := range jobIDOrder { // it is possible to have fragmented deletes, spec.json should always be there _, found := lastUpdatedMaps[jobID]["spec.json"] if !found { go config.AWS.DeleteS3Dir(config.ClusterConfig.Bucket, path.Join(apiPrefix, jobID), true) continue } jobState := getJobStateFromFiles(spec.JobKey{ APIName: apiName, ID: jobID, Kind: kind, }, lastUpdatedMaps[jobID]) jobStates = append(jobStates, &jobState) jobStateCount++ if jobStateCount == count { break } } return jobStates, nil } func SetStatusForJob(jobKey spec.JobKey, jobStatus status.JobCode) error { switch jobStatus { case status.JobEnqueuing: return SetEnqueuingStatus(jobKey) case status.JobRunning: return SetRunningStatus(jobKey) case status.JobEnqueueFailed: return SetEnqueueFailedStatus(jobKey) case status.JobCompletedWithFailures: return SetCompletedWithFailuresStatus(jobKey) case status.JobSucceeded: return SetSucceededStatus(jobKey) case status.JobUnexpectedError: return SetUnexpectedErrorStatus(jobKey) case status.JobWorkerError: return SetWorkerErrorStatus(jobKey) case status.JobWorkerOOM: return SetWorkerOOMStatus(jobKey) case status.JobTimedOut: return SetTimedOutStatus(jobKey) case status.JobStopped: return SetStoppedStatus(jobKey) } return nil } func UpdateLiveness(jobKey spec.JobKey) error { s3Key := path.Join(jobKey.Prefix(config.ClusterConfig.ClusterUID), _enqueuingLivenessFile) err := config.AWS.UploadJSONToS3(time.Now(), config.ClusterConfig.Bucket, s3Key) if err != nil { return errors.Wrap(err, "failed to update liveness", jobKey.UserString()) } return nil } func SetEnqueuingStatus(jobKey spec.JobKey) error { err := UpdateLiveness(jobKey) if err != nil { return err } err = config.AWS.UploadStringToS3("", config.ClusterConfig.Bucket, path.Join(jobKey.Prefix(config.ClusterConfig.ClusterUID), status.JobEnqueuing.String())) if err != nil { return err } err = uploadInProgressFile(jobKey) if err != nil { return err } return nil } func SetFailedStatus(jobKey spec.JobKey) error { err := config.AWS.UploadStringToS3("", config.ClusterConfig.Bucket, path.Join(jobKey.Prefix(config.ClusterConfig.ClusterUID), status.JobEnqueueFailed.String())) if err != nil { return err } err = DeleteInProgressFile(jobKey) if err != nil { return err } return nil } func SetRunningStatus(jobKey spec.JobKey) error { err := config.AWS.UploadStringToS3("", config.ClusterConfig.Bucket, path.Join(jobKey.Prefix(config.ClusterConfig.ClusterUID), status.JobRunning.String())) if err != nil { return err } err = uploadInProgressFile(jobKey) // in progress file should already be there but just in case if err != nil { return err } return nil } func SetStoppedStatus(jobKey spec.JobKey) error { err := config.AWS.UploadStringToS3("", config.ClusterConfig.Bucket, path.Join(jobKey.Prefix(config.ClusterConfig.ClusterUID), status.JobStopped.String())) if err != nil { return err } err = DeleteInProgressFile(jobKey) if err != nil { return err } return nil } func SetSucceededStatus(jobKey spec.JobKey) error { err := config.AWS.UploadStringToS3("", config.ClusterConfig.Bucket, path.Join(jobKey.Prefix(config.ClusterConfig.ClusterUID), status.JobSucceeded.String())) if err != nil { return err } err = DeleteInProgressFile(jobKey) if err != nil { return err } return nil } func SetCompletedWithFailuresStatus(jobKey spec.JobKey) error { err := config.AWS.UploadStringToS3("", config.ClusterConfig.Bucket, path.Join(jobKey.Prefix(config.ClusterConfig.ClusterUID), status.JobCompletedWithFailures.String())) if err != nil { return err } err = DeleteInProgressFile(jobKey) if err != nil { return err } return nil } func SetWorkerErrorStatus(jobKey spec.JobKey) error { err := config.AWS.UploadStringToS3("", config.ClusterConfig.Bucket, path.Join(jobKey.Prefix(config.ClusterConfig.ClusterUID), status.JobWorkerError.String())) if err != nil { return err } err = DeleteInProgressFile(jobKey) if err != nil { return err } return nil } func SetWorkerOOMStatus(jobKey spec.JobKey) error { err := config.AWS.UploadStringToS3("", config.ClusterConfig.Bucket, path.Join(jobKey.Prefix(config.ClusterConfig.ClusterUID), status.JobWorkerOOM.String())) if err != nil { return err } err = DeleteInProgressFile(jobKey) if err != nil { return err } return nil } func SetEnqueueFailedStatus(jobKey spec.JobKey) error { err := config.AWS.UploadStringToS3("", config.ClusterConfig.Bucket, path.Join(jobKey.Prefix(config.ClusterConfig.ClusterUID), status.JobEnqueueFailed.String())) if err != nil { return err } err = DeleteInProgressFile(jobKey) if err != nil { return err } return nil } func SetUnexpectedErrorStatus(jobKey spec.JobKey) error { err := config.AWS.UploadStringToS3("", config.ClusterConfig.Bucket, path.Join(jobKey.Prefix(config.ClusterConfig.ClusterUID), status.JobUnexpectedError.String())) if err != nil { return err } err = DeleteInProgressFile(jobKey) if err != nil { return err } return nil } func SetTimedOutStatus(jobKey spec.JobKey) error { err := config.AWS.UploadStringToS3("", config.ClusterConfig.Bucket, path.Join(jobKey.Prefix(config.ClusterConfig.ClusterUID), status.JobTimedOut.String())) if err != nil { return err } err = DeleteInProgressFile(jobKey) if err != nil { return err } return nil } ================================================ FILE: pkg/operator/resources/job/taskapi/api.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package taskapi import ( "fmt" "path/filepath" "time" "github.com/cortexlabs/cortex/pkg/config" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/k8s" "github.com/cortexlabs/cortex/pkg/lib/parallel" "github.com/cortexlabs/cortex/pkg/lib/pointer" "github.com/cortexlabs/cortex/pkg/lib/sets/strset" "github.com/cortexlabs/cortex/pkg/operator/lib/routines" "github.com/cortexlabs/cortex/pkg/operator/operator" "github.com/cortexlabs/cortex/pkg/operator/resources/job" "github.com/cortexlabs/cortex/pkg/operator/schema" "github.com/cortexlabs/cortex/pkg/types/spec" "github.com/cortexlabs/cortex/pkg/types/status" "github.com/cortexlabs/cortex/pkg/types/userconfig" "github.com/cortexlabs/cortex/pkg/workloads" istioclientnetworking "istio.io/client-go/pkg/apis/networking/v1beta1" kbatch "k8s.io/api/batch/v1" kcore "k8s.io/api/core/v1" ) const _taskDashboardUID = "taskapi" // UpdateAPI deploys or update a task api without triggering any task func UpdateAPI(apiConfig *userconfig.API) (*spec.API, string, error) { prevVirtualService, err := config.K8s.GetVirtualService(workloads.K8sName(apiConfig.Name)) if err != nil { return nil, "", err } initialDeploymentTime := time.Now().UnixNano() if prevVirtualService != nil && prevVirtualService.Labels["initialDeploymentTime"] != "" { var err error initialDeploymentTime, err = k8s.ParseInt64Label(prevVirtualService, "initialDeploymentTime") if err != nil { return nil, "", err } } api := spec.GetAPISpec(apiConfig, initialDeploymentTime, "", config.ClusterConfig.ClusterUID) // Deployment ID not needed for TaskAPI spec if prevVirtualService == nil { if err := config.AWS.UploadJSONToS3(api, config.ClusterConfig.Bucket, api.Key); err != nil { return nil, "", errors.Wrap(err, "upload api spec") } err = applyK8sResources(api, prevVirtualService) if err != nil { routines.RunWithPanicHandler(func() { deleteK8sResources(api.Name) }) return nil, "", err } return api, fmt.Sprintf("created %s", api.Resource.UserString()), nil } if prevVirtualService.Labels["specID"] != api.SpecID { if err := config.AWS.UploadJSONToS3(api, config.ClusterConfig.Bucket, api.Key); err != nil { return nil, "", errors.Wrap(err, "upload api spec") } err = applyK8sResources(api, prevVirtualService) if err != nil { return nil, "", err } return api, fmt.Sprintf("updated %s", api.Resource.UserString()), nil } return api, fmt.Sprintf("%s is up to date", api.Resource.UserString()), nil } // DeleteAPI deletes a task api func DeleteAPI(apiName string, keepCache bool) error { err := parallel.RunFirstErr( func() error { return deleteK8sResources(apiName) }, func() error { if keepCache { return nil } return deleteS3Resources(apiName) }, ) if err != nil { return err } return nil } func deleteS3Resources(apiName string) error { _ = job.DeleteAllInProgressFilesByAPI(userconfig.TaskAPIKind, apiName) // not useful xml error is thrown, swallow the error return parallel.RunFirstErr( func() error { prefix := filepath.Join(config.ClusterConfig.ClusterUID, "apis", apiName) return config.AWS.DeleteS3Dir(config.ClusterConfig.Bucket, prefix, true) }, func() error { prefix := spec.JobAPIPrefix(config.ClusterConfig.ClusterUID, userconfig.TaskAPIKind, apiName) go func() { _ = config.AWS.DeleteS3Dir(config.ClusterConfig.Bucket, prefix, true) // deleting job files may take a while }() return nil }, ) } // GetAllAPIs returns all task APIs, for each API returning the most recently submitted job and all running jobs func GetAllAPIs(virtualServices []istioclientnetworking.VirtualService, k8sJobs []kbatch.Job, pods []kcore.Pod) ([]schema.APIResponse, error) { taskAPIsMap := map[string]*schema.APIResponse{} jobIDToK8sJobMap := map[string]*kbatch.Job{} for i, kJob := range k8sJobs { jobIDToK8sJobMap[kJob.Labels["jobID"]] = &k8sJobs[i] } jobIDToPodsMap := map[string][]kcore.Pod{} for _, pod := range pods { if pod.Labels["jobID"] != "" { jobIDToPodsMap[pod.Labels["jobID"]] = append(jobIDToPodsMap[pod.Labels["jobID"]], pod) } } for i := range virtualServices { apiName := virtualServices[i].Labels["apiName"] metadata, err := spec.MetadataFromVirtualService(&virtualServices[i]) if err != nil { return nil, errors.Wrap(err, fmt.Sprintf("api %s", apiName)) } jobStates, err := job.GetMostRecentlySubmittedJobStates(metadata.Name, 1, userconfig.TaskAPIKind) jobStatuses := []status.TaskJobStatus{} if len(jobStates) > 0 { jobStatus, err := getJobStatusFromJobState(jobStates[0], jobIDToK8sJobMap[jobStates[0].ID], jobIDToPodsMap[jobStates[0].ID]) if err != nil { return nil, err } jobStatuses = append(jobStatuses, *jobStatus) } taskAPIsMap[metadata.Name] = &schema.APIResponse{ Metadata: metadata, TaskJobStatuses: jobStatuses, } } inProgressJobKeys, err := job.ListAllInProgressJobKeys(userconfig.TaskAPIKind) if err != nil { return nil, err } for _, jobKey := range inProgressJobKeys { alreadyAdded := false for _, jobStatus := range taskAPIsMap[jobKey.APIName].TaskJobStatuses { if jobStatus.ID == jobKey.ID { alreadyAdded = true break } } if alreadyAdded { continue } jobStatus, err := getJobStatusFromK8sJob(jobKey, jobIDToK8sJobMap[jobKey.ID], jobIDToPodsMap[jobKey.ID]) if err != nil { return nil, err } if jobStatus.Status.IsInProgress() { taskAPIsMap[jobKey.APIName].TaskJobStatuses = append(taskAPIsMap[jobKey.APIName].TaskJobStatuses, *jobStatus) } } taskAPIList := make([]schema.APIResponse, 0, len(taskAPIsMap)) for _, taskAPI := range taskAPIsMap { taskAPIList = append(taskAPIList, *taskAPI) } return taskAPIList, nil } // GetAPIByName returns a single task API and its most recently submitted job along with all running task jobs func GetAPIByName(deployedResource *operator.DeployedResource) ([]schema.APIResponse, error) { metadata, err := spec.MetadataFromVirtualService(deployedResource.VirtualService) if err != nil { return nil, err } api, err := operator.DownloadAPISpec(deployedResource.Name, metadata.APIID) if err != nil { return nil, err } k8sJobs, err := config.K8s.ListJobsByLabel("apiName", deployedResource.Name) if err != nil { return nil, err } jobIDToK8sJobMap := map[string]*kbatch.Job{} for i, kJob := range k8sJobs { jobIDToK8sJobMap[kJob.Labels["jobID"]] = &k8sJobs[i] } endpoint, err := operator.APIEndpoint(api) if err != nil { return nil, err } pods, err := config.K8s.ListPodsByLabel("apiName", deployedResource.Name) if err != nil { return nil, err } jobIDToPodsMap := map[string][]kcore.Pod{} for _, pod := range pods { jobIDToPodsMap[pod.Labels["jobID"]] = append(jobIDToPodsMap[pod.Labels["jobID"]], pod) } inProgressJobKeys, err := job.ListAllInProgressJobKeysByAPI(userconfig.TaskAPIKind, deployedResource.Name) if err != nil { return nil, err } jobStatuses := []status.TaskJobStatus{} jobIDSet := strset.New() for _, jobKey := range inProgressJobKeys { jobStatus, err := getJobStatusFromK8sJob(jobKey, jobIDToK8sJobMap[jobKey.ID], jobIDToPodsMap[jobKey.ID]) if err != nil { return nil, err } jobStatuses = append(jobStatuses, *jobStatus) jobIDSet.Add(jobKey.ID) } if len(jobStatuses) < 10 { jobStates, err := job.GetMostRecentlySubmittedJobStates(deployedResource.Name, 10+len(jobStatuses), userconfig.TaskAPIKind) if err != nil { return nil, err } for _, jobState := range jobStates { if jobIDSet.Has(jobState.ID) { continue } jobIDSet.Add(jobState.ID) jobStatus, err := getJobStatusFromJobState(jobState, nil, nil) if err != nil { return nil, err } jobStatuses = append(jobStatuses, *jobStatus) if len(jobStatuses) == 10 { break } } } dashboardURL := pointer.String(getDashboardURL(api.Name)) return []schema.APIResponse{ { Spec: api, Metadata: metadata, TaskJobStatuses: jobStatuses, Endpoint: &endpoint, DashboardURL: dashboardURL, }, }, nil } func getDashboardURL(apiName string) string { loadBalancerURL, err := operator.LoadBalancerURL() if err != nil { return "" } dashboardURL := fmt.Sprintf( "%s/dashboard/d/%s/taskapi?orgId=1&refresh=30s&var-api_name=%s", loadBalancerURL, _taskDashboardUID, apiName, ) return dashboardURL } ================================================ FILE: pkg/operator/resources/job/taskapi/cron.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package taskapi import ( "fmt" "time" "github.com/cortexlabs/cortex/pkg/config" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/k8s" "github.com/cortexlabs/cortex/pkg/lib/logging" "github.com/cortexlabs/cortex/pkg/lib/sets/strset" "github.com/cortexlabs/cortex/pkg/lib/telemetry" "github.com/cortexlabs/cortex/pkg/operator/operator" "github.com/cortexlabs/cortex/pkg/operator/resources/job" "github.com/cortexlabs/cortex/pkg/types/spec" "github.com/cortexlabs/cortex/pkg/types/status" "github.com/cortexlabs/cortex/pkg/types/userconfig" kbatch "k8s.io/api/batch/v1" kmeta "k8s.io/apimachinery/pkg/apis/meta/v1" klabels "k8s.io/apimachinery/pkg/labels" ) const ( ManageJobResourcesCronPeriod = 60 * time.Second _k8sJobExistenceGracePeriod = 10 * time.Second ) var operatorLogger = logging.GetLogger() var _inProgressJobSpecMap = map[string]*spec.TaskJob{} func ManageJobResources() error { inProgressJobKeys, err := job.ListAllInProgressJobKeys(userconfig.TaskAPIKind) if err != nil { return err } inProgressJobIDSet := strset.Set{} for _, jobKey := range inProgressJobKeys { inProgressJobIDSet.Add(jobKey.ID) } for jobID := range _inProgressJobSpecMap { if !inProgressJobIDSet.Has(jobID) { delete(_inProgressJobSpecMap, jobID) } } jobs, err := config.K8s.ListJobs( &kmeta.ListOptions{ LabelSelector: klabels.SelectorFromSet( map[string]string{"apiKind": userconfig.TaskAPIKind.String()}, ).String(), }, ) if err != nil { return err } k8sJobMap := map[string]kbatch.Job{} k8sJobIDSet := strset.Set{} for _, kJob := range jobs { k8sJobMap[kJob.Labels["jobID"]] = kJob k8sJobIDSet.Add(kJob.Labels["jobID"]) } for _, jobKey := range inProgressJobKeys { jobLogger, err := operator.GetJobLogger(jobKey) if err != nil { telemetry.Error(err) operatorLogger.Error(err) continue } k8sJob, jobFound := k8sJobMap[jobKey.ID] jobState, err := job.GetJobState(jobKey) if err != nil { jobLogger.Error(err) jobLogger.Error("terminating job and cleaning up job resources") err := errors.FirstError( job.DeleteInProgressFile(jobKey), deleteJobRuntimeResources(jobKey), recordFailure(jobKey), ) if err != nil { telemetry.Error(err) operatorLogger.Error(err) } continue } if !jobState.Status.IsInProgress() { // best effort cleanup _ = job.DeleteInProgressFile(jobKey) _ = deleteJobRuntimeResources(jobKey) continue } // reconcile job state and k8s job newStatusCode, msg := reconcileInProgressJob(jobState, jobFound) if err != nil { telemetry.Error(err) operatorLogger.Error(err) continue } if newStatusCode != jobState.Status { jobLogger.Error(msg) err := job.SetStatusForJob(jobKey, newStatusCode) if err != nil { telemetry.Error(err) operatorLogger.Error(err) continue } } if _, ok := _inProgressJobSpecMap[jobKey.ID]; !ok { jobSpec, err := operator.DownloadTaskJobSpec(jobKey) if err != nil { jobLogger.Error(err) jobLogger.Error("terminating job and cleaning up job resources") err := errors.FirstError( job.DeleteInProgressFile(jobKey), deleteJobRuntimeResources(jobKey), recordFailure(jobKey), ) if err != nil { telemetry.Error(err) operatorLogger.Error(err) } continue } _inProgressJobSpecMap[jobKey.ID] = jobSpec } jobSpec := _inProgressJobSpecMap[jobKey.ID] if jobSpec.Timeout != nil && time.Since(jobSpec.StartTime) > time.Second*time.Duration(*jobSpec.Timeout) { jobLogger.Errorf("terminating job after exceeding the specified timeout of %d seconds", *jobSpec.Timeout) err := errors.FirstError( job.SetTimedOutStatus(jobKey), deleteJobRuntimeResources(jobKey), recordFailure(jobKey), ) if err != nil { telemetry.Error(err) operatorLogger.Error(err) } continue } if jobState.Status == status.JobRunning { err = checkIfJobCompleted(jobKey, jobSpec.StartTime, k8sJob) if err != nil { telemetry.Error(err) operatorLogger.Error(err) } } } // existing K8s job but job is not in progress for jobID := range strset.Difference(k8sJobIDSet, inProgressJobIDSet) { jobKey := spec.JobKey{ APIName: k8sJobMap[jobID].Labels["apiName"], ID: k8sJobMap[jobID].Labels["jobID"], } err := deleteJobRuntimeResources(jobKey) if err != nil { telemetry.Error(err) operatorLogger.Error(err) } } return nil } // verifies k8s job exists for a job in running status, if verification fails return a job code to reflect the state func reconcileInProgressJob(jobState *job.State, jobFound bool) (status.JobCode, string) { if jobState.Status == status.JobRunning { if time.Since(jobState.LastUpdatedMap[status.JobRunning.String()]) <= _k8sJobExistenceGracePeriod { return jobState.Status, "" } if !jobFound { // unexpected k8s job missing return status.JobUnexpectedError, fmt.Sprintf("terminating job %s; unable to find kubernetes job", jobState.JobKey.UserString()) } } return jobState.Status, "" } func checkIfJobCompleted(jobKey spec.JobKey, jobStartTime time.Time, k8sJob kbatch.Job) error { pods, _ := config.K8s.ListPodsByLabel("jobID", jobKey.ID) for i := range pods { if k8s.WasPodOOMKilled(&pods[i]) { return errors.FirstError( job.SetWorkerOOMStatus(jobKey), deleteJobRuntimeResources(jobKey), recordFailure(jobKey), ) } } if int(k8sJob.Status.Failed) == 1 { return errors.FirstError( job.SetWorkerErrorStatus(jobKey), deleteJobRuntimeResources(jobKey), recordFailure(jobKey), ) } else if int(k8sJob.Status.Succeeded) == 1 && len(pods) > 0 { return errors.FirstError( job.SetSucceededStatus(jobKey), deleteJobRuntimeResources(jobKey), recordSuccess(jobKey), recordTimePerTask(jobKey, time.Since(jobStartTime)), ) } return nil } ================================================ FILE: pkg/operator/resources/job/taskapi/job.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package taskapi import ( "time" "github.com/cortexlabs/cortex/pkg/config" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/telemetry" "github.com/cortexlabs/cortex/pkg/operator/lib/routines" "github.com/cortexlabs/cortex/pkg/operator/operator" "github.com/cortexlabs/cortex/pkg/operator/resources/job" "github.com/cortexlabs/cortex/pkg/operator/schema" "github.com/cortexlabs/cortex/pkg/types/spec" "github.com/cortexlabs/cortex/pkg/workloads" ) func SubmitJob(apiName string, submission *schema.TaskJobSubmission) (*spec.TaskJob, error) { err := validateJobSubmission(submission) if err != nil { return nil, err } virtualService, err := config.K8s.GetVirtualService(workloads.K8sName(apiName)) if err != nil { return nil, err } apiID := virtualService.Labels["apiID"] apiSpec, err := operator.DownloadAPISpec(apiName, apiID) if err != nil { return nil, err } jobID := spec.MonotonicallyDecreasingID() jobKey := spec.JobKey{ APIName: apiSpec.Name, ID: jobID, Kind: apiSpec.Kind, } jobSpec := spec.TaskJob{ JobKey: jobKey, RuntimeTaskJobConfig: submission.RuntimeTaskJobConfig, APIID: apiSpec.ID, SpecID: apiSpec.SpecID, PodID: apiSpec.PodID, StartTime: time.Now(), } if err := uploadJobSpec(&jobSpec); err != nil { return nil, err } deployJob(apiSpec, &jobSpec) return &jobSpec, nil } func uploadJobSpec(jobSpec *spec.TaskJob) error { if err := config.AWS.UploadJSONToS3( jobSpec, config.ClusterConfig.Bucket, jobSpec.SpecFilePath(config.ClusterConfig.ClusterUID), ); err != nil { return err } return nil } func deployJob(apiSpec *spec.API, jobSpec *spec.TaskJob) { err := createJobConfigMap(*apiSpec, *jobSpec) if err != nil { handleJobSubmissionError(jobSpec.JobKey, err) } err = createK8sJob(apiSpec, jobSpec) if err != nil { handleJobSubmissionError(jobSpec.JobKey, err) } err = job.SetRunningStatus(jobSpec.JobKey) if err != nil { handleJobSubmissionError(jobSpec.JobKey, err) } } func createJobConfigMap(apiSpec spec.API, jobSpec spec.TaskJob) error { configMapConfig := workloads.ConfigMapConfig{ TaskJob: &jobSpec, } configMapData, err := configMapConfig.GenerateConfigMapData() if err != nil { return err } return createK8sConfigMap(k8sConfigMap(apiSpec, jobSpec, configMapData)) } func handleJobSubmissionError(jobKey spec.JobKey, jobErr error) { jobLogger, err := operator.GetJobLogger(jobKey) if err != nil { telemetry.Error(err) operatorLogger.Error(err) return } jobLogger.Error(jobErr.Error()) err = errors.FirstError( job.SetUnexpectedErrorStatus(jobKey), deleteJobRuntimeResources(jobKey), ) if err != nil { telemetry.Error(err) errors.PrintError(err) } } func deleteJobRuntimeResources(jobKey spec.JobKey) error { return errors.FirstError( deleteK8sJob(jobKey), deleteK8sConfigMap(jobKey), ) } func StopJob(jobKey spec.JobKey) error { jobState, err := job.GetJobState(jobKey) if err != nil { routines.RunWithPanicHandler(func() { deleteJobRuntimeResources(jobKey) }) return err } if !jobState.Status.IsInProgress() { routines.RunWithPanicHandler(func() { deleteJobRuntimeResources(jobKey) }) return errors.Wrap(job.ErrorJobIsNotInProgress(jobKey.Kind), jobKey.UserString()) } jobLogger, err := operator.GetJobLogger(jobKey) if err == nil { jobLogger.Warn("request received to stop job; performing cleanup...") } return errors.FirstError( deleteJobRuntimeResources(jobKey), job.SetStoppedStatus(jobKey), ) } ================================================ FILE: pkg/operator/resources/job/taskapi/job_status.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package taskapi import ( "github.com/cortexlabs/cortex/pkg/config" "github.com/cortexlabs/cortex/pkg/operator/operator" "github.com/cortexlabs/cortex/pkg/operator/resources/job" "github.com/cortexlabs/cortex/pkg/types/spec" "github.com/cortexlabs/cortex/pkg/types/status" kbatch "k8s.io/api/batch/v1" kcore "k8s.io/api/core/v1" ) func GetJobStatus(jobKey spec.JobKey) (*status.TaskJobStatus, error) { jobState, err := job.GetJobState(jobKey) if err != nil { return nil, err } k8sJob, err := config.K8s.GetJob(jobKey.K8sName()) if err != nil { return nil, err } pods, err := config.K8s.ListPodsByLabels(map[string]string{"apiName": jobKey.APIName, "jobID": jobKey.ID}) if err != nil { return nil, err } return getJobStatusFromJobState(jobState, k8sJob, pods) } func getJobStatusFromK8sJob(jobKey spec.JobKey, k8sJob *kbatch.Job, pods []kcore.Pod) (*status.TaskJobStatus, error) { jobState, err := job.GetJobState(jobKey) if err != nil { return nil, err } return getJobStatusFromJobState(jobState, k8sJob, pods) } func getJobStatusFromJobState(jobState *job.State, k8sJob *kbatch.Job, pods []kcore.Pod) (*status.TaskJobStatus, error) { jobKey := jobState.JobKey jobSpec, err := operator.DownloadTaskJobSpec(jobKey) if err != nil { return nil, err } jobStatus := status.TaskJobStatus{ TaskJob: *jobSpec, EndTime: jobState.EndTime, Status: jobState.Status, } if jobState.Status.IsInProgress() && k8sJob != nil { workerCounts := job.GetWorkerCountsForJob(*k8sJob, pods) jobStatus.WorkerCounts = &workerCounts } return &jobStatus, nil } ================================================ FILE: pkg/operator/resources/job/taskapi/k8s_specs.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package taskapi import ( "path" "github.com/cortexlabs/cortex/pkg/config" "github.com/cortexlabs/cortex/pkg/consts" "github.com/cortexlabs/cortex/pkg/lib/k8s" "github.com/cortexlabs/cortex/pkg/lib/parallel" "github.com/cortexlabs/cortex/pkg/lib/pointer" s "github.com/cortexlabs/cortex/pkg/lib/strings" "github.com/cortexlabs/cortex/pkg/types/spec" "github.com/cortexlabs/cortex/pkg/types/userconfig" "github.com/cortexlabs/cortex/pkg/workloads" istioclientnetworking "istio.io/client-go/pkg/apis/networking/v1beta1" kbatch "k8s.io/api/batch/v1" kcore "k8s.io/api/core/v1" kmeta "k8s.io/apimachinery/pkg/apis/meta/v1" klabels "k8s.io/apimachinery/pkg/labels" ) const _operatorService = "operator" func virtualServiceSpec(api *spec.API) *istioclientnetworking.VirtualService { return k8s.VirtualService(&k8s.VirtualServiceSpec{ Name: workloads.K8sName(api.Name), Gateways: []string{"apis-gateway"}, Destinations: []k8s.Destination{{ ServiceName: _operatorService, Weight: 100, Port: uint32(consts.ProxyPortInt32), }}, PrefixPath: api.Networking.Endpoint, Rewrite: pointer.String(path.Join("tasks", api.Name)), Annotations: api.ToK8sAnnotations(), Labels: map[string]string{ "apiName": api.Name, "apiID": api.ID, "specID": api.SpecID, "podID": api.PodID, "initialDeploymentTime": s.Int64(api.InitialDeploymentTime), "apiKind": api.Kind.String(), "cortex.dev/api": "true", }, }) } func k8sJobSpec(api *spec.API, job *spec.TaskJob) *kbatch.Job { containers, volumes := workloads.TaskContainers(*api, &job.JobKey) return k8s.Job(&k8s.JobSpec{ Name: job.JobKey.K8sName(), Parallelism: int32(job.Workers), Labels: map[string]string{ "apiName": api.Name, "apiID": api.ID, "specID": api.SpecID, "podID": api.PodID, "jobID": job.ID, "apiKind": api.Kind.String(), "cortex.dev/api": "true", }, PodSpec: k8s.PodSpec{ Labels: map[string]string{ "apiName": api.Name, "podID": api.PodID, "jobID": job.ID, "apiKind": api.Kind.String(), "cortex.dev/api": "true", }, Annotations: map[string]string{ "traffic.sidecar.istio.io/excludeOutboundIPRanges": "0.0.0.0/0", "cluster-autoscaler.kubernetes.io/safe-to-evict": "false", }, K8sPodSpec: kcore.PodSpec{ RestartPolicy: "Never", InitContainers: []kcore.Container{ workloads.KubexitInitContainer(), }, Containers: containers, NodeSelector: workloads.NodeSelectors(), Tolerations: workloads.GenerateResourceTolerations(), Affinity: workloads.GenerateNodeAffinities(api.NodeGroups), Volumes: volumes, ServiceAccountName: workloads.ServiceAccountName, }, }, }) } func k8sConfigMap(api spec.API, job spec.TaskJob, configMapData map[string]string) kcore.ConfigMap { return *k8s.ConfigMap(&k8s.ConfigMapSpec{ Name: job.JobKey.K8sName(), Data: configMapData, Labels: map[string]string{ "apiName": api.Name, "apiKind": api.Kind.String(), "cortex.dev/api": "true", }, }) } func applyK8sResources(api *spec.API, prevVirtualService *istioclientnetworking.VirtualService) error { newVirtualService := virtualServiceSpec(api) if prevVirtualService == nil { _, err := config.K8s.CreateVirtualService(newVirtualService) return err } _, err := config.K8s.UpdateVirtualService(prevVirtualService, newVirtualService) return err } func deleteK8sResources(apiName string) error { return parallel.RunFirstErr( func() error { _, err := config.K8s.DeleteJobs(&kmeta.ListOptions{ LabelSelector: klabels.SelectorFromSet( map[string]string{ "apiName": apiName, "apiKind": userconfig.TaskAPIKind.String(), }).String(), }) return err }, func() error { _, err := config.K8s.DeleteVirtualService(workloads.K8sName(apiName)) return err }, ) } func deleteK8sJob(jobKey spec.JobKey) error { _, err := config.K8s.DeleteJobs(&kmeta.ListOptions{ LabelSelector: klabels.SelectorFromSet( map[string]string{ "apiName": jobKey.APIName, "apiKind": userconfig.TaskAPIKind.String(), "jobID": jobKey.ID, }).String(), }) return err } func createK8sJob(apiSpec *spec.API, jobSpec *spec.TaskJob) error { k8sJob := k8sJobSpec(apiSpec, jobSpec) _, err := config.K8s.CreateJob(k8sJob) if err != nil { return err } return nil } func deleteK8sConfigMap(jobKey spec.JobKey) error { _, err := config.K8s.DeleteConfigMap(jobKey.K8sName()) return err } func createK8sConfigMap(configMap kcore.ConfigMap) error { _, err := config.K8s.CreateConfigMap(&configMap) return err } ================================================ FILE: pkg/operator/resources/job/taskapi/metrics.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package taskapi import ( "time" "github.com/cortexlabs/cortex/pkg/config" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/types/spec" ) func recordSuccess(jobKey spec.JobKey) error { tags := []string{ "api_name:" + jobKey.APIName, "job_id:" + jobKey.ID, } err := config.MetricsClient.Incr("cortex_task_succeeded", tags, 1.0) if err != nil { return errors.WithStack(err) } return nil } func recordFailure(jobKey spec.JobKey) error { tags := []string{ "api_name:" + jobKey.APIName, "job_id:" + jobKey.ID, } err := config.MetricsClient.Incr("cortex_task_failed", tags, 1.0) if err != nil { return errors.WithStack(err) } return nil } func recordTimePerTask(jobKey spec.JobKey, elapsedTime time.Duration) error { tags := []string{ "api_name:" + jobKey.APIName, "job_id:" + jobKey.ID, } err := config.MetricsClient.Histogram("cortex_time_per_task", elapsedTime.Seconds(), tags, 1.0) if err != nil { return errors.WithStack(err) } return nil } ================================================ FILE: pkg/operator/resources/job/taskapi/validations.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package taskapi import ( cr "github.com/cortexlabs/cortex/pkg/lib/configreader" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/operator/schema" ) func validateJobSubmission(submission *schema.TaskJobSubmission) error { if submission.Workers != 1 { return errors.Wrap(cr.ErrorInvalidInt(submission.Workers, 1), schema.WorkersKey) } return nil } ================================================ FILE: pkg/operator/resources/job/worker_stats.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package job import ( "github.com/cortexlabs/cortex/pkg/lib/k8s" "github.com/cortexlabs/cortex/pkg/types/status" kbatch "k8s.io/api/batch/v1" kcore "k8s.io/api/core/v1" ) func GetWorkerCountsForJob(k8sJob kbatch.Job, pods []kcore.Pod) status.WorkerCounts { if k8sJob.Status.Failed > 0 { return status.WorkerCounts{ Failed: *k8sJob.Spec.Parallelism, // When one worker fails, the rest of the pods get deleted so you won't be able to get their statuses } } workerCounts := status.WorkerCounts{} for i := range pods { addPodToWorkerCounts(&pods[i], &workerCounts) } return workerCounts } func addPodToWorkerCounts(pod *kcore.Pod, workerCounts *status.WorkerCounts) { if k8s.IsPodReady(pod) { workerCounts.Ready++ return } switch k8s.GetPodStatus(pod) { case k8s.PodStatusPending: workerCounts.Pending++ case k8s.PodStatusStalled: workerCounts.Stalled++ case k8s.PodStatusCreating: workerCounts.Creating++ case k8s.PodStatusNotReady: workerCounts.NotReady++ case k8s.PodStatusErrImagePull: workerCounts.ErrImagePull++ case k8s.PodStatusTerminating: workerCounts.Terminating++ case k8s.PodStatusFailed: workerCounts.Failed++ case k8s.PodStatusKilled: workerCounts.Killed++ case k8s.PodStatusKilledOOM: workerCounts.KilledOOM++ case k8s.PodStatusSucceeded: workerCounts.Succeeded++ case k8s.PodStatusUnknown: workerCounts.Unknown++ } } ================================================ FILE: pkg/operator/resources/realtimeapi/api.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package realtimeapi import ( "fmt" "path/filepath" "sort" "time" "github.com/cortexlabs/cortex/pkg/config" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/k8s" "github.com/cortexlabs/cortex/pkg/lib/parallel" "github.com/cortexlabs/cortex/pkg/lib/pointer" "github.com/cortexlabs/cortex/pkg/operator/lib/routines" "github.com/cortexlabs/cortex/pkg/operator/operator" "github.com/cortexlabs/cortex/pkg/operator/schema" "github.com/cortexlabs/cortex/pkg/types/spec" "github.com/cortexlabs/cortex/pkg/types/status" "github.com/cortexlabs/cortex/pkg/types/userconfig" "github.com/cortexlabs/cortex/pkg/workloads" istioclientnetworking "istio.io/client-go/pkg/apis/networking/v1beta1" kapps "k8s.io/api/apps/v1" kcore "k8s.io/api/core/v1" ) const _realtimeDashboardUID = "realtimeapi" func generateDeploymentID() string { return k8s.RandomName()[:10] } func UpdateAPI(apiConfig *userconfig.API, force bool) (*spec.API, string, error) { prevDeployment, prevService, prevVirtualService, err := getK8sResources(apiConfig.Name) if err != nil { return nil, "", err } initialDeploymentTime := time.Now().UnixNano() deploymentID := generateDeploymentID() if prevVirtualService != nil && prevVirtualService.Labels["initialDeploymentTime"] != "" { var err error initialDeploymentTime, err = k8s.ParseInt64Label(prevVirtualService, "initialDeploymentTime") if err != nil { return nil, "", err } deploymentID = prevVirtualService.Labels["deploymentID"] } api := spec.GetAPISpec(apiConfig, initialDeploymentTime, deploymentID, config.ClusterConfig.ClusterUID) if prevDeployment == nil { if err := config.AWS.UploadJSONToS3(api, config.ClusterConfig.Bucket, api.Key); err != nil { return nil, "", errors.Wrap(err, "upload api spec") } if err := applyK8sResources(api, prevDeployment, prevService, prevVirtualService); err != nil { routines.RunWithPanicHandler(func() { _ = deleteK8sResources(api.Name) }) return nil, "", err } return api, fmt.Sprintf("creating %s", api.Resource.UserString()), nil } if prevVirtualService.Labels["specID"] != api.SpecID || prevVirtualService.Labels["deploymentID"] != api.DeploymentID { isUpdating, err := isAPIUpdating(prevDeployment) if err != nil { return nil, "", err } if isUpdating && !force { return nil, "", ErrorAPIUpdating(api.Name) } if err := config.AWS.UploadJSONToS3(api, config.ClusterConfig.Bucket, api.Key); err != nil { return nil, "", errors.Wrap(err, "upload api spec") } if err := applyK8sResources(api, prevDeployment, prevService, prevVirtualService); err != nil { return nil, "", err } return api, fmt.Sprintf("updating %s", api.Resource.UserString()), nil } // deployment didn't change isUpdating, err := isAPIUpdating(prevDeployment) if err != nil { return nil, "", err } if isUpdating { return api, fmt.Sprintf("%s is already updating", api.Resource.UserString()), nil } return api, fmt.Sprintf("%s is up to date", api.Resource.UserString()), nil } func RefreshAPI(apiName string, force bool) (string, error) { prevDeployment, prevService, prevVirtualService, err := getK8sResources(apiName) if err != nil { return "", err } else if prevDeployment == nil || prevVirtualService == nil { return "", errors.ErrorUnexpected("unable to find deployment", apiName) } isUpdating, err := isAPIUpdating(prevDeployment) if err != nil { return "", err } if isUpdating && !force { return "", ErrorAPIUpdating(apiName) } apiID, err := k8s.GetLabel(prevDeployment, "apiID") if err != nil { return "", err } api, err := operator.DownloadAPISpec(apiName, apiID) if err != nil { return "", err } initialDeploymentTime, err := k8s.ParseInt64Label(prevVirtualService, "initialDeploymentTime") if err != nil { return "", err } api = spec.GetAPISpec(api.API, initialDeploymentTime, generateDeploymentID(), config.ClusterConfig.ClusterUID) if err := config.AWS.UploadJSONToS3(api, config.ClusterConfig.Bucket, api.Key); err != nil { return "", errors.Wrap(err, "upload api spec") } if err := applyK8sResources(api, prevDeployment, prevService, prevVirtualService); err != nil { return "", err } return fmt.Sprintf("updating %s", api.Resource.UserString()), nil } func DeleteAPI(apiName string, keepCache bool) error { err := parallel.RunFirstErr( func() error { return deleteK8sResources(apiName) }, func() error { if keepCache { return nil } // best effort deletion, swallow errors because there could be weird error messages _ = deleteBucketResources(apiName) return nil }, ) if err != nil { return err } return nil } func GetAllAPIs(deployments []kapps.Deployment) ([]schema.APIResponse, error) { realtimeAPIs := make([]schema.APIResponse, len(deployments)) mappedRealtimeAPIs := make(map[string]schema.APIResponse, len(deployments)) apiNames := make([]string, len(deployments)) for i := range deployments { apiName := deployments[i].Labels["apiName"] apiNames[i] = apiName metadata, err := spec.MetadataFromDeployment(&deployments[i]) if err != nil { return nil, errors.Wrap(err, fmt.Sprintf("api %s", apiName)) } mappedRealtimeAPIs[apiName] = schema.APIResponse{ Status: status.FromDeployment(&deployments[i]), Metadata: metadata, } } sort.Strings(apiNames) for i := range apiNames { realtimeAPIs[i] = mappedRealtimeAPIs[apiNames[i]] } return realtimeAPIs, nil } func GetAPIByName(deployedResource *operator.DeployedResource) ([]schema.APIResponse, error) { deployment, err := config.K8s.GetDeployment(workloads.K8sName(deployedResource.Name)) if err != nil { return nil, err } if deployment == nil { return nil, errors.ErrorUnexpected("unable to find deployment", deployedResource.Name) } apiStatus := status.FromDeployment(deployment) apiMetadata, err := spec.MetadataFromDeployment(deployment) if err != nil { return nil, errors.ErrorUnexpected("unable to obtain metadata", deployedResource.Name) } api, err := operator.DownloadAPISpec(apiMetadata.Name, apiMetadata.APIID) if err != nil { return nil, err } apiEndpoint, err := operator.APIEndpoint(api) if err != nil { return nil, err } dashboardURL := pointer.String(getDashboardURL(api.Name)) return []schema.APIResponse{ { Spec: api, Metadata: apiMetadata, Status: apiStatus, Endpoint: &apiEndpoint, DashboardURL: dashboardURL, }, }, nil } func DescribeAPIByName(deployedResource *operator.DeployedResource) ([]schema.APIResponse, error) { deployment, err := config.K8s.GetDeployment(workloads.K8sName(deployedResource.Name)) if err != nil { return nil, err } if deployment == nil { return nil, errors.ErrorUnexpected("unable to find deployment", deployedResource.Name) } apiStatus := status.FromDeployment(deployment) apiMetadata, err := spec.MetadataFromDeployment(deployment) if err != nil { return nil, errors.ErrorUnexpected("unable to obtain metadata", deployedResource.Name) } pods, err := config.K8s.ListPodsByLabel("apiName", deployment.Labels["apiName"]) if err != nil { return nil, err } apiStatus.ReplicaCounts = GetReplicaCounts(deployment, pods) apiEndpoint, err := operator.APIEndpointFromResource(deployedResource) if err != nil { return nil, err } dashboardURL := pointer.String(getDashboardURL(deployedResource.Name)) return []schema.APIResponse{ { Metadata: apiMetadata, Status: apiStatus, Endpoint: &apiEndpoint, DashboardURL: dashboardURL, }, }, nil } func getK8sResources(apiName string) (*kapps.Deployment, *kcore.Service, *istioclientnetworking.VirtualService, error) { var deployment *kapps.Deployment var service *kcore.Service var virtualService *istioclientnetworking.VirtualService err := parallel.RunFirstErr( func() error { var err error deployment, err = config.K8s.GetDeployment(workloads.K8sName(apiName)) return err }, func() error { var err error service, err = config.K8s.GetService(workloads.K8sName(apiName)) return err }, func() error { var err error virtualService, err = config.K8s.GetVirtualService(workloads.K8sName(apiName)) return err }, ) return deployment, service, virtualService, err } func applyK8sResources(api *spec.API, prevDeployment *kapps.Deployment, prevService *kcore.Service, prevVirtualService *istioclientnetworking.VirtualService) error { return parallel.RunFirstErr( func() error { return applyK8sDeployment(api, prevDeployment) }, func() error { return applyK8sService(api, prevService) }, func() error { return applyK8sVirtualService(api, prevVirtualService) }, ) } func applyK8sDeployment(api *spec.API, prevDeployment *kapps.Deployment) error { newDeployment := deploymentSpec(api, prevDeployment) if prevDeployment == nil { _, err := config.K8s.CreateDeployment(newDeployment) if err != nil { return err } } else if prevDeployment.Status.ReadyReplicas == 0 { // Delete deployment if it never became ready _, _ = config.K8s.DeleteDeployment(workloads.K8sName(api.Name)) _, err := config.K8s.CreateDeployment(newDeployment) if err != nil { return err } } else { _, err := config.K8s.UpdateDeployment(newDeployment) if err != nil { return err } } return nil } func applyK8sService(api *spec.API, prevService *kcore.Service) error { newService := serviceSpec(api) if prevService == nil { _, err := config.K8s.CreateService(newService) return err } _, err := config.K8s.UpdateService(prevService, newService) return err } func applyK8sVirtualService(api *spec.API, prevVirtualService *istioclientnetworking.VirtualService) error { newVirtualService := virtualServiceSpec(api) if prevVirtualService == nil { _, err := config.K8s.CreateVirtualService(newVirtualService) return err } _, err := config.K8s.UpdateVirtualService(prevVirtualService, newVirtualService) return err } func deleteK8sResources(apiName string) error { return parallel.RunFirstErr( func() error { _, err := config.K8s.DeleteDeployment(workloads.K8sName(apiName)) return err }, func() error { _, err := config.K8s.DeleteService(workloads.K8sName(apiName)) return err }, func() error { _, err := config.K8s.DeleteVirtualService(workloads.K8sName(apiName)) return err }, ) } func deleteBucketResources(apiName string) error { prefix := filepath.Join(config.ClusterConfig.ClusterUID, "apis", apiName) return config.AWS.DeleteS3Dir(config.ClusterConfig.Bucket, prefix, true) } // returns true if min_replicas are not ready and no updated replicas have errored func isAPIUpdating(deployment *kapps.Deployment) (bool, error) { pods, err := config.K8s.ListPodsByLabel("apiName", deployment.Labels["apiName"]) if err != nil { return false, err } replicaCounts := GetReplicaCounts(deployment, pods) autoscalingSpec, err := userconfig.AutoscalingFromAnnotations(deployment) if err != nil { return false, err } if replicaCounts.Ready < autoscalingSpec.MinReplicas && replicaCounts.TotalFailed() == 0 { return true, nil } return false, nil } func isPodSpecLatest(deployment *kapps.Deployment, pod *kcore.Pod) bool { return deployment.Spec.Template.Labels["podID"] == pod.Labels["podID"] && deployment.Spec.Template.Labels["deploymentID"] == pod.Labels["deploymentID"] } func getDashboardURL(apiName string) string { loadBalancerURL, err := operator.LoadBalancerURL() if err != nil { return "" } dashboardURL := fmt.Sprintf( "%s/dashboard/d/%s/realtimeapi?orgId=1&refresh=30s&var-api_name=%s", loadBalancerURL, _realtimeDashboardUID, apiName, ) return dashboardURL } ================================================ FILE: pkg/operator/resources/realtimeapi/errors.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package realtimeapi import ( "fmt" "github.com/cortexlabs/cortex/pkg/lib/errors" ) const ( ErrAPIUpdating = "realtimeapi.api_updating" ) func ErrorAPIUpdating(apiName string) error { return errors.WithStack(&errors.Error{ Kind: ErrAPIUpdating, Message: fmt.Sprintf("%s is updating (override with --force)", apiName), }) } ================================================ FILE: pkg/operator/resources/realtimeapi/k8s_specs.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package realtimeapi import ( "fmt" "github.com/cortexlabs/cortex/pkg/consts" "github.com/cortexlabs/cortex/pkg/lib/k8s" "github.com/cortexlabs/cortex/pkg/lib/pointer" s "github.com/cortexlabs/cortex/pkg/lib/strings" "github.com/cortexlabs/cortex/pkg/types/spec" "github.com/cortexlabs/cortex/pkg/workloads" istionetworking "istio.io/api/networking/v1beta1" istioclientnetworking "istio.io/client-go/pkg/apis/networking/v1beta1" kapps "k8s.io/api/apps/v1" kcore "k8s.io/api/core/v1" ) var _terminationGracePeriodSeconds int64 = 60 // seconds func deploymentSpec(api *spec.API, prevDeployment *kapps.Deployment) *kapps.Deployment { containers, volumes := workloads.RealtimeContainers(*api) return k8s.Deployment(&k8s.DeploymentSpec{ Name: workloads.K8sName(api.Name), Replicas: getRequestedReplicasFromDeployment(*api, prevDeployment), MaxSurge: pointer.String(api.UpdateStrategy.MaxSurge), MaxUnavailable: pointer.String(api.UpdateStrategy.MaxUnavailable), Labels: map[string]string{ "apiName": api.Name, "apiKind": api.Kind.String(), "apiID": api.ID, "specID": api.SpecID, "initialDeploymentTime": s.Int64(api.InitialDeploymentTime), "deploymentID": api.DeploymentID, "podID": api.PodID, "cortex.dev/api": "true", }, Annotations: api.ToK8sAnnotations(), Selector: map[string]string{ "apiName": api.Name, "apiKind": api.Kind.String(), }, PodSpec: k8s.PodSpec{ Labels: map[string]string{ "apiName": api.Name, "apiKind": api.Kind.String(), "initialDeploymentTime": s.Int64(api.InitialDeploymentTime), "deploymentID": api.DeploymentID, "podID": api.PodID, "cortex.dev/api": "true", }, Annotations: map[string]string{ "traffic.sidecar.istio.io/excludeOutboundIPRanges": "0.0.0.0/0", }, K8sPodSpec: kcore.PodSpec{ RestartPolicy: "Always", TerminationGracePeriodSeconds: pointer.Int64(_terminationGracePeriodSeconds), Containers: containers, NodeSelector: workloads.NodeSelectors(), Tolerations: workloads.GenerateResourceTolerations(), Affinity: workloads.GenerateNodeAffinities(api.NodeGroups), Volumes: volumes, ServiceAccountName: workloads.ServiceAccountName, }, }, }) } func serviceSpec(api *spec.API) *kcore.Service { return k8s.Service(&k8s.ServiceSpec{ Name: workloads.K8sName(api.Name), PortName: "http", Port: consts.ProxyPortInt32, TargetPort: consts.ProxyPortInt32, Annotations: api.ToK8sAnnotations(), Labels: map[string]string{ "apiName": api.Name, "apiKind": api.Kind.String(), "cortex.dev/api": "true", }, Selector: map[string]string{ "apiName": api.Name, "apiKind": api.Kind.String(), }, }) } func virtualServiceSpec(api *spec.API) *istioclientnetworking.VirtualService { var activatorWeight int32 if api.Autoscaling.InitReplicas == 0 { activatorWeight = 100 } return k8s.VirtualService(&k8s.VirtualServiceSpec{ Name: workloads.K8sName(api.Name), Gateways: []string{"apis-gateway"}, Destinations: []k8s.Destination{ { ServiceName: workloads.K8sName(api.Name), Weight: 100 - activatorWeight, Port: uint32(consts.ProxyPortInt32), Headers: &istionetworking.Headers{ Response: &istionetworking.Headers_HeaderOperations{ Set: map[string]string{ consts.CortexOriginHeader: "api", }, }, }, }, { ServiceName: consts.ActivatorName, Weight: activatorWeight, Port: uint32(consts.ActivatorPortInt32), Headers: &istionetworking.Headers{ Request: &istionetworking.Headers_HeaderOperations{ Set: map[string]string{ consts.CortexAPINameHeader: api.Name, consts.CortexTargetServiceHeader: fmt.Sprintf( "http://%s.%s:%d", workloads.K8sName(api.Name), consts.DefaultNamespace, consts.ProxyPortInt32, ), }, }, Response: &istionetworking.Headers_HeaderOperations{ Set: map[string]string{ consts.CortexOriginHeader: consts.ActivatorName, }, }, }, }, }, PrefixPath: api.Networking.Endpoint, Rewrite: pointer.String("/"), Retries: pointer.Int32(0), Annotations: api.ToK8sAnnotations(), Labels: map[string]string{ "apiName": api.Name, "apiKind": api.Kind.String(), "apiID": api.ID, "specID": api.SpecID, "initialDeploymentTime": s.Int64(api.InitialDeploymentTime), "deploymentID": api.DeploymentID, "podID": api.PodID, "cortex.dev/api": "true", }, }) } func getRequestedReplicasFromDeployment(api spec.API, deployment *kapps.Deployment) int32 { requestedReplicas := api.Autoscaling.InitReplicas if deployment != nil && deployment.Spec.Replicas != nil && *deployment.Spec.Replicas > 0 { requestedReplicas = *deployment.Spec.Replicas } if requestedReplicas < api.Autoscaling.MinReplicas { requestedReplicas = api.Autoscaling.MinReplicas } if requestedReplicas > api.Autoscaling.MaxReplicas { requestedReplicas = api.Autoscaling.MaxReplicas } return requestedReplicas } ================================================ FILE: pkg/operator/resources/realtimeapi/status.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package realtimeapi import ( "github.com/cortexlabs/cortex/pkg/lib/k8s" "github.com/cortexlabs/cortex/pkg/types/status" kapps "k8s.io/api/apps/v1" kcore "k8s.io/api/core/v1" ) func GetReplicaCounts(deployment *kapps.Deployment, pods []kcore.Pod) *status.ReplicaCounts { counts := status.ReplicaCounts{} counts.Requested = *deployment.Spec.Replicas for i := range pods { pod := pods[i] if pod.Labels["apiName"] != deployment.Labels["apiName"] { continue } addPodToReplicaCounts(&pods[i], deployment, &counts) } return &counts } func addPodToReplicaCounts(pod *kcore.Pod, deployment *kapps.Deployment, counts *status.ReplicaCounts) { latest := false if isPodSpecLatest(deployment, pod) { latest = true } isPodReady := k8s.IsPodReady(pod) if latest && isPodReady { counts.Ready++ return } else if !latest && isPodReady { counts.ReadyOutOfDate++ return } podStatus := k8s.GetPodStatus(pod) if podStatus == k8s.PodStatusTerminating { counts.Terminating++ return } if !latest { return } switch podStatus { case k8s.PodStatusPending: counts.Pending++ case k8s.PodStatusStalled: counts.Stalled++ case k8s.PodStatusCreating: counts.Creating++ case k8s.PodStatusReady: counts.Ready++ case k8s.PodStatusNotReady: counts.NotReady++ case k8s.PodStatusErrImagePull: counts.ErrImagePull++ case k8s.PodStatusFailed: counts.Failed++ case k8s.PodStatusKilled: counts.Killed++ case k8s.PodStatusKilledOOM: counts.KilledOOM++ case k8s.PodStatusUnknown: counts.Unknown++ } } ================================================ FILE: pkg/operator/resources/resources.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package resources import ( "context" "fmt" "github.com/cortexlabs/cortex/pkg/config" "github.com/cortexlabs/cortex/pkg/consts" batch "github.com/cortexlabs/cortex/pkg/crds/apis/batch/v1alpha1" "github.com/cortexlabs/cortex/pkg/lib/aws" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/logging" "github.com/cortexlabs/cortex/pkg/lib/parallel" "github.com/cortexlabs/cortex/pkg/lib/pointer" "github.com/cortexlabs/cortex/pkg/lib/telemetry" "github.com/cortexlabs/cortex/pkg/operator/lib/routines" "github.com/cortexlabs/cortex/pkg/operator/operator" "github.com/cortexlabs/cortex/pkg/operator/resources/asyncapi" "github.com/cortexlabs/cortex/pkg/operator/resources/job/batchapi" "github.com/cortexlabs/cortex/pkg/operator/resources/job/taskapi" "github.com/cortexlabs/cortex/pkg/operator/resources/realtimeapi" "github.com/cortexlabs/cortex/pkg/operator/resources/trafficsplitter" "github.com/cortexlabs/cortex/pkg/operator/schema" "github.com/cortexlabs/cortex/pkg/types/spec" "github.com/cortexlabs/cortex/pkg/types/userconfig" "github.com/cortexlabs/cortex/pkg/workloads" istioclientnetworking "istio.io/client-go/pkg/apis/networking/v1beta1" kapps "k8s.io/api/apps/v1" kbatch "k8s.io/api/batch/v1" kcore "k8s.io/api/core/v1" kmeta "k8s.io/apimachinery/pkg/apis/meta/v1" klabels "k8s.io/apimachinery/pkg/labels" ) var operatorLogger = logging.GetLogger() // Returns an error if resource doesn't exist func GetDeployedResourceByName(resourceName string) (*operator.DeployedResource, error) { resource, err := GetDeployedResourceByNameOrNil(resourceName) if err != nil { return nil, err } if resource == nil { return nil, ErrorAPINotDeployed(resourceName) } return resource, nil } func GetDeployedResourceByNameOrNil(resourceName string) (*operator.DeployedResource, error) { virtualService, err := config.K8s.GetVirtualService(workloads.K8sName(resourceName)) if err != nil { return nil, err } if virtualService == nil { return nil, nil } return &operator.DeployedResource{ Resource: userconfig.Resource{ Name: virtualService.Labels["apiName"], Kind: userconfig.KindFromString(virtualService.Labels["apiKind"]), }, VirtualService: virtualService, }, nil } func Deploy(configFileName string, configBytes []byte, force bool) ([]schema.DeployResult, error) { apiConfigs, err := spec.ExtractAPIConfigs(configBytes, configFileName) if err != nil { return nil, err } err = ValidateClusterAPIs(apiConfigs) if err != nil { err = errors.Append(err, fmt.Sprintf("\n\napi configuration schema can be found at https://docs.cortexlabs.com/v/%s/", consts.CortexVersionMinor)) return nil, err } // This is done if user specifies RealtimeAPIs in same file as TrafficSplitter apiConfigs = append(ExclusiveFilterAPIsByKind(apiConfigs, userconfig.TrafficSplitterKind), InclusiveFilterAPIsByKind(apiConfigs, userconfig.TrafficSplitterKind)...) results := make([]schema.DeployResult, 0, len(apiConfigs)) for i := range apiConfigs { apiConfig := apiConfigs[i] api, msg, err := UpdateAPI(&apiConfig, force) result := schema.DeployResult{ Message: msg, API: api, } if err != nil { result.Error = errors.ErrorStr(err) } results = append(results, result) } return results, nil } func UpdateAPI(apiConfig *userconfig.API, force bool) (*schema.APIResponse, string, error) { deployedResource, err := GetDeployedResourceByNameOrNil(apiConfig.Name) if err != nil { return nil, "", err } if deployedResource != nil && deployedResource.Kind != apiConfig.Kind { return nil, "", ErrorCannotChangeKindOfDeployedAPI(apiConfig.Name, apiConfig.Kind, deployedResource.Kind) } telemetry.Event("operator.deploy", apiConfig.TelemetryEvent()) var api *spec.API var msg string switch apiConfig.Kind { case userconfig.RealtimeAPIKind: api, msg, err = realtimeapi.UpdateAPI(apiConfig, force) case userconfig.BatchAPIKind: api, msg, err = batchapi.UpdateAPI(apiConfig) case userconfig.TaskAPIKind: api, msg, err = taskapi.UpdateAPI(apiConfig) case userconfig.AsyncAPIKind: api, msg, err = asyncapi.UpdateAPI(*apiConfig, force) case userconfig.TrafficSplitterKind: api, msg, err = trafficsplitter.UpdateAPI(apiConfig) default: return nil, "", ErrorOperationIsOnlySupportedForKind( *deployedResource, userconfig.RealtimeAPIKind, userconfig.AsyncAPIKind, userconfig.BatchAPIKind, userconfig.TrafficSplitterKind, userconfig.TaskAPIKind, ) // unexpected } if err == nil && api != nil { apiEndpoint, _ := operator.APIEndpoint(api) return &schema.APIResponse{ Spec: api, Endpoint: &apiEndpoint, }, msg, nil } return nil, msg, err } func RefreshAPI(apiName string, force bool) (string, error) { deployedResource, err := GetDeployedResourceByName(apiName) if err != nil { return "", err } switch deployedResource.Kind { case userconfig.RealtimeAPIKind: return realtimeapi.RefreshAPI(apiName, force) case userconfig.AsyncAPIKind: return asyncapi.RefreshAPI(apiName, force) default: return "", ErrorOperationIsOnlySupportedForKind(*deployedResource, userconfig.RealtimeAPIKind, userconfig.AsyncAPIKind) } } func DeleteAPI(apiName string, keepCache bool) (*schema.DeleteResponse, error) { deployedResource, err := GetDeployedResourceByNameOrNil(apiName) if err != nil { return nil, err } if deployedResource == nil { // Delete anyways just to be sure everything is deleted routines.RunWithPanicHandler(func() { err := parallel.RunFirstErr( func() error { return realtimeapi.DeleteAPI(apiName, keepCache) }, func() error { return batchapi.DeleteAPI(apiName, keepCache) }, func() error { return trafficsplitter.DeleteAPI(apiName, keepCache) }, func() error { return taskapi.DeleteAPI(apiName, keepCache) }, func() error { return asyncapi.DeleteAPI(apiName, keepCache) }, ) if err != nil { telemetry.Error(err) } }) return nil, ErrorAPINotDeployed(apiName) } switch deployedResource.Kind { case userconfig.RealtimeAPIKind: err := checkIfUsedByTrafficSplitter(apiName) if err != nil { return nil, err } err = realtimeapi.DeleteAPI(apiName, keepCache) if err != nil { return nil, err } case userconfig.TrafficSplitterKind: err := trafficsplitter.DeleteAPI(apiName, keepCache) if err != nil { return nil, err } case userconfig.BatchAPIKind: err := batchapi.DeleteAPI(apiName, keepCache) if err != nil { return nil, err } case userconfig.TaskAPIKind: err := taskapi.DeleteAPI(apiName, keepCache) if err != nil { return nil, err } case userconfig.AsyncAPIKind: err = asyncapi.DeleteAPI(apiName, keepCache) if err != nil { return nil, err } default: return nil, ErrorOperationIsOnlySupportedForKind(*deployedResource, userconfig.RealtimeAPIKind, userconfig.AsyncAPIKind, userconfig.BatchAPIKind, userconfig.TrafficSplitterKind) // unexpected } return &schema.DeleteResponse{ Message: fmt.Sprintf("deleting %s", apiName), }, nil } func GetAPIs() ([]schema.APIResponse, error) { var deployments []kapps.Deployment var k8sTaskJobs []kbatch.Job var taskAPIPods []kcore.Pod var virtualServices []istioclientnetworking.VirtualService var batchJobList batch.BatchJobList err := parallel.RunFirstErr( func() error { var err error deployments, err = config.K8s.ListDeploymentsWithLabelKeys("apiName") return err }, func() error { var err error taskAPIPods, err = config.K8s.ListPodsByLabel("apiKind", userconfig.TaskAPIKind.String()) return err }, func() error { var err error k8sTaskJobs, err = config.K8s.ListJobs( &kmeta.ListOptions{ LabelSelector: klabels.SelectorFromSet( map[string]string{ "apiKind": userconfig.TaskAPIKind.String(), }, ).String(), }, ) return err }, func() error { var err error virtualServices, err = config.K8s.ListVirtualServicesWithLabelKeys("apiName") return err }, func() error { return config.K8s.List(context.Background(), &batchJobList) }, ) if err != nil { return nil, err } var realtimeAPIDeployments []kapps.Deployment var asyncAPIDeployments []kapps.Deployment for _, deployment := range deployments { switch deployment.Labels["apiKind"] { case userconfig.RealtimeAPIKind.String(): realtimeAPIDeployments = append(realtimeAPIDeployments, deployment) case userconfig.AsyncAPIKind.String(): asyncAPIDeployments = append(asyncAPIDeployments, deployment) } } var batchAPIVirtualServices []istioclientnetworking.VirtualService var taskAPIVirtualServices []istioclientnetworking.VirtualService var trafficSplitterVirtualServices []istioclientnetworking.VirtualService for _, vs := range virtualServices { switch vs.Labels["apiKind"] { case userconfig.BatchAPIKind.String(): batchAPIVirtualServices = append(batchAPIVirtualServices, vs) case userconfig.TrafficSplitterKind.String(): trafficSplitterVirtualServices = append(trafficSplitterVirtualServices, vs) case userconfig.TaskAPIKind.String(): taskAPIVirtualServices = append(taskAPIVirtualServices, vs) } } realtimeAPIList, err := realtimeapi.GetAllAPIs(realtimeAPIDeployments) if err != nil { return nil, err } var taskAPIList []schema.APIResponse taskAPIList, err = taskapi.GetAllAPIs(taskAPIVirtualServices, k8sTaskJobs, taskAPIPods) if err != nil { return nil, err } batchAPIList, err := batchapi.GetAllAPIs(batchAPIVirtualServices, batchJobList.Items) if err != nil { return nil, err } asyncAPIList, err := asyncapi.GetAllAPIs(asyncAPIDeployments) if err != nil { return nil, err } trafficSplitterList, err := trafficsplitter.GetAllAPIs(trafficSplitterVirtualServices) if err != nil { return nil, err } response := make([]schema.APIResponse, 0, len(realtimeAPIList)+len(batchAPIList)+len(trafficSplitterList)) response = append(response, realtimeAPIList...) response = append(response, batchAPIList...) response = append(response, taskAPIList...) response = append(response, asyncAPIList...) response = append(response, trafficSplitterList...) return response, nil } func GetAPI(apiName string) ([]schema.APIResponse, error) { deployedResource, err := GetDeployedResourceByName(apiName) if err != nil { return nil, err } var apiResponse []schema.APIResponse switch deployedResource.Kind { case userconfig.RealtimeAPIKind: apiResponse, err = realtimeapi.GetAPIByName(deployedResource) if err != nil { return nil, err } case userconfig.BatchAPIKind: apiResponse, err = batchapi.GetAPIByName(deployedResource) if err != nil { return nil, err } case userconfig.TaskAPIKind: apiResponse, err = taskapi.GetAPIByName(deployedResource) if err != nil { return nil, err } case userconfig.AsyncAPIKind: apiResponse, err = asyncapi.GetAPIByName(deployedResource) if err != nil { return nil, err } case userconfig.TrafficSplitterKind: apiResponse, err = trafficsplitter.GetAPIByName(deployedResource) if err != nil { return nil, err } default: return nil, ErrorOperationIsOnlySupportedForKind( *deployedResource, userconfig.RealtimeAPIKind, userconfig.BatchAPIKind, userconfig.TaskAPIKind, userconfig.TrafficSplitterKind, userconfig.AsyncAPIKind, ) // unexpected } // Get past API deploy times if len(apiResponse) > 0 { apiResponse[0].APIVersions, err = getPastAPIDeploys(deployedResource.Name) if err != nil { return nil, err } } return apiResponse, nil } func GetAPIByID(apiName string, apiID string) ([]schema.APIResponse, error) { // check if the API is currently running, so that additional information can be returned deployedResource, err := GetDeployedResourceByName(apiName) if err == nil && deployedResource != nil && deployedResource.ID() == apiID { return GetAPI(apiName) } // search for the API spec with the old ID apiSpec, err := operator.DownloadAPISpec(apiName, apiID) if err != nil { if aws.IsGenericNotFoundErr(err) { return nil, ErrorAPIIDNotFound(apiName, apiID) } return nil, err } return []schema.APIResponse{ { Spec: apiSpec, }, }, nil } func getPastAPIDeploys(apiName string) ([]schema.APIVersion, error) { var apiVersions []schema.APIVersion apiIDs, err := config.AWS.ListS3DirOneLevel(config.ClusterConfig.Bucket, spec.KeysPrefix(apiName, config.ClusterConfig.ClusterUID), pointer.Int64(10), nil) if err != nil { return nil, err } for _, apiID := range apiIDs { lastUpdated, err := spec.TimeFromAPIID(apiID) if err != nil { return nil, err } apiVersions = append(apiVersions, schema.APIVersion{ APIID: apiID, LastUpdated: lastUpdated.Unix(), }) } return apiVersions, nil } // checkIfUsedByTrafficSplitter checks if api is used by a deployed TrafficSplitter func checkIfUsedByTrafficSplitter(apiName string) error { virtualServices, err := config.K8s.ListVirtualServicesByLabel("apiKind", userconfig.TrafficSplitterKind.String()) if err != nil { return err } var usedByTrafficSplitters []string for _, vs := range virtualServices { trafficSplitterSpec, err := operator.DownloadAPISpec(vs.Labels["apiName"], vs.Labels["apiID"]) if err != nil { return err } for _, api := range trafficSplitterSpec.APIs { if apiName == api.Name { usedByTrafficSplitters = append(usedByTrafficSplitters, trafficSplitterSpec.Name) } } } if len(usedByTrafficSplitters) > 0 { return ErrorAPIUsedByTrafficSplitter(usedByTrafficSplitters) } return nil } func DescribeAPI(apiName string) ([]schema.APIResponse, error) { deployedResource, err := GetDeployedResourceByName(apiName) if err != nil { return nil, err } var apiResponse []schema.APIResponse switch deployedResource.Kind { case userconfig.RealtimeAPIKind: apiResponse, err = realtimeapi.DescribeAPIByName(deployedResource) if err != nil { return nil, err } case userconfig.AsyncAPIKind: apiResponse, err = asyncapi.DescribeAPIByName(deployedResource) if err != nil { return nil, err } default: return nil, ErrorOperationIsOnlySupportedForKind( *deployedResource, userconfig.RealtimeAPIKind, userconfig.AsyncAPIKind, ) // unexpected } return apiResponse, nil } ================================================ FILE: pkg/operator/resources/trafficsplitter/api.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package trafficsplitter import ( "fmt" "path/filepath" "time" "github.com/cortexlabs/cortex/pkg/config" "github.com/cortexlabs/cortex/pkg/consts" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/k8s" "github.com/cortexlabs/cortex/pkg/lib/parallel" "github.com/cortexlabs/cortex/pkg/lib/pointer" "github.com/cortexlabs/cortex/pkg/operator/lib/routines" "github.com/cortexlabs/cortex/pkg/operator/operator" "github.com/cortexlabs/cortex/pkg/operator/schema" "github.com/cortexlabs/cortex/pkg/types/spec" "github.com/cortexlabs/cortex/pkg/types/userconfig" "github.com/cortexlabs/cortex/pkg/workloads" istioclientnetworking "istio.io/client-go/pkg/apis/networking/v1beta1" ) // UpdateAPI creates or updates a traffic splitter API kind func UpdateAPI(apiConfig *userconfig.API) (*spec.API, string, error) { prevVirtualService, err := config.K8s.GetVirtualService(workloads.K8sName(apiConfig.Name)) if err != nil { return nil, "", err } initialDeploymentTime := time.Now().UnixNano() if prevVirtualService != nil && prevVirtualService.Labels["initialDeploymentTime"] != "" { var err error initialDeploymentTime, err = k8s.ParseInt64Label(prevVirtualService, "initialDeploymentTime") if err != nil { return nil, "", err } } api := spec.GetAPISpec(apiConfig, initialDeploymentTime, "", config.ClusterConfig.ClusterUID) if prevVirtualService == nil { if err := config.AWS.UploadJSONToS3(api, config.ClusterConfig.Bucket, api.Key); err != nil { return nil, "", errors.Wrap(err, "failed to upload api spec") } if err := applyK8sVirtualService(api, prevVirtualService); err != nil { routines.RunWithPanicHandler(func() { _ = deleteK8sResources(api.Name) }) return nil, "", err } return api, fmt.Sprintf("created %s", api.Resource.UserString()), nil } if prevVirtualService.Labels["specID"] != api.SpecID { if err := config.AWS.UploadJSONToS3(api, config.ClusterConfig.Bucket, api.Key); err != nil { return nil, "", errors.Wrap(err, "failed to upload api spec") } if err := applyK8sVirtualService(api, prevVirtualService); err != nil { return nil, "", err } return api, fmt.Sprintf("updated %s", api.Resource.UserString()), nil } return api, fmt.Sprintf("%s is up to date", api.Resource.UserString()), nil } // DeleteAPI deletes all the resources related to a given traffic splitter API func DeleteAPI(apiName string, keepCache bool) error { err := parallel.RunFirstErr( func() error { return deleteK8sResources(apiName) }, func() error { if keepCache { return nil } // best effort deletion _ = deleteS3Resources(apiName) return nil }, ) if err != nil { return err } return nil } func applyK8sVirtualService(trafficSplitter *spec.API, prevVirtualService *istioclientnetworking.VirtualService) error { newVirtualService := virtualServiceSpec(trafficSplitter) if prevVirtualService == nil { _, err := config.K8s.CreateVirtualService(newVirtualService) return err } _, err := config.K8s.UpdateVirtualService(prevVirtualService, newVirtualService) return err } func getTrafficSplitterDestinations(trafficSplitter *spec.API) []k8s.Destination { destinations := make([]k8s.Destination, len(trafficSplitter.APIs)) for i, api := range trafficSplitter.APIs { destinations[i] = k8s.Destination{ ServiceName: workloads.K8sName(api.Name), Weight: api.Weight, Port: uint32(consts.ProxyPortInt32), Shadow: api.Shadow, } } return destinations } // GetAllAPIs returns a list of metadata, in the form of schema.APIResponse, about all the created traffic splitter APIs func GetAllAPIs(virtualServices []istioclientnetworking.VirtualService) ([]schema.APIResponse, error) { var trafficSplitters []schema.APIResponse for i := range virtualServices { apiName := virtualServices[i].Labels["apiName"] metadata, err := spec.MetadataFromVirtualService(&virtualServices[i]) if err != nil { return nil, errors.Wrap(err, fmt.Sprintf("api %s", apiName)) } if metadata.Kind != userconfig.TrafficSplitterKind { continue } targets, err := userconfig.TrafficSplitterTargetsFromAnnotations(&virtualServices[i]) if err != nil { return nil, errors.Wrap(err, fmt.Sprintf("api %s", apiName)) } trafficSplitters = append(trafficSplitters, schema.APIResponse{ Metadata: metadata, NumTrafficSplitterTargets: pointer.Int32(targets), }) } return trafficSplitters, nil } // GetAPIByName retrieves the metadata, in the form of schema.APIResponse, of a single traffic splitter API func GetAPIByName(deployedResource *operator.DeployedResource) ([]schema.APIResponse, error) { metadata, err := spec.MetadataFromVirtualService(deployedResource.VirtualService) if err != nil { return nil, err } api, err := operator.DownloadAPISpec(deployedResource.Name, metadata.APIID) if err != nil { return nil, err } endpoint, err := operator.APIEndpoint(api) if err != nil { return nil, err } return []schema.APIResponse{ { Spec: api, Metadata: metadata, Endpoint: &endpoint, }, }, nil } func deleteK8sResources(apiName string) error { _, err := config.K8s.DeleteVirtualService(workloads.K8sName(apiName)) return err } func deleteS3Resources(apiName string) error { prefix := filepath.Join(config.ClusterConfig.ClusterUID, "apis", apiName) return config.AWS.DeleteS3Dir(config.ClusterConfig.Bucket, prefix, true) } ================================================ FILE: pkg/operator/resources/trafficsplitter/k8s_specs.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package trafficsplitter import ( "github.com/cortexlabs/cortex/pkg/lib/k8s" "github.com/cortexlabs/cortex/pkg/lib/pointer" s "github.com/cortexlabs/cortex/pkg/lib/strings" "github.com/cortexlabs/cortex/pkg/types/spec" "github.com/cortexlabs/cortex/pkg/workloads" istioclientnetworking "istio.io/client-go/pkg/apis/networking/v1beta1" ) func virtualServiceSpec(trafficSplitter *spec.API) *istioclientnetworking.VirtualService { return k8s.VirtualService(&k8s.VirtualServiceSpec{ Name: workloads.K8sName(trafficSplitter.Name), Gateways: []string{"apis-gateway"}, Destinations: getTrafficSplitterDestinations(trafficSplitter), ExactPath: trafficSplitter.Networking.Endpoint, Rewrite: pointer.String("/"), Retries: pointer.Int32(0), Annotations: trafficSplitter.ToK8sAnnotations(), Labels: map[string]string{ "apiName": trafficSplitter.Name, "apiKind": trafficSplitter.Kind.String(), "apiID": trafficSplitter.ID, "specID": trafficSplitter.SpecID, "initialDeploymentTime": s.Int64(trafficSplitter.InitialDeploymentTime), "cortex.dev/api": "true", }, }) } ================================================ FILE: pkg/operator/resources/validations.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package resources import ( "github.com/cortexlabs/cortex/pkg/config" "github.com/cortexlabs/cortex/pkg/consts" "github.com/cortexlabs/cortex/pkg/lib/aws" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/k8s" "github.com/cortexlabs/cortex/pkg/lib/sets/strset" "github.com/cortexlabs/cortex/pkg/lib/slices" s "github.com/cortexlabs/cortex/pkg/lib/strings" "github.com/cortexlabs/cortex/pkg/operator/operator" "github.com/cortexlabs/cortex/pkg/types/spec" "github.com/cortexlabs/cortex/pkg/types/userconfig" istioclientnetworking "istio.io/client-go/pkg/apis/networking/v1beta1" kresource "k8s.io/apimachinery/pkg/api/resource" ) func ValidateClusterAPIs(apis []userconfig.API) error { if len(apis) == 0 { return spec.ErrorNoAPIs() } if len(config.ClusterConfig.NodeGroups) == 0 { return ErrorNoNodeGroups() } virtualServices, err := config.K8s.ListVirtualServices(nil) if err != nil { return err } deployedRealtimeAPIs := strset.New() for _, virtualService := range virtualServices { if virtualService.Labels["apiKind"] == userconfig.RealtimeAPIKind.String() { deployedRealtimeAPIs.Add(virtualService.Labels["apiName"]) } } realtimeAPIs := InclusiveFilterAPIsByKind(apis, userconfig.RealtimeAPIKind) for i := range apis { api := &apis[i] if api.Kind == userconfig.RealtimeAPIKind || api.Kind == userconfig.BatchAPIKind || api.Kind == userconfig.TaskAPIKind || api.Kind == userconfig.AsyncAPIKind { if err := spec.ValidateAPI(api, config.AWS, config.K8s); err != nil { return errors.Wrap(err, api.Identify()) } if err := validateEndpointCollisions(api, virtualServices); err != nil { return err } } if api.Kind == userconfig.TrafficSplitterKind { if err := spec.ValidateTrafficSplitter(api); err != nil { return errors.Wrap(err, api.Identify()) } if err := checkIfAPIExists(api.APIs, realtimeAPIs, deployedRealtimeAPIs); err != nil { return errors.Wrap(err, api.Identify()) } if err := validateEndpointCollisions(api, virtualServices); err != nil { return errors.Wrap(err, api.Identify()) } } } maxMemMap, err := operator.UpdateMemoryCapacityConfigMap() if err != nil { return err } for i := range apis { api := &apis[i] if api.Kind != userconfig.TrafficSplitterKind { if err := validateK8sCompute(api, maxMemMap); err != nil { return errors.Wrap(err, api.Identify()) } } } dups := spec.FindDuplicateNames(apis) if len(dups) > 0 { return spec.ErrorDuplicateName(dups) } dups = findDuplicateEndpoints(apis) if len(dups) > 0 { return spec.ErrorDuplicateEndpointInOneDeploy(dups) } return nil } var _nvidiaDevicePluginCPUReserve = kresource.MustParse("100m") var _nvidiaDevicePluginMemReserve = kresource.MustParse("100Mi") var _nvidiaDCGMExporterCPUReserve = kresource.MustParse("50m") var _nvidiaDCGMExporterMemReserve = kresource.MustParse("50Mi") var _neuronDevicePluginCPUReserve = kresource.MustParse("100m") var _neuronDevicePluginMemReserve = kresource.MustParse("100Mi") func validateK8sCompute(api *userconfig.API, maxMemMap map[string]kresource.Quantity) error { clusterNodeGroupNames := strset.New(config.ClusterConfig.GetNodeGroupNames()...) for _, ngName := range api.NodeGroups { if !clusterNodeGroupNames.Has(ngName) { return errors.Wrap(ErrorInvalidNodeGroupSelector(ngName, config.ClusterConfig.GetNodeGroupNames()), userconfig.NodeGroupsKey) } } compute := userconfig.GetPodComputeRequest(api) for _, ng := range config.ClusterConfig.NodeGroups { if api.NodeGroups != nil && !slices.HasString(api.NodeGroups, ng.Name) { continue } nodeCPU, nodeMem, nodeGPU, nodeInf := getNodeCapacity(ng.InstanceType, maxMemMap) if compute.CPU != nil && nodeCPU.Cmp(compute.CPU.Quantity) < 0 { continue } else if compute.Mem != nil && nodeMem.Cmp(compute.Mem.Quantity) < 0 { continue } else if compute.GPU > nodeGPU { continue } else if compute.Inf > nodeInf { continue } // we found a node group that has capacity return nil } // no nodegroups have capacity return ErrorNoAvailableNodeComputeLimit(api, compute, maxMemMap) } func getNodeCapacity(instanceType string, maxMemMap map[string]kresource.Quantity) (kresource.Quantity, kresource.Quantity, int64, int64) { instanceMetadata := aws.InstanceMetadatas[config.ClusterConfig.Region][instanceType] cpu := instanceMetadata.CPU.DeepCopy() cpu.Sub(consts.CortexCPUPodReserved) cpu.Sub(consts.CortexCPUK8sReserved) mem := maxMemMap[instanceType].DeepCopy() mem.Sub(consts.CortexMemPodReserved) mem.Sub(consts.CortexMemK8sReserved) gpu := instanceMetadata.GPU if gpu > 0 { // Reserve resources for nvidia device plugin daemonset cpu.Sub(_nvidiaDevicePluginCPUReserve) mem.Sub(_nvidiaDevicePluginMemReserve) // Reserve resources for nvidia dcgm prometheus exporter cpu.Sub(_nvidiaDCGMExporterCPUReserve) mem.Sub(_nvidiaDCGMExporterMemReserve) } inf := instanceMetadata.Inf if inf > 0 { // Reserve resources for neuron device plugin daemonset cpu.Sub(_neuronDevicePluginCPUReserve) mem.Sub(_neuronDevicePluginMemReserve) } return cpu, mem, gpu, inf } func validateEndpointCollisions(api *userconfig.API, virtualServices []istioclientnetworking.VirtualService) error { for i := range virtualServices { virtualService := virtualServices[i] gateways := k8s.ExtractVirtualServiceGateways(&virtualService) if !gateways.Has("apis-gateway") { continue } endpoints := k8s.ExtractVirtualServiceEndpoints(&virtualService) for endpoint := range endpoints { if s.EnsureSuffix(endpoint, "/") == s.EnsureSuffix(*api.Networking.Endpoint, "/") && virtualService.Labels["apiName"] != api.Name { return errors.Wrap(spec.ErrorDuplicateEndpoint(virtualService.Labels["apiName"]), userconfig.NetworkingKey, userconfig.EndpointKey, endpoint) } } } return nil } func findDuplicateEndpoints(apis []userconfig.API) []userconfig.API { endpoints := make(map[string][]userconfig.API) for _, api := range apis { endpoints[*api.Networking.Endpoint] = append(endpoints[*api.Networking.Endpoint], api) } for endpoint := range endpoints { if len(endpoints[endpoint]) > 1 { return endpoints[endpoint] } } return nil } // InclusiveFilterAPIsByKind includes only provided Kinds func InclusiveFilterAPIsByKind(apis []userconfig.API, kindsToInclude ...userconfig.Kind) []userconfig.API { kindsToIncludeSet := strset.New() for _, kind := range kindsToInclude { kindsToIncludeSet.Add(kind.String()) } fileredAPIs := []userconfig.API{} for _, api := range apis { if kindsToIncludeSet.Has(api.Kind.String()) { fileredAPIs = append(fileredAPIs, api) } } return fileredAPIs } func ExclusiveFilterAPIsByKind(apis []userconfig.API, kindsToExclude ...userconfig.Kind) []userconfig.API { kindsToExcludeSet := strset.New() for _, kind := range kindsToExclude { kindsToExcludeSet.Add(kind.String()) } fileredAPIs := []userconfig.API{} for _, api := range apis { if !kindsToExcludeSet.Has(api.Kind.String()) { fileredAPIs = append(fileredAPIs, api) } } return fileredAPIs } // checkIfAPIExists checks if referenced apis in trafficsplitter are either defined in yaml or already deployed. func checkIfAPIExists(trafficSplitterAPIs []*userconfig.TrafficSplit, apis []userconfig.API, deployedRealtimeAPIs strset.Set) error { var missingAPIs []string // check if apis named in trafficsplitter are either defined in same yaml or already deployed for _, trafficSplitAPI := range trafficSplitterAPIs { // check if already deployed deployed := deployedRealtimeAPIs.Has(trafficSplitAPI.Name) // check defined apis for _, definedAPI := range apis { if trafficSplitAPI.Name == definedAPI.Name { deployed = true } } if !deployed { missingAPIs = append(missingAPIs, trafficSplitAPI.Name) } } if len(missingAPIs) != 0 { return ErrorAPIsNotDeployed(missingAPIs) } return nil } ================================================ FILE: pkg/operator/schema/config_key.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package schema const ( // Job Submission BatchSizeKey = "batch_size" ItemsKey = "items" ItemListKey = "item_list" FilePathListerKey = "file_path_lister" DelimitedFilesKey = "delimited_files" S3PathsKey = "s3_paths" IncludesKey = "includes" ExcludesKey = "excludes" WorkersKey = "workers" TimeoutKey = "timeout" MaxReceiveCountKey = "max_receive_count" ARNKey = "arn" SQSDeadLetterQueueKey = "sqs_dead_letter_queue" ) ================================================ FILE: pkg/operator/schema/job_submission.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package schema import ( "encoding/json" "github.com/cortexlabs/cortex/pkg/types/spec" ) type ItemList struct { Items []json.RawMessage `json:"items"` BatchSize int `json:"batch_size"` } type S3Lister struct { S3Paths []string `json:"s3_paths"` // s3:///key Includes []string `json:"includes"` Excludes []string `json:"excludes"` MaxResults *int64 `json:"-"` // this is not currently exposed to the user (it's used for validations) } type FilePathLister struct { S3Lister BatchSize int `json:"batch_size"` } type DelimitedFiles struct { S3Lister BatchSize int `json:"batch_size"` } type BatchJobSubmission struct { spec.RuntimeBatchJobConfig ItemList *ItemList `json:"item_list"` FilePathLister *FilePathLister `json:"file_path_lister"` DelimitedFiles *DelimitedFiles `json:"delimited_files"` } type TaskJobSubmission struct { spec.RuntimeTaskJobConfig } ================================================ FILE: pkg/operator/schema/schema.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package schema import ( "github.com/cortexlabs/cortex/pkg/types/clusterconfig" "github.com/cortexlabs/cortex/pkg/types/metrics" "github.com/cortexlabs/cortex/pkg/types/spec" "github.com/cortexlabs/cortex/pkg/types/status" "github.com/cortexlabs/cortex/pkg/types/userconfig" ) type InfoResponse struct { ClusterConfig clusterconfig.InternalConfig `json:"cluster_config" yaml:"cluster_config"` WorkerNodeInfos []WorkerNodeInfo `json:"worker_node_infos" yaml:"worker_node_infos"` OperatorNodeInfos []NodeInfo `json:"operator_node_infos" yaml:"operator_node_infos"` NumPendingReplicas int `json:"num_pending_replicas" yaml:"num_pending_replicas"` } type WorkerNodeInfo struct { NodeInfo Name string `json:"name" yaml:"name"` NumReplicas int `json:"num_replicas" yaml:"num_replicas"` NumEnqueuerReplicas int `json:"num_enqueuer_replicas" yaml:"num_enqueuer_replicas"` ComputeUserCapacity userconfig.Compute `json:"compute_user_capacity" yaml:"compute_user_capacity"` // the total resources available to the user on a node ComputeAvailable userconfig.Compute `json:"compute_available" yaml:"compute_unavailable"` // unused resources on a node ComputeUserRequested userconfig.Compute `json:"compute_user_requested" yaml:"compute_user_requested"` // total resources requested by user on a node } type NodeInfo struct { NodeGroupName string `json:"nodegroup_name" yaml:"nodegroup_name"` InstanceType string `json:"instance_type" yaml:"instance_type"` IsSpot bool `json:"is_spot" yaml:"is_spot"` Price float64 `json:"price" yaml:"price"` } type DeployResult struct { API *APIResponse `json:"api" yaml:"api"` Message string `json:"message" yaml:"message"` Error string `json:"error" yaml:"error"` } type APIResponse struct { Spec *spec.API `json:"spec,omitempty" yaml:"spec,omitempty"` Metadata *spec.Metadata `json:"metadata,omitempty" yaml:"metadata,omitempty"` Status *status.Status `json:"status,omitempty" yaml:"status,omitempty"` NumTrafficSplitterTargets *int32 `json:"num_traffic_splitter_targets,omitempty" yaml:"num_traffic_splitter_targets,omitempty"` Endpoint *string `json:"endpoint,omitempty" yaml:"endpoint,omitempty"` DashboardURL *string `json:"dashboard_url,omitempty" yaml:"dashboard_url,omitempty"` BatchJobStatuses []status.BatchJobStatus `json:"batch_job_statuses,omitempty" yaml:"batch_job_statuses,omitempty"` TaskJobStatuses []status.TaskJobStatus `json:"task_job_statuses,omitempty" yaml:"task_job_statuses,omitempty"` APIVersions []APIVersion `json:"api_versions,omitempty" yaml:"api_versions,omitempty"` } type LogResponse struct { LogURL string `json:"log_url"` } type BatchJobResponse struct { APISpec spec.API `json:"api_spec" yaml:"api_spec"` JobStatus status.BatchJobStatus `json:"job_status" yaml:"job_status"` Metrics *metrics.BatchMetrics `json:"metrics,omitempty" yaml:"metrics,omitempty"` Endpoint string `json:"endpoint" yaml:"endpoint"` } type TaskJobResponse struct { APISpec spec.API `json:"api_spec" yaml:"api_spec"` JobStatus status.TaskJobStatus `json:"job_status" yaml:"job_status"` Endpoint string `json:"endpoint" yaml:"endpoint"` } type DeleteResponse struct { Message string `json:"message"` } type RefreshResponse struct { Message string `json:"message"` } type ErrorResponse struct { Kind string `json:"kind"` Message string `json:"message"` } type APIVersion struct { APIID string `json:"api_id" yaml:"api_id"` LastUpdated int64 `json:"last_updated" yaml:"last_updated"` } type VerifyCortexResponse struct{} func (ir InfoResponse) GetNodesWithNodeGroupName(ngName string) []WorkerNodeInfo { nodesInfo := []WorkerNodeInfo{} for _, nodeInfo := range ir.WorkerNodeInfos { if nodeInfo.NodeGroupName == ngName { nodesInfo = append(nodesInfo, nodeInfo) } } return nodesInfo } ================================================ FILE: pkg/probe/handler.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package probe import ( "net/http" "strings" "github.com/cortexlabs/cortex/pkg/consts" ) func IsRequestKubeletProbe(r *http.Request) bool { return strings.HasPrefix(r.Header.Get(consts.UserAgentKey), consts.KubeProbeUserAgentPrefix) } ================================================ FILE: pkg/probe/handler_test.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package probe_test import ( "net/http" "net/http/httptest" "net/url" "testing" "time" "github.com/cortexlabs/cortex/pkg/probe" "github.com/stretchr/testify/require" kcore "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/intstr" ) func generateHandler(pb *probe.Probe) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { if !pb.IsHealthy() { w.WriteHeader(http.StatusInternalServerError) _, _ = w.Write([]byte("unhealthy")) return } w.WriteHeader(http.StatusOK) _, _ = w.Write([]byte("healthy")) } } func TestHandlerSuccessTCP(t *testing.T) { t.Parallel() log := newLogger(t) defer func() { _ = log.Sync() }() var userHandler http.HandlerFunc = func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) } server := httptest.NewServer(userHandler) pb := probe.NewDefaultProbe(server.URL, log) handler := generateHandler(pb) r := httptest.NewRequest(http.MethodGet, "http://fake.cortex.dev/healthz", nil) w := httptest.NewRecorder() stopper := pb.StartProbing() defer func() { stopper <- struct{}{} }() for { if pb.HasRunOnce() { break } time.Sleep(time.Second) } handler(w, r) require.Equal(t, http.StatusOK, w.Code) require.Equal(t, "healthy", w.Body.String()) } func TestHandlerSuccessHTTP(t *testing.T) { t.Parallel() log := newLogger(t) defer func() { _ = log.Sync() }() headers := []kcore.HTTPHeader{ { Name: "X-Cortex-Blah", Value: "Blah", }, } var userHandler http.HandlerFunc = func(w http.ResponseWriter, r *http.Request) { require.True(t, probe.IsRequestKubeletProbe(r)) for _, header := range headers { require.Equal(t, header.Value, r.Header.Get(header.Name)) } w.WriteHeader(http.StatusOK) } server := httptest.NewServer(userHandler) targetURL, err := url.Parse(server.URL) require.NoError(t, err) pb := probe.NewProbe( &kcore.Probe{ Handler: kcore.Handler{ HTTPGet: &kcore.HTTPGetAction{ Path: "/", Port: intstr.FromString(targetURL.Port()), Host: targetURL.Hostname(), HTTPHeaders: headers, }, }, TimeoutSeconds: 3, PeriodSeconds: 1, SuccessThreshold: 1, FailureThreshold: 3, }, log, ) handler := generateHandler(pb) r := httptest.NewRequest(http.MethodGet, "http://fake.cortex.dev/healthz", nil) w := httptest.NewRecorder() stopper := pb.StartProbing() defer func() { stopper <- struct{}{} }() for { if pb.HasRunOnce() { break } time.Sleep(time.Second) } handler(w, r) require.Equal(t, http.StatusOK, w.Code) require.Equal(t, "healthy", w.Body.String()) } ================================================ FILE: pkg/probe/probe.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package probe import ( "fmt" "io" "io/ioutil" "net" "net/http" "net/url" "sync" "time" "github.com/cortexlabs/cortex/pkg/consts" "github.com/cortexlabs/cortex/pkg/lib/errors" s "github.com/cortexlabs/cortex/pkg/lib/strings" "go.uber.org/zap" kcore "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/intstr" ) const ( _defaultInitialDelaySeconds = int32(1) _defaultTimeoutSeconds = int32(1) _defaultPeriodSeconds = int32(1) _defaultSuccessThreshold = int32(1) _defaultFailureThreshold = int32(1) ) type Probe struct { *kcore.Probe sync.RWMutex logger *zap.SugaredLogger healthy bool hasRunOnce bool } func NewProbe(probe *kcore.Probe, logger *zap.SugaredLogger) *Probe { return &Probe{ Probe: probe, logger: logger, } } func NewDefaultProbe(target string, logger *zap.SugaredLogger) *Probe { targetURL, err := url.Parse(target) if err != nil { panic(fmt.Sprintf("failed to parse target URL: %v", err)) } return &Probe{ Probe: &kcore.Probe{ Handler: kcore.Handler{ TCPSocket: &kcore.TCPSocketAction{ Port: intstr.FromString(targetURL.Port()), Host: targetURL.Hostname(), }, }, InitialDelaySeconds: _defaultInitialDelaySeconds, TimeoutSeconds: _defaultTimeoutSeconds, PeriodSeconds: _defaultPeriodSeconds, SuccessThreshold: _defaultSuccessThreshold, FailureThreshold: _defaultFailureThreshold, }, logger: logger, } } func (p *Probe) StartProbing() chan struct{} { stop := make(chan struct{}) time.AfterFunc(time.Duration(p.InitialDelaySeconds)*time.Second, func() { ticker := time.NewTicker(time.Duration(p.PeriodSeconds) * time.Second) successCount := int32(0) failureCount := int32(0) for { select { case <-stop: return case <-ticker.C: healthy := p.probeContainer() if healthy { successCount++ failureCount = 0 } else { failureCount++ successCount = 0 } p.Lock() if successCount >= p.SuccessThreshold { p.healthy = true } else if failureCount >= p.FailureThreshold { p.healthy = false } p.hasRunOnce = true p.Unlock() } } }) return stop } func (p *Probe) IsHealthy() bool { p.RLock() defer p.RUnlock() return p.healthy } func (p *Probe) HasRunOnce() bool { p.RLock() defer p.RUnlock() return p.hasRunOnce } func AreProbesHealthy(probes []*Probe) bool { for _, probe := range probes { if probe == nil { continue } if !probe.IsHealthy() { return false } } return true } func (p *Probe) probeContainer() bool { var err error var probeType string switch { case p.HTTPGet != nil: err = p.httpProbe() probeType = "http" case p.TCPSocket != nil: err = p.tcpProbe() probeType = "tcp" case p.Exec != nil: // Should never be reachable. p.logger.Error("exec probe not supported") return false default: p.logger.Warn("no probe found") return false } if err != nil { p.logger.Warn(errors.Wrapf(err, "%s probe to user container failed", probeType)) return false } return true } func (p *Probe) httpProbe() error { // to mimic k8s probe functionality targetHost := p.HTTPGet.Host if p.HTTPGet.Host == "" { targetHost = "localhost" } targetURL := s.EnsurePrefix( net.JoinHostPort(targetHost, p.HTTPGet.Port.String())+s.EnsurePrefix(p.HTTPGet.Path, "/"), "http://", ) httpClient := &http.Client{ Timeout: time.Duration(p.TimeoutSeconds) * time.Second, } req, err := http.NewRequest(http.MethodGet, targetURL, nil) if err != nil { return err } req.Header.Add(consts.UserAgentKey, consts.KubeProbeUserAgentPrefix) for _, header := range p.HTTPGet.HTTPHeaders { req.Header.Add(header.Name, header.Value) } res, err := httpClient.Do(req) if err != nil { return err } defer func() { // Ensure body is both read _and_ closed so it can be reused for keep-alive. // No point handling errors, connection just won't be reused. _, _ = io.Copy(ioutil.Discard, res.Body) _ = res.Body.Close() }() // response status code between 200-399 indicates success if !(res.StatusCode >= 200 && res.StatusCode < 400) { return fmt.Errorf("HTTP probe did not respond Ready, got status code: %d", res.StatusCode) } return nil } func (p *Probe) tcpProbe() error { // to mimic k8s probe functionality targetHost := p.TCPSocket.Host if p.TCPSocket.Host == "" { targetHost = "localhost" } timeout := time.Duration(p.TimeoutSeconds) * time.Second address := net.JoinHostPort(targetHost, p.TCPSocket.Port.String()) conn, err := net.DialTimeout("tcp", address, timeout) if err != nil { return err } _ = conn.Close() return nil } ================================================ FILE: pkg/probe/probe_test.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package probe_test import ( "net/http" "net/http/httptest" "net/url" "testing" "time" "github.com/cortexlabs/cortex/pkg/probe" "github.com/stretchr/testify/require" "go.uber.org/zap" kcore "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/intstr" ) func newLogger(t *testing.T) *zap.SugaredLogger { t.Helper() config := zap.NewDevelopmentConfig() config.Level = zap.NewAtomicLevelAt(zap.FatalLevel) logger, err := config.Build() require.NoError(t, err) log := logger.Sugar() return log } func TestDefaultProbeSuccess(t *testing.T) { t.Parallel() log := newLogger(t) defer func() { _ = log.Sync() }() var handler http.HandlerFunc = func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) } server := httptest.NewServer(handler) pb := probe.NewDefaultProbe(server.URL, log) stopper := pb.StartProbing() defer func() { stopper <- struct{}{} }() for { if pb.HasRunOnce() { break } time.Sleep(time.Second) } require.True(t, pb.IsHealthy()) } func TestDefaultProbeFailure(t *testing.T) { t.Parallel() log := newLogger(t) defer func() { _ = log.Sync() }() target := "http://127.0.0.1:12345" pb := probe.NewDefaultProbe(target, log) stopper := pb.StartProbing() defer func() { stopper <- struct{}{} }() for { if pb.HasRunOnce() { break } time.Sleep(time.Second) } require.False(t, pb.IsHealthy()) } func TestProbeHTTPFailure(t *testing.T) { t.Parallel() log := newLogger(t) defer func() { _ = log.Sync() }() pb := probe.NewProbe( &kcore.Probe{ Handler: kcore.Handler{ HTTPGet: &kcore.HTTPGetAction{ Path: "/healthz", Port: intstr.FromString("12345"), Host: "127.0.0.1", }, }, InitialDelaySeconds: 1, TimeoutSeconds: 3, PeriodSeconds: 1, SuccessThreshold: 1, FailureThreshold: 1, }, log, ) stopper := pb.StartProbing() defer func() { stopper <- struct{}{} }() for { if pb.HasRunOnce() { break } time.Sleep(time.Second) } require.False(t, pb.IsHealthy()) } func TestProbeHTTPSuccess(t *testing.T) { t.Parallel() log := newLogger(t) defer func() { _ = log.Sync() }() var handler http.HandlerFunc = func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) } server := httptest.NewServer(handler) targetURL, err := url.Parse(server.URL) require.NoError(t, err) pb := probe.NewProbe( &kcore.Probe{ Handler: kcore.Handler{ HTTPGet: &kcore.HTTPGetAction{ Path: "/healthz", Port: intstr.FromString(targetURL.Port()), Host: targetURL.Hostname(), }, }, InitialDelaySeconds: 1, TimeoutSeconds: 3, PeriodSeconds: 1, SuccessThreshold: 1, FailureThreshold: 1, }, log, ) stopper := pb.StartProbing() defer func() { stopper <- struct{}{} }() for { if pb.HasRunOnce() { break } time.Sleep(time.Second) } require.True(t, pb.IsHealthy()) } ================================================ FILE: pkg/proxy/breaker.go ================================================ /* Copyright 2018 The Knative Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. Modifications Copyright 2022 Cortex Labs, Inc. */ package proxy import ( "context" "errors" "fmt" "go.uber.org/atomic" ) var ( // ErrRequestQueueFull indicates the breaker queue depth was exceeded. ErrRequestQueueFull = errors.New("pending request queue full") ) // BreakerParams defines the parameters of the breaker. type BreakerParams struct { QueueDepth int MaxConcurrency int InitialCapacity int } // Breaker is a component that enforces a concurrency limit on the // execution of a function. It also maintains a queue of function // executions in excess of the concurrency limit. Function call attempts // beyond the limit of the queue are failed immediately. type Breaker struct { inFlight atomic.Int64 totalSlots int64 sem *semaphore // release is the callback function returned to callers by Reserve to // allow the reservation made by Reserve to be released. release func() } // NewBreaker creates a Breaker with the desired queue depth, // concurrency limit and initial capacity. func NewBreaker(params BreakerParams) *Breaker { if params.QueueDepth <= 0 { panic(fmt.Sprintf("Queue depth must be greater than 0. Got %v.", params.QueueDepth)) } if params.MaxConcurrency < 0 { panic(fmt.Sprintf("Max concurrency must be 0 or greater. Got %v.", params.MaxConcurrency)) } if params.InitialCapacity < 0 || params.InitialCapacity > params.MaxConcurrency { panic(fmt.Sprintf("Initial capacity must be between 0 and max concurrency. Got %v.", params.InitialCapacity)) } b := &Breaker{ totalSlots: int64(params.QueueDepth + params.MaxConcurrency), sem: newSemaphore(params.MaxConcurrency, params.InitialCapacity), } // Allocating the closure returned by Reserve here avoids an allocation in Reserve. b.release = func() { b.sem.release() b.releasePending() } return b } // tryAcquirePending tries to acquire a slot on the pending "queue". func (b *Breaker) tryAcquirePending() bool { // This is an atomic version of: // // if inFlight == totalSlots { // return false // } else { // inFlight++ // return true // } // // We can't just use an atomic increment as we need to check if we're // "allowed" to increment first. Since a Load and a CompareAndSwap are // not done atomically, we need to retry until the CompareAndSwap succeeds // (it fails if we're raced to it) or if we don't fulfill the condition // anymore. for { cur := b.inFlight.Load() if cur == b.totalSlots { return false } if b.inFlight.CAS(cur, cur+1) { return true } } } // releasePending releases a slot on the pending "queue". func (b *Breaker) releasePending() { b.inFlight.Dec() } // Reserve reserves an execution slot in the breaker, to permit // richer semantics in the caller. // The caller on success must execute the callback when done with work. func (b *Breaker) Reserve(_ context.Context) (func(), bool) { if !b.tryAcquirePending() { return nil, false } if !b.sem.tryAcquire() { b.releasePending() return nil, false } return b.release, true } // Maybe conditionally executes thunk based on the Breaker concurrency // and queue parameters. If the concurrency limit and queue capacity are // already consumed, Maybe returns immediately without calling thunk. If // the thunk was executed, Maybe returns true, else false. func (b *Breaker) Maybe(ctx context.Context, thunk func()) error { if !b.tryAcquirePending() { return ErrRequestQueueFull } defer b.releasePending() // Wait for capacity in the active queue. if err := b.sem.acquire(ctx); err != nil { return err } // Defer releasing capacity in the active. // It's safe to ignore the error returned by release since we // make sure the semaphore is only manipulated here and acquire // + release calls are equally paired. defer b.sem.release() // Do the thing. thunk() // Report success return nil } // InFlight returns the number of requests currently in flight in this breaker. func (b *Breaker) InFlight() int64 { return b.inFlight.Load() } // UpdateConcurrency updates the maximum number of in-flight requests. func (b *Breaker) UpdateConcurrency(size int) { b.sem.updateCapacity(size) } // UpdateQueueLength updates the number of allowed requests in-queue func (b *Breaker) UpdateQueueLength(size int) { b.totalSlots = int64(b.sem.Capacity() + size) } // Capacity returns the number of allowed in-flight requests on this breaker. func (b *Breaker) Capacity() int { return b.sem.Capacity() } func (b *Breaker) QueueLength() int64 { return b.totalSlots - int64(b.sem.Capacity()) } // newSemaphore creates a semaphore with the desired initial capacity. func newSemaphore(maxCapacity, initialCapacity int) *semaphore { queue := make(chan struct{}, maxCapacity) sem := &semaphore{queue: queue} sem.updateCapacity(initialCapacity) return sem } // semaphore is an implementation of a semaphore based on packed integers and a channel. // state is an uint64 that has two uint32s packed into it: capacity and inFlight. The // former specifies how many request are allowed at any given time into the semaphore // while the latter refers to the currently in-flight requests. // Packing them both into one uint64 allows us to optimize access semantics using atomic // operations, which can't be guaranteed on 2 individual values. // The channel is merely used as a vehicle to be able to "wake up" individual goroutines // if capacity becomes free. It's not consistently used in accordance to actual capacity // but is rather a communication vehicle to ensure waiting routines are properly woken // up. type semaphore struct { state atomic.Uint64 queue chan struct{} } // tryAcquire receives a token from the semaphore if there is one otherwise returns false. func (s *semaphore) tryAcquire() bool { for { old := s.state.Load() capacity, in := unpack(old) if in >= capacity { return false } in++ if s.state.CAS(old, pack(capacity, in)) { return true } } } // acquire acquires capacity from the semaphore. func (s *semaphore) acquire(ctx context.Context) error { for { old := s.state.Load() capacity, in := unpack(old) if in >= capacity { select { case <-ctx.Done(): return ctx.Err() case <-s.queue: } // Force reload state. continue } in++ if s.state.CAS(old, pack(capacity, in)) { return nil } } } // release releases capacity in the semaphore. // If the semaphore capacity was reduced in between and as a result inFlight is greater // than capacity, we don't wake up goroutines as they'd not get any capacity anyway. func (s *semaphore) release() { for { old := s.state.Load() capacity, in := unpack(old) if in == 0 { panic("release and acquire are not paired") } in-- if s.state.CAS(old, pack(capacity, in)) { if in < capacity { select { case s.queue <- struct{}{}: default: // We generate more wakeups than we might need as we don't know // how many goroutines are waiting here. It is therefore okay // to drop the poke on the floor here as this case would mean we // have enough wakeups to wake up as many goroutines as this semaphore // can take, which is guaranteed to be enough. } } return } } } // updateCapacity updates the capacity of the semaphore to the desired size. func (s *semaphore) updateCapacity(size int) { s64 := uint64(size) for { old := s.state.Load() capacity, in := unpack(old) if capacity == s64 { // Nothing to do, exit early. return } if s.state.CAS(old, pack(s64, in)) { if s64 > capacity { for i := uint64(0); i < s64-capacity; i++ { select { case s.queue <- struct{}{}: default: // See comment in `release` for explanation of this case. } } } return } } } // Capacity is the capacity of the semaphore. func (s *semaphore) Capacity() int { capacity, _ := unpack(s.state.Load()) return int(capacity) } // unpack takes an uint64 and returns two uint32 (as uint64) comprised of the leftmost // and the rightmost bits respectively. func unpack(in uint64) (uint64, uint64) { return in >> 32, in & 0xffffffff } // pack takes two uint32 (as uint64 to avoid casting) and packs them into a single uint64 // at the leftmost and the rightmost bits respectively. // It's up to the caller to ensure that left and right actually fit into 32 bit. func pack(left, right uint64) uint64 { return left<<32 | right } ================================================ FILE: pkg/proxy/breaker_test.go ================================================ /* Copyright 2018 The Knative Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. Modifications Copyright 2022 Cortex Labs, Inc. */ package proxy import ( "context" "fmt" "testing" "time" "github.com/stretchr/testify/require" "go.uber.org/atomic" ) const ( // semAcquireTimeout is a timeout for tests that try to acquire // a token of a semaphore. semAcquireTimeout = 10 * time.Second // semNoChangeTimeout is some additional wait time after a number // of acquires is reached to assert that no more acquires get through. semNoChangeTimeout = 50 * time.Millisecond ) func TestBreakerInvalidConstructor(t *testing.T) { tests := []struct { name string options BreakerParams }{{ name: "QueueDepth = 0", options: BreakerParams{QueueDepth: 0, MaxConcurrency: 1, InitialCapacity: 1}, }, { name: "MaxConcurrency negative", options: BreakerParams{QueueDepth: 1, MaxConcurrency: -1, InitialCapacity: 1}, }, { name: "InitialCapacity negative", options: BreakerParams{QueueDepth: 1, MaxConcurrency: 1, InitialCapacity: -1}, }, { name: "InitialCapacity out-of-bounds", options: BreakerParams{QueueDepth: 1, MaxConcurrency: 5, InitialCapacity: 6}, }} for _, test := range tests { t.Run(test.name, func(t *testing.T) { defer func() { if r := recover(); r == nil { t.Error("Expected a panic but the code didn't panic.") } }() NewBreaker(test.options) }) } } func TestBreakerReserveOverload(t *testing.T) { params := BreakerParams{QueueDepth: 1, MaxConcurrency: 1, InitialCapacity: 1} b := NewBreaker(params) // Breaker capacity = 2 cb1, rr := b.Reserve(context.Background()) if !rr { t.Fatal("Reserve1 failed") } _, rr = b.Reserve(context.Background()) if rr { t.Fatal("Reserve2 was an unexpected success.") } // Release a slot. cb1() // And reserve it again. cb2, rr := b.Reserve(context.Background()) if !rr { t.Fatal("Reserve2 failed") } cb2() } func TestBreakerOverloadMixed(t *testing.T) { // This tests when reservation and maybe are intermised. params := BreakerParams{QueueDepth: 1, MaxConcurrency: 1, InitialCapacity: 1} b := NewBreaker(params) // Breaker capacity = 2 reqs := newRequestor(b) // Bring breaker to capacity. reqs.request() // This happens in go-routine, so spin. for _, in := unpack(b.sem.state.Load()); in != 1; _, in = unpack(b.sem.state.Load()) { time.Sleep(time.Millisecond * 2) } _, rr := b.Reserve(context.Background()) if rr { t.Fatal("Reserve was an unexpected success.") } // Open a slot. reqs.processSuccessfully(t) // Now reservation should work. cb, rr := b.Reserve(context.Background()) if !rr { t.Fatal("Reserve unexpectedly failed") } // Process the reservation. cb() } func TestBreakerOverload(t *testing.T) { params := BreakerParams{QueueDepth: 1, MaxConcurrency: 1, InitialCapacity: 1} b := NewBreaker(params) // Breaker capacity = 2 reqs := newRequestor(b) // Bring breaker to capacity. reqs.request() reqs.request() // Overshoot by one. reqs.request() reqs.expectFailure(t) // The remainer should succeed. reqs.processSuccessfully(t) reqs.processSuccessfully(t) } func TestBreakerQueueing(t *testing.T) { params := BreakerParams{QueueDepth: 2, MaxConcurrency: 1, InitialCapacity: 0} b := NewBreaker(params) // Breaker capacity = 2 reqs := newRequestor(b) // Bring breaker to capacity. Doesn't error because queue subsumes these requests. reqs.request() reqs.request() // Update concurrency to allow the requests to be processed. b.UpdateConcurrency(1) // They should pass just fine. reqs.processSuccessfully(t) reqs.processSuccessfully(t) } func TestBreakerInflight(t *testing.T) { params := BreakerParams{QueueDepth: 2, MaxConcurrency: 1, InitialCapacity: 1} b := NewBreaker(params) // Breaker capacity = 2 reqs := newRequestor(b) // Bring breaker to capacity. Doesn't error because queue subsumes these requests. reqs.request() reqs.request() reqs.request() require.Eventually(t, func() bool { return b.InFlight() == int64(3) }, time.Second, 10*time.Millisecond) require.Equal(t, reqs.InProgress.Load(), int64(1)) } func TestBreakerNoOverload(t *testing.T) { params := BreakerParams{QueueDepth: 1, MaxConcurrency: 1, InitialCapacity: 1} b := NewBreaker(params) // Breaker capacity = 2 reqs := newRequestor(b) // Bring request to capacity. reqs.request() reqs.request() // Process one, send a new one in, at capacity again. reqs.processSuccessfully(t) reqs.request() // Process one, send a new one in, at capacity again. reqs.processSuccessfully(t) reqs.request() // Process the remainder successfully. reqs.processSuccessfully(t) reqs.processSuccessfully(t) } func TestBreakerCancel(t *testing.T) { params := BreakerParams{QueueDepth: 1, MaxConcurrency: 1, InitialCapacity: 0} b := NewBreaker(params) reqs := newRequestor(b) // Cancel a request which cannot get capacity. ctx1, cancel1 := context.WithCancel(context.Background()) reqs.requestWithContext(ctx1) cancel1() reqs.expectFailure(t) // This request cannot get capacity either. This reproduced a bug we had when // freeing slots on the pendingRequests channel. ctx2, cancel2 := context.WithCancel(context.Background()) reqs.requestWithContext(ctx2) cancel2() reqs.expectFailure(t) // Let through a request with capacity then timeout following request b.UpdateConcurrency(1) reqs.request() // Exceed capacity and assert one failure. This makes sure the Breaker is consistently // at capacity. reqs.request() reqs.request() reqs.expectFailure(t) // This request cannot get capacity. ctx3, cancel3 := context.WithCancel(context.Background()) reqs.requestWithContext(ctx3) cancel3() reqs.expectFailure(t) // The requests that were put in earlier should succeed. reqs.processSuccessfully(t) reqs.processSuccessfully(t) } func TestBreakerUpdateConcurrency(t *testing.T) { params := BreakerParams{QueueDepth: 1, MaxConcurrency: 1, InitialCapacity: 0} b := NewBreaker(params) b.UpdateConcurrency(1) if got, want := b.Capacity(), 1; got != want { t.Errorf("Capacity() = %d, want: %d", got, want) } b.UpdateConcurrency(0) if got, want := b.Capacity(), 0; got != want { t.Errorf("Capacity() = %d, want: %d", got, want) } } // Test empty semaphore, token cannot be acquired func TestSemaphoreAcquireHasNoCapacity(t *testing.T) { gotChan := make(chan struct{}, 1) sem := newSemaphore(1, 0) tryAcquire(sem, gotChan) select { case <-gotChan: t.Error("Token was acquired but shouldn't have been") case <-time.After(semNoChangeTimeout): // Test succeeds, semaphore didn't change in configured time. } } func TestSemaphoreAcquireNonBlockingHasNoCapacity(t *testing.T) { sem := newSemaphore(1, 0) if sem.tryAcquire() { t.Error("Should have failed immediately") } } // Test empty semaphore, add capacity, token can be acquired func TestSemaphoreAcquireHasCapacity(t *testing.T) { gotChan := make(chan struct{}, 1) want := 1 sem := newSemaphore(1, 0) tryAcquire(sem, gotChan) sem.updateCapacity(1) // Allows 1 acquire for i := 0; i < want; i++ { select { case <-gotChan: // Successfully acquired a token. case <-time.After(semAcquireTimeout): t.Error("Was not able to acquire token before timeout") } } select { case <-gotChan: t.Errorf("Got more acquires than wanted, want = %d, got at least %d", want, want+1) case <-time.After(semNoChangeTimeout): // No change happened, success. } } func TestSemaphoreRelease(t *testing.T) { sem := newSemaphore(1, 1) sem.acquire(context.Background()) func() { defer func() { if e := recover(); e != nil { t.Error("Expected no panic, got message:", e) } sem.release() }() }() func() { defer func() { if e := recover(); e == nil { t.Error("Expected panic, but got none") } }() sem.release() }() } func TestSemaphoreUpdateCapacity(t *testing.T) { const initialCapacity = 1 sem := newSemaphore(3, initialCapacity) if got, want := sem.Capacity(), 1; got != want { t.Errorf("Capacity = %d, want: %d", got, want) } sem.acquire(context.Background()) sem.updateCapacity(initialCapacity + 2) if got, want := sem.Capacity(), 3; got != want { t.Errorf("Capacity = %d, want: %d", got, want) } } func TestPackUnpack(t *testing.T) { wantL := uint64(256) wantR := uint64(513) gotL, gotR := unpack(pack(wantL, wantR)) if gotL != wantL || gotR != wantR { t.Fatalf("Got %d, %d want %d, %d", gotL, gotR, wantL, wantR) } } func tryAcquire(sem *semaphore, gotChan chan struct{}) { go func() { // blocking until someone puts the token into the semaphore sem.acquire(context.Background()) gotChan <- struct{}{} }() } // requestor is a set of test helpers around breaker testing. type requestor struct { breaker *Breaker acceptedCh chan bool barrierCh chan struct{} InProgress atomic.Int64 } func newRequestor(breaker *Breaker) *requestor { return &requestor{ breaker: breaker, acceptedCh: make(chan bool), barrierCh: make(chan struct{}), } } // request is the same as requestWithContext but with a default context. func (r *requestor) request() { r.requestWithContext(context.Background()) } // requestWithContext simulates a request in a separate goroutine. The // request will either fail immediately (as observable via expectFailure) // or block until processSuccessfully is called. func (r *requestor) requestWithContext(ctx context.Context) { go func() { err := r.breaker.Maybe(ctx, func() { r.InProgress.Inc() <-r.barrierCh }) r.acceptedCh <- err == nil }() } // expectFailure waits for a request to finish and asserts it to be failed. func (r *requestor) expectFailure(t *testing.T) { t.Helper() if <-r.acceptedCh { t.Error("expected request to fail but it succeeded") } } // processSuccessfully allows a request to pass the barrier, waits for it to // be finished and asserts it to succeed. func (r *requestor) processSuccessfully(t *testing.T) { t.Helper() r.barrierCh <- struct{}{} if !<-r.acceptedCh { t.Error("expected request to succeed but it failed") } } func BenchmarkBreakerMaybe(b *testing.B) { op := func() {} for _, c := range []int{1, 10, 100, 1000} { breaker := NewBreaker(BreakerParams{QueueDepth: 10000000, MaxConcurrency: c, InitialCapacity: c}) b.Run(fmt.Sprintf("%d-sequential", c), func(b *testing.B) { for j := 0; j < b.N; j++ { breaker.Maybe(context.Background(), op) } }) b.Run(fmt.Sprintf("%d-parallel", c), func(b *testing.B) { b.RunParallel(func(pb *testing.PB) { for pb.Next() { breaker.Maybe(context.Background(), op) } }) }) } } func BenchmarkBreakerReserve(b *testing.B) { op := func() {} breaker := NewBreaker(BreakerParams{QueueDepth: 1, MaxConcurrency: 10000000, InitialCapacity: 10000000}) b.Run("sequential", func(b *testing.B) { for j := 0; j < b.N; j++ { free, got := breaker.Reserve(context.Background()) op() if got { free() } } }) b.Run("parallel", func(b *testing.B) { b.RunParallel(func(pb *testing.PB) { for pb.Next() { free, got := breaker.Reserve(context.Background()) op() if got { free() } } }) }) } ================================================ FILE: pkg/proxy/handler.go ================================================ /* Copyright 2018 The Knative Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. Modifications Copyright 2022 Cortex Labs, Inc. */ package proxy import ( "context" "errors" "net/http" "github.com/cortexlabs/cortex/pkg/lib/telemetry" "github.com/cortexlabs/cortex/pkg/probe" ) func Handler(breaker *Breaker, next http.Handler) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { if probe.IsRequestKubeletProbe(r) || breaker == nil { next.ServeHTTP(w, r) return } if err := breaker.Maybe(r.Context(), func() { next.ServeHTTP(w, r) }); err != nil { if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, ErrRequestQueueFull) { http.Error(w, err.Error(), http.StatusServiceUnavailable) } else { w.WriteHeader(http.StatusInternalServerError) telemetry.Error(err) } } } } ================================================ FILE: pkg/proxy/handler_test.go ================================================ /* Copyright 2018 The Knative Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. Modifications Copyright 2022 Cortex Labs, Inc. */ package proxy_test import ( "context" "net/http" "net/http/httptest" "strings" "testing" "time" "github.com/cortexlabs/cortex/pkg/proxy" "github.com/stretchr/testify/require" ) const ( userContainerHost = "http://user-container.cortex.dev" ) func TestProxyHandlerQueueFull(t *testing.T) { // This test sends three requests of which one should fail immediately as the queue // saturates. resp := make(chan struct{}) blockHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { <-resp }) breaker := proxy.NewBreaker( proxy.BreakerParams{ QueueDepth: 1, MaxConcurrency: 1, InitialCapacity: 1, }, ) h := proxy.Handler(breaker, blockHandler) req := httptest.NewRequest(http.MethodGet, userContainerHost, nil) resps := make(chan *httptest.ResponseRecorder) for i := 0; i < 3; i++ { go func() { rec := httptest.NewRecorder() h(rec, req) resps <- rec }() } // One of the three requests fails and it should be the first we see since the others // are still held by the resp channel. failure := <-resps require.Equal(t, http.StatusServiceUnavailable, failure.Code) require.True(t, strings.Contains(failure.Body.String(), "pending request queue full")) // Allow the remaining requests to pass. close(resp) for i := 0; i < 2; i++ { res := <-resps require.Equal(t, http.StatusOK, res.Code) } } func TestProxyHandlerBreakerTimeout(t *testing.T) { // This test sends a request which will take a long time to complete. // Then another one with a very short context timeout. // Verifies that the second one fails with timeout. seen := make(chan struct{}) resp := make(chan struct{}) defer close(resp) // Allow all requests to pass through. blockHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { seen <- struct{}{} <-resp }) breaker := proxy.NewBreaker(proxy.BreakerParams{ QueueDepth: 1, MaxConcurrency: 1, InitialCapacity: 1, }) h := proxy.Handler(breaker, blockHandler) go func() { h(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, userContainerHost, nil)) }() // Wait until the first request has entered the handler. <-seen ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) defer cancel() rec := httptest.NewRecorder() h(rec, httptest.NewRequest(http.MethodGet, userContainerHost, nil).WithContext(ctx)) require.Equal(t, http.StatusServiceUnavailable, rec.Code) require.True(t, strings.Contains(rec.Body.String(), context.DeadlineExceeded.Error())) } ================================================ FILE: pkg/proxy/proxy.go ================================================ /* Copyright 2018 The Knative Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. Modifications Copyright 2022 Cortex Labs, Inc. */ package proxy import ( "net/http" "net/http/httputil" "net/url" ) // NewReverseProxy creates a new cortex base reverse proxy func NewReverseProxy(target string, maxIdle, maxIdlePerHost int) *httputil.ReverseProxy { targetURL, err := url.Parse(target) if err != nil { panic(err) } httpProxy := httputil.NewSingleHostReverseProxy(targetURL) httpProxy.Transport = buildHTTPTransport(maxIdle, maxIdlePerHost) return httpProxy } func buildHTTPTransport(maxIdle, maxIdlePerHost int) http.RoundTripper { transport := http.DefaultTransport.(*http.Transport).Clone() transport.DisableKeepAlives = false transport.MaxIdleConns = maxIdle transport.MaxIdleConnsPerHost = maxIdlePerHost transport.ForceAttemptHTTP2 = false transport.DisableCompression = true return transport } ================================================ FILE: pkg/proxy/proxy_test.go ================================================ /* Copyright 2018 The Knative Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. Modifications Copyright 2022 Cortex Labs, Inc. */ package proxy_test import ( "net/http" "net/http/httptest" "testing" "github.com/cortexlabs/cortex/pkg/proxy" "github.com/stretchr/testify/require" ) func TestNewReverseProxy(t *testing.T) { var isHandlerCalled bool var handler http.HandlerFunc = func(w http.ResponseWriter, r *http.Request) { isHandlerCalled = true } server := httptest.NewServer(handler) httpProxy := proxy.NewReverseProxy(server.URL, 1000, 1000) resp := httptest.NewRecorder() req := httptest.NewRequest(http.MethodPost, "http://user-container.cortex.dev", nil) httpProxy.ServeHTTP(resp, req) require.True(t, isHandlerCalled) } ================================================ FILE: pkg/proxy/request_stats.go ================================================ /* Copyright 2018 The Knative Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. Modifications Copyright 2022 Cortex Labs, Inc. */ package proxy import ( "net/http" "sync" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/client_golang/prometheus/promhttp" ) type RequestStats struct { sync.Mutex counts []int64 } func (s *RequestStats) Append(val int64) { s.Lock() defer s.Unlock() s.counts = append(s.counts, val) } func (s *RequestStats) GetAllAndDelete() []int64 { var output []int64 s.Lock() defer s.Unlock() output = s.counts s.counts = []int64{} return output } func (s *RequestStats) Report() RequestStatsReport { requestCounts := s.GetAllAndDelete() total := 0.0 if len(requestCounts) > 0 { for _, val := range requestCounts { total += float64(val) } total /= float64(len(requestCounts)) } return RequestStatsReport{AvgInFlight: total} } type RequestStatsReport struct { AvgInFlight float64 } type PrometheusStatsReporter struct { handler http.Handler inFlightRequests prometheus.Gauge } func NewPrometheusStatsReporter() *PrometheusStatsReporter { inFlightRequestsGauge := promauto.NewGauge(prometheus.GaugeOpts{ Name: "cortex_in_flight_requests", Help: "The number of in-flight requests for a cortex API", }) return &PrometheusStatsReporter{ handler: promhttp.Handler(), inFlightRequests: inFlightRequestsGauge, } } func (r *PrometheusStatsReporter) Report(stats RequestStatsReport) { r.inFlightRequests.Set(stats.AvgInFlight) } func (r *PrometheusStatsReporter) ServeHTTP(w http.ResponseWriter, req *http.Request) { r.handler.ServeHTTP(w, req) } ================================================ FILE: pkg/types/async/s3_paths.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package async import ( "fmt" ) func StoragePath(clusterUID, apiName string) string { return fmt.Sprintf("%s/workloads/%s", clusterUID, apiName) } func PayloadPath(storagePath string, requestID string) string { return fmt.Sprintf("%s/%s/payload", storagePath, requestID) } func HeadersPath(storagePath string, requestID string) string { return fmt.Sprintf("%s/%s/headers.json", storagePath, requestID) } func ResultPath(storagePath string, requestID string) string { return fmt.Sprintf("%s/%s/result.json", storagePath, requestID) } func StatusPrefixPath(storagePath string, requestID string) string { return fmt.Sprintf("%s/%s/status", storagePath, requestID) } func StatusPath(storagePath string, requestID string, status Status) string { return fmt.Sprintf("%s/%s", StatusPrefixPath(storagePath, requestID), status) } ================================================ FILE: pkg/types/async/status.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package async // Status is an enum type for workload status type Status string // Different possible workload status const ( StatusNotFound Status = "not_found" StatusFailed Status = "failed" StatusInProgress Status = "in_progress" StatusInQueue Status = "in_queue" StatusCompleted Status = "completed" ) func (status Status) String() string { return string(status) } func (status Status) Valid() bool { switch status { case StatusNotFound, StatusFailed, StatusInProgress, StatusInQueue, StatusCompleted: return true default: return false } } ================================================ FILE: pkg/types/clusterconfig/availability_zones.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package clusterconfig import ( "github.com/cortexlabs/cortex/pkg/lib/aws" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/sets/strset" s "github.com/cortexlabs/cortex/pkg/lib/strings" ) var _azBlacklist = strset.New("us-east-1e") func (cc *Config) setAvailabilityZones(awsClient *aws.Client) error { if len(cc.AvailabilityZones) == 0 { if err := cc.setDefaultAvailabilityZones(awsClient); err != nil { return err } return nil } if err := cc.validateUserAvailabilityZones(awsClient); err != nil { return err } return nil } func (cc *Config) setDefaultAvailabilityZones(awsClient *aws.Client) error { instanceTypes := strset.New() for _, ng := range cc.NodeGroups { instanceTypes.Add(ng.InstanceType) } instanceTypesSlice := instanceTypes.Slice() var zones strset.Set var err error if len(instanceTypesSlice) > 0 { zones, err = awsClient.ListSupportedAvailabilityZones(instanceTypesSlice[0], instanceTypesSlice[1:]...) } if len(zones) == 0 || err != nil { // Try without checking instance types zones, err = awsClient.ListAvailabilityZonesInRegion() if err != nil { return nil // Let eksctl choose the availability zones } } zones.Subtract(_azBlacklist) if len(zones) < 2 { return ErrorNotEnoughDefaultSupportedZones(awsClient.Region, zones, instanceTypesSlice[0], instanceTypesSlice[1:]...) } // See https://github.com/weaveworks/eksctl/blob/master/pkg/eks/api.go if awsClient.Region == "us-east-1" { zones.ShrinkSorted(2) } else { zones.ShrinkSorted(3) } cc.AvailabilityZones = zones.SliceSorted() return nil } func (cc *Config) validateUserAvailabilityZones(awsClient *aws.Client) error { allZones, err := awsClient.ListAvailabilityZonesInRegion() if err != nil { return nil // Skip validation } for _, userZone := range cc.AvailabilityZones { if !allZones.Has(userZone) { return ErrorInvalidAvailabilityZone(userZone, allZones, awsClient.Region) } } if len(cc.NodeGroups) > 0 { instanceTypes := strset.New() for _, ng := range cc.NodeGroups { instanceTypes.Add(ng.InstanceType) } instanceTypesSlice := instanceTypes.Slice() supportedZones, err := awsClient.ListSupportedAvailabilityZones(instanceTypesSlice[0], instanceTypesSlice[1:]...) if err != nil { // Skip validation instance-based validation supportedZones = strset.Difference(allZones, _azBlacklist) } for _, userZone := range cc.AvailabilityZones { if !supportedZones.Has(userZone) { return ErrorUnsupportedAvailabilityZone(userZone, instanceTypesSlice[0], instanceTypesSlice[1:]...) } } } return nil } func (cc *Config) validateSubnets(awsClient *aws.Client) error { if len(cc.Subnets) == 0 { return nil } allZones, err := awsClient.ListAvailabilityZonesInRegion() if err != nil { return nil // Skip validation } userZones := strset.New() for i, subnetConfig := range cc.Subnets { if !allZones.Has(subnetConfig.AvailabilityZone) { return errors.Wrap(ErrorInvalidAvailabilityZone(subnetConfig.AvailabilityZone, allZones, cc.Region), s.Index(i), AvailabilityZoneKey) } if userZones.Has(subnetConfig.AvailabilityZone) { return ErrorAvailabilityZoneSpecifiedTwice(subnetConfig.AvailabilityZone) } userZones.Add(subnetConfig.AvailabilityZone) } return nil } ================================================ FILE: pkg/types/clusterconfig/aws_policy.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package clusterconfig import ( "bytes" "encoding/json" "fmt" "text/template" "github.com/aws/aws-sdk-go/service/iam" "github.com/cortexlabs/cortex/pkg/lib/aws" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/pointer" ) func DefaultPolicyName(clusterName string, region string) string { return fmt.Sprintf("cortex-%s-%s", clusterName, region) } func DefaultPolicyARN(accountID string, clusterName string, region string) string { return fmt.Sprintf("arn:%s:iam::%s:policy/%s", aws.PartitionFromRegion(region), accountID, DefaultPolicyName(clusterName, region)) } var _cortexPolicy = ` { "Version": "2012-10-17", "Statement": [ { "Action": [ "sts:GetCallerIdentity", "ecr:GetAuthorizationToken", "ecr:BatchGetImage", "sqs:ListQueues", "ec2:DescribeSpotPriceHistory" ], "Effect": "Allow", "Resource": "*" }, { "Effect": "Allow", "Action": "sqs:*", "Resource": "arn:*:sqs:{{ .Region }}:{{ .AccountID }}:cx_*" }, { "Effect": "Allow", "Action": "s3:*", "Resource": "arn:*:s3:::{{ .Bucket }}" }, { "Effect": "Allow", "Action": "s3:*", "Resource": "arn:*:s3:::{{ .Bucket }}/*" }, { "Effect": "Allow", "Action": [ "logs:CreateLogStream", "logs:DescribeLogStreams", "logs:PutLogEvents", "logs:CreateLogGroup" ], "Resource": "arn:*:logs:{{ .Region }}:{{ .AccountID }}:log-group:{{ .LogGroup }}:*" }, { "Effect": "Allow", "Action": "logs:CreateLogGroup", "Resource": "arn:*:logs:{{ .Region }}:{{ .AccountID }}:log-group:{{ .LogGroup }}" } ] } ` type CortexPolicyTemplateArgs struct { ClusterName string LogGroup string Region string Bucket string AccountID string } func CreateDefaultPolicy(awsClient *aws.Client, args CortexPolicyTemplateArgs) error { policyName := DefaultPolicyName(args.ClusterName, args.Region) accountID, _, err := awsClient.GetCachedAccountID() if err != nil { return err } policyARN := DefaultPolicyARN(accountID, args.ClusterName, args.Region) policyTemplate, err := template.New("policy").Parse(_cortexPolicy) if err != nil { return errors.Wrap(err, "failed to parse aws policy template") } buf := &bytes.Buffer{} err = policyTemplate.Execute(buf, args) if err != nil { return errors.Wrap(err, "failed to execute aws policy template") } compactBuf := &bytes.Buffer{} err = json.Compact(compactBuf, buf.Bytes()) if err != nil { return errors.Wrap(err, "failed to parse and remove whitespace from aws policy json") } policyDocument := compactBuf.String() _, err = awsClient.IAM().CreatePolicy(&iam.CreatePolicyInput{ PolicyDocument: &policyDocument, PolicyName: &policyName, }) if err != nil { if aws.IsErrCode(err, iam.ErrCodeEntityAlreadyExistsException) { err := AddNewPolicyVersion(awsClient, policyARN, policyDocument) if err != nil { return errors.Wrap(err, "failed to create iam policy for cortex") } } else { return errors.Wrap(err, "failed to create iam policy for cortex") } } return nil } func AddNewPolicyVersion(awsClient *aws.Client, policyARN string, policyDocument string) error { policies, err := awsClient.IAM().ListPolicyVersions(&iam.ListPolicyVersionsInput{ PolicyArn: &policyARN, }) if err != nil { return err } if len(policies.Versions) == 0 { return errors.ErrorUnexpected("encountered a policy without any policy versions") } numPolicies := len(policies.Versions) oldestPolicy := *policies.Versions[0] for _, policyPtr := range policies.Versions { policy := *policyPtr if policy.CreateDate.Before(*oldestPolicy.CreateDate) && !*policy.IsDefaultVersion { oldestPolicy = policy } } // can only have a max of 5 versions, so delete the oldest non-default version before adding a new policy version if numPolicies > 4 { _, err := awsClient.IAM().DeletePolicyVersion(&iam.DeletePolicyVersionInput{ PolicyArn: &policyARN, VersionId: oldestPolicy.VersionId, }) if err != nil { return errors.WithStack(err) } } _, err = awsClient.IAM().CreatePolicyVersion(&iam.CreatePolicyVersionInput{ SetAsDefault: pointer.Bool(true), PolicyDocument: &policyDocument, PolicyArn: &policyARN, }) if err != nil { return errors.WithStack(err) } return nil } ================================================ FILE: pkg/types/clusterconfig/cluster_config.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package clusterconfig import ( "crypto/sha256" "encoding/hex" "fmt" "math" "net" "regexp" "strconv" "strings" "time" "github.com/aws/amazon-vpc-cni-k8s/pkg/awsutils" "github.com/aws/aws-sdk-go/service/cloudformation" "github.com/aws/aws-sdk-go/service/iam" "github.com/cortexlabs/cortex/pkg/consts" "github.com/cortexlabs/cortex/pkg/lib/aws" cr "github.com/cortexlabs/cortex/pkg/lib/configreader" "github.com/cortexlabs/cortex/pkg/lib/errors" libhash "github.com/cortexlabs/cortex/pkg/lib/hash" "github.com/cortexlabs/cortex/pkg/lib/k8s" libmath "github.com/cortexlabs/cortex/pkg/lib/math" "github.com/cortexlabs/cortex/pkg/lib/pointer" "github.com/cortexlabs/cortex/pkg/lib/sets/strset" "github.com/cortexlabs/cortex/pkg/lib/slices" libstr "github.com/cortexlabs/cortex/pkg/lib/strings" s "github.com/cortexlabs/cortex/pkg/lib/strings" "github.com/cortexlabs/cortex/pkg/lib/structs" "github.com/cortexlabs/yaml" ) const ( // MaxNodeGroups represents the max number of node groups in a cluster MaxNodeGroups = 100 // MaxNodesToAddOnClusterUp represents the max number of nodes to add on cluster up // Limited to 200 nodes (rounded down from 248 nodes) for two reasons: // // * To prevent overloading the API servers when the nodes are being added. // // * To prevent hitting the 500 targets per LB (when the cross-load balancing is enabled) limit (quota code L-B211E961); // 500 divided by 2 target listeners - 1 operator node - 1 prometheus node => 248 MaxNodesToAddOnClusterUp = 200 // MaxNodesToAddOnClusterConfigure represents the max number of nodes to add on cluster up/configure MaxNodesToAddOnClusterConfigure = 100 // ClusterNameTag is the tag used for storing a cluster's name in AWS resources ClusterNameTag = "cortex.dev/cluster-name" // SQSQueueDelimiter is the delimiter character used for naming cortex SQS queues (e.g. cx__b__) // In this case, _ was chosen to simplify the retrieval of information for the queue's name, // since the api naming scheme does not allow this character. SQSQueueDelimiter = "_" ) var ( _operatorNodeGroupInstanceType = "t3.medium" _maxNodeGroupLengthWithPrefix = 32 _maxNodeGroupLength = _maxNodeGroupLengthWithPrefix - len("cx-wd-") // or cx-ws- _maxInstancePools = 20 _defaultIAMPolicies = []string{"arn:aws:iam::aws:policy/AmazonS3FullAccess"} _invalidTagPrefixes = []string{"kubernetes.io/", "k8s.io/", "eksctl.", "alpha.eksctl.", "beta.eksctl.", "aws:", "Aws:", "aWs:", "awS:", "aWS:", "AwS:", "aWS:", "AWS:"} _smallestIOPSForIO1VolumeType = int64(100) _highestIOPSForIO1VolumeType = int64(64000) _smallestIOPSForGP3VolumeType = int64(3000) _highestIOPSForGP3VolumeType = int64(16000) _maxIOPSToVolumeSizeRatioForIO1 = int64(50) _maxIOPSToVolumeSizeRatioForGP3 = int64(500) _minIOPSToThroughputRatioForGP3 = int64(4) _minSubnetMask = 16 _maxSubnetMask = 24 // This regex is stricter than the actual S3 rules _strictS3BucketRegex = regexp.MustCompile(`^([a-z0-9])+(-[a-z0-9]+)*$`) ) type CoreConfig struct { ClusterName string `json:"cluster_name" yaml:"cluster_name"` Region string `json:"region" yaml:"region"` PrometheusInstanceType string `json:"prometheus_instance_type" yaml:"prometheus_instance_type"` ImageOperator string `json:"image_operator" yaml:"image_operator"` ImageControllerManager string `json:"image_controller_manager" yaml:"image_controller_manager"` ImageManager string `json:"image_manager" yaml:"image_manager"` ImageKubexit string `json:"image_kubexit" yaml:"image_kubexit"` ImageProxy string `json:"image_proxy" yaml:"image_proxy"` ImageActivator string `json:"image_activator" yaml:"image_activator"` ImageAutoscaler string `json:"image_autoscaler" yaml:"image_autoscaler"` ImageAsyncGateway string `json:"image_async_gateway" yaml:"image_async_gateway"` ImageEnqueuer string `json:"image_enqueuer" yaml:"image_enqueuer"` ImageDequeuer string `json:"image_dequeuer" yaml:"image_dequeuer"` ImageClusterAutoscaler string `json:"image_cluster_autoscaler" yaml:"image_cluster_autoscaler"` ImageMetricsServer string `json:"image_metrics_server" yaml:"image_metrics_server"` ImageNvidiaDevicePlugin string `json:"image_nvidia_device_plugin" yaml:"image_nvidia_device_plugin"` ImageNeuronDevicePlugin string `json:"image_neuron_device_plugin" yaml:"image_neuron_device_plugin"` ImageNeuronScheduler string `json:"image_neuron_scheduler" yaml:"image_neuron_scheduler"` ImageFluentBit string `json:"image_fluent_bit" yaml:"image_fluent_bit"` ImageIstioProxy string `json:"image_istio_proxy" yaml:"image_istio_proxy"` ImageIstioPilot string `json:"image_istio_pilot" yaml:"image_istio_pilot"` ImagePrometheus string `json:"image_prometheus" yaml:"image_prometheus"` ImagePrometheusConfigReloader string `json:"image_prometheus_config_reloader" yaml:"image_prometheus_config_reloader"` ImagePrometheusOperator string `json:"image_prometheus_operator" yaml:"image_prometheus_operator"` ImagePrometheusStatsDExporter string `json:"image_prometheus_statsd_exporter" yaml:"image_prometheus_statsd_exporter"` ImagePrometheusDCGMExporter string `json:"image_prometheus_dcgm_exporter" yaml:"image_prometheus_dcgm_exporter"` ImagePrometheusKubeStateMetrics string `json:"image_prometheus_kube_state_metrics" yaml:"image_prometheus_kube_state_metrics"` ImagePrometheusNodeExporter string `json:"image_prometheus_node_exporter" yaml:"image_prometheus_node_exporter"` ImageKubeRBACProxy string `json:"image_kube_rbac_proxy" yaml:"image_kube_rbac_proxy"` ImageGrafana string `json:"image_grafana" yaml:"image_grafana"` ImageEventExporter string `json:"image_event_exporter" yaml:"image_event_exporter"` NodeGroups []*NodeGroup `json:"node_groups" yaml:"node_groups"` Tags map[string]string `json:"tags" yaml:"tags"` AvailabilityZones []string `json:"availability_zones" yaml:"availability_zones"` SSLCertificateARN *string `json:"ssl_certificate_arn,omitempty" yaml:"ssl_certificate_arn,omitempty"` IAMPolicyARNs []string `json:"iam_policy_arns" yaml:"iam_policy_arns"` SubnetVisibility SubnetVisibility `json:"subnet_visibility" yaml:"subnet_visibility"` Subnets []*Subnet `json:"subnets,omitempty" yaml:"subnets,omitempty"` NATGateway NATGateway `json:"nat_gateway" yaml:"nat_gateway"` APILoadBalancerType LoadBalancerType `json:"api_load_balancer_type" yaml:"api_load_balancer_type"` APILoadBalancerScheme LoadBalancerScheme `json:"api_load_balancer_scheme" yaml:"api_load_balancer_scheme"` OperatorLoadBalancerScheme LoadBalancerScheme `json:"operator_load_balancer_scheme" yaml:"operator_load_balancer_scheme"` APILoadBalancerCIDRWhiteList []string `json:"api_load_balancer_cidr_white_list,omitempty" yaml:"api_load_balancer_cidr_white_list,omitempty"` OperatorLoadBalancerCIDRWhiteList []string `json:"operator_load_balancer_cidr_white_list,omitempty" yaml:"operator_load_balancer_cidr_white_list,omitempty"` VPCCIDR *string `json:"vpc_cidr,omitempty" yaml:"vpc_cidr,omitempty"` Telemetry bool `json:"telemetry" yaml:"telemetry"` } type ManagedConfig struct { // fields that must be set by Cortex CortexPolicyARN string `json:"cortex_policy_arn" yaml:"cortex_policy_arn"` AccountID string `json:"account_id" yaml:"account_id"` ClusterUID string `json:"cluster_uid" yaml:"cluster_uid"` Bucket string `json:"bucket" yaml:"bucket"` } type NodeGroup struct { Name string `json:"name" yaml:"name"` InstanceType string `json:"instance_type" yaml:"instance_type"` MinInstances int64 `json:"min_instances" yaml:"min_instances"` MaxInstances int64 `json:"max_instances" yaml:"max_instances"` Priority int64 `json:"priority" yaml:"priority"` InstanceVolumeSize int64 `json:"instance_volume_size" yaml:"instance_volume_size"` InstanceVolumeType VolumeType `json:"instance_volume_type" yaml:"instance_volume_type"` InstanceVolumeIOPS *int64 `json:"instance_volume_iops" yaml:"instance_volume_iops"` InstanceVolumeThroughput *int64 `json:"instance_volume_throughput" yaml:"instance_volume_throughput"` Spot bool `json:"spot" yaml:"spot"` SpotConfig *SpotConfig `json:"spot_config" yaml:"spot_config"` } // compares the supported updatable fields of a nodegroup func (ng *NodeGroup) HasChanged(old *NodeGroup) bool { return ng.MaxInstances != old.MaxInstances || ng.MinInstances != old.MinInstances || ng.Priority != old.Priority } func (ng *NodeGroup) UpdatePlan(old *NodeGroup) string { var changes []string if old.MinInstances != ng.MinInstances { changes = append(changes, fmt.Sprintf("%s %d->%d", MinInstancesKey, old.MinInstances, ng.MinInstances)) } if old.MaxInstances != ng.MaxInstances { changes = append(changes, fmt.Sprintf("%s %d->%d", MaxInstancesKey, old.MaxInstances, ng.MaxInstances)) } if old.Priority != ng.Priority { changes = append(changes, fmt.Sprintf("%s %d->%d", PriorityKey, old.Priority, ng.Priority)) } return fmt.Sprintf("nodegroup %s will be updated with the following changes: %s", ng.Name, s.StrsAnd(changes)) } type SpotConfig struct { InstanceDistribution []string `json:"instance_distribution" yaml:"instance_distribution"` OnDemandBaseCapacity *int64 `json:"on_demand_base_capacity" yaml:"on_demand_base_capacity"` OnDemandPercentageAboveBaseCapacity *int64 `json:"on_demand_percentage_above_base_capacity" yaml:"on_demand_percentage_above_base_capacity"` MaxPrice *float64 `json:"max_price" yaml:"max_price"` InstancePools *int64 `json:"instance_pools" yaml:"instance_pools"` } type Subnet struct { AvailabilityZone string `json:"availability_zone" yaml:"availability_zone"` SubnetID string `json:"subnet_id" yaml:"subnet_id"` } type Config struct { CoreConfig `yaml:",inline"` ManagedConfig `yaml:",inline"` } type OperatorMetadata struct { APIVersion string `json:"api_version" yaml:"api_version"` OperatorID string `json:"operator_id" yaml:"operator_id"` ClusterID string `json:"cluster_id" yaml:"cluster_id"` IsOperatorInCluster bool `json:"is_operator_in_cluster" yaml:"is_operator_in_cluster"` } type InternalConfig struct { Config // Populated by operator OperatorMetadata } // The bare minimum to identify a cluster type AccessConfig struct { ClusterName string `json:"cluster_name" yaml:"cluster_name"` Region string `json:"region" yaml:"region"` ImageManager string `json:"image_manager" yaml:"image_manager"` } type ConfigureChanges struct { NodeGroupsToAdd []string NodeGroupsToRemove []string NodeGroupsToUpdate []string EKSNodeGroupsToRemove []string // EKS node group names of (NodeGroupsToRemove ∩ Cortex-converted EKS node groups) ∪ (Cortex-converted EKS node groups - the new cluster config's nodegroups) FieldsToUpdate []string } func (c *ConfigureChanges) HasChanges() bool { return len(c.NodeGroupsToAdd)+len(c.NodeGroupsToRemove)+len(c.NodeGroupsToUpdate)+len(c.EKSNodeGroupsToRemove)+len(c.FieldsToUpdate) != 0 } // GetGhostEKSNodeGroups returns the set difference between EKSNodeGroupsToRemove and the EKS-converted NodeGroupsToRemove func (c *ConfigureChanges) GetGhostEKSNodeGroups() []string { if len(c.EKSNodeGroupsToRemove) <= len(c.NodeGroupsToRemove) { return nil } eksNodeGroupPrefix := "cx-wx-" var ghostEKSNodeGroups []string for _, eksNgToRemove := range c.EKSNodeGroupsToRemove { if !slices.HasString(c.NodeGroupsToRemove, eksNgToRemove[len(eksNodeGroupPrefix):]) { ghostEKSNodeGroups = append(ghostEKSNodeGroups, eksNgToRemove) } } return ghostEKSNodeGroups } // NewForFile initializes and validates the cluster config from the YAML config file func NewForFile(clusterConfigPath string) (*Config, error) { config := Config{} errs := cr.ParseYAMLFile(&config, FullConfigValidation, clusterConfigPath) if errors.HasError(errs) { return nil, errors.FirstError(errs...) } return &config, nil } func ValidateRegion(region string) error { if !aws.EKSSupportedRegions.Has(region) { return ErrorInvalidRegion(region) } return nil } func RegionValidator(region string) (string, error) { if err := ValidateRegion(region); err != nil { return "", err } return region, nil } func (cc *Config) DeepCopy() (Config, error) { deepCopied := Config{} err := structs.DeepCopy(&deepCopied, cc) if err != nil { return Config{}, err } return deepCopied, nil } func (cc *Config) Hash() (string, error) { bytes, err := yaml.Marshal(cc) if err != nil { return "", err } configHash := sha256.New() configHash.Write(bytes) return hex.EncodeToString(configHash.Sum(nil)), nil } var CoreConfigStructFieldValidations = []*cr.StructFieldValidation{ { StructField: "ClusterName", StringValidation: &cr.StringValidation{ Default: "cortex", MaxLength: 54, // leaves room for 8 char uniqueness string (and "-") for bucket name (63 chars max) MinLength: 3, Validator: validateClusterName, }, }, { StructField: "Region", StringValidation: &cr.StringValidation{ Required: true, MinLength: 1, Validator: RegionValidator, }, }, { StructField: "PrometheusInstanceType", StringValidation: &cr.StringValidation{ MinLength: 1, Default: "t3.medium", Validator: validatePrometheusInstanceType, }, }, { StructField: "Telemetry", BoolValidation: &cr.BoolValidation{ Default: true, }, }, { StructField: "ImageOperator", StringValidation: &cr.StringValidation{ Default: consts.DefaultRegistry() + "/operator:" + consts.CortexVersion, Validator: validateImageVersion, }, }, { StructField: "ImageControllerManager", StringValidation: &cr.StringValidation{ Default: consts.DefaultRegistry() + "/controller-manager:" + consts.CortexVersion, Validator: validateImageVersion, }, }, { StructField: "ImageManager", StringValidation: &cr.StringValidation{ Default: consts.DefaultRegistry() + "/manager:" + consts.CortexVersion, Validator: validateImageVersion, }, }, { StructField: "ImageKubexit", StringValidation: &cr.StringValidation{ Default: consts.DefaultRegistry() + "/kubexit:" + consts.CortexVersion, Validator: validateImageVersion, }, }, { StructField: "ImageProxy", StringValidation: &cr.StringValidation{ Default: consts.DefaultRegistry() + "/proxy:" + consts.CortexVersion, Validator: validateImageVersion, }, }, { StructField: "ImageActivator", StringValidation: &cr.StringValidation{ Default: consts.DefaultRegistry() + "/activator:" + consts.CortexVersion, Validator: validateImageVersion, }, }, { StructField: "ImageAutoscaler", StringValidation: &cr.StringValidation{ Default: consts.DefaultRegistry() + "/autoscaler:" + consts.CortexVersion, Validator: validateImageVersion, }, }, { StructField: "ImageAsyncGateway", StringValidation: &cr.StringValidation{ Default: consts.DefaultRegistry() + "/async-gateway:" + consts.CortexVersion, Validator: validateImageVersion, }, }, { StructField: "ImageEnqueuer", StringValidation: &cr.StringValidation{ Default: consts.DefaultRegistry() + "/enqueuer:" + consts.CortexVersion, Validator: validateImageVersion, }, }, { StructField: "ImageDequeuer", StringValidation: &cr.StringValidation{ Default: consts.DefaultRegistry() + "/dequeuer:" + consts.CortexVersion, Validator: validateImageVersion, }, }, { StructField: "ImageClusterAutoscaler", StringValidation: &cr.StringValidation{ Default: consts.DefaultRegistry() + "/cluster-autoscaler:" + consts.CortexVersion, Validator: validateImageVersion, }, }, { StructField: "ImageMetricsServer", StringValidation: &cr.StringValidation{ Default: consts.DefaultRegistry() + "/metrics-server:" + consts.CortexVersion, Validator: validateImageVersion, }, }, { StructField: "ImageNvidiaDevicePlugin", StringValidation: &cr.StringValidation{ Default: consts.DefaultRegistry() + "/nvidia-device-plugin:" + consts.CortexVersion, Validator: validateImageVersion, }, }, { StructField: "ImageNeuronDevicePlugin", StringValidation: &cr.StringValidation{ Default: consts.DefaultRegistry() + "/neuron-device-plugin:" + consts.CortexVersion, Validator: validateImageVersion, }, }, { StructField: "ImageNeuronScheduler", StringValidation: &cr.StringValidation{ Default: consts.DefaultRegistry() + "/neuron-scheduler:" + consts.CortexVersion, Validator: validateImageVersion, }, }, { StructField: "ImageFluentBit", StringValidation: &cr.StringValidation{ Default: consts.DefaultRegistry() + "/fluent-bit:" + consts.CortexVersion, Validator: validateImageVersion, }, }, { StructField: "ImageIstioProxy", StringValidation: &cr.StringValidation{ Default: consts.DefaultRegistry() + "/istio-proxy:" + consts.CortexVersion, Validator: validateImageVersion, }, }, { StructField: "ImageIstioPilot", StringValidation: &cr.StringValidation{ Default: consts.DefaultRegistry() + "/istio-pilot:" + consts.CortexVersion, Validator: validateImageVersion, }, }, { StructField: "ImagePrometheus", StringValidation: &cr.StringValidation{ Default: consts.DefaultRegistry() + "/prometheus:" + consts.CortexVersion, Validator: validateImageVersion, }, }, { StructField: "ImagePrometheusConfigReloader", StringValidation: &cr.StringValidation{ Default: consts.DefaultRegistry() + "/prometheus-config-reloader:" + consts.CortexVersion, Validator: validateImageVersion, }, }, { StructField: "ImagePrometheusOperator", StringValidation: &cr.StringValidation{ Default: consts.DefaultRegistry() + "/prometheus-operator:" + consts.CortexVersion, Validator: validateImageVersion, }, }, { StructField: "ImagePrometheusStatsDExporter", StringValidation: &cr.StringValidation{ Default: consts.DefaultRegistry() + "/prometheus-statsd-exporter:" + consts.CortexVersion, Validator: validateImageVersion, }, }, { StructField: "ImagePrometheusDCGMExporter", StringValidation: &cr.StringValidation{ Default: consts.DefaultRegistry() + "/prometheus-dcgm-exporter:" + consts.CortexVersion, Validator: validateImageVersion, }, }, { StructField: "ImagePrometheusKubeStateMetrics", StringValidation: &cr.StringValidation{ Default: consts.DefaultRegistry() + "/prometheus-kube-state-metrics:" + consts.CortexVersion, Validator: validateImageVersion, }, }, { StructField: "ImagePrometheusNodeExporter", StringValidation: &cr.StringValidation{ Default: consts.DefaultRegistry() + "/prometheus-node-exporter:" + consts.CortexVersion, Validator: validateImageVersion, }, }, { StructField: "ImageKubeRBACProxy", StringValidation: &cr.StringValidation{ Default: consts.DefaultRegistry() + "/kube-rbac-proxy:" + consts.CortexVersion, Validator: validateImageVersion, }, }, { StructField: "ImageGrafana", StringValidation: &cr.StringValidation{ Default: consts.DefaultRegistry() + "/grafana:" + consts.CortexVersion, Validator: validateImageVersion, }, }, { StructField: "ImageEventExporter", StringValidation: &cr.StringValidation{ Default: consts.DefaultRegistry() + "/event-exporter:" + consts.CortexVersion, Validator: validateImageVersion, }, }, { StructField: "NodeGroups", StructListValidation: &cr.StructListValidation{ AllowExplicitNull: true, TreatNullAsEmpty: true, StructValidation: nodeGroupsFieldValidation, }, }, { StructField: "Tags", StringMapValidation: &cr.StringMapValidation{ AllowExplicitNull: true, AllowEmpty: true, ConvertNullToEmpty: true, KeyStringValidator: &cr.StringValidation{ MinLength: 1, MaxLength: 127, DisallowLeadingWhitespace: true, DisallowTrailingWhitespace: true, InvalidPrefixes: _invalidTagPrefixes, AWSTag: true, }, ValueStringValidator: &cr.StringValidation{ MinLength: 1, MaxLength: 255, DisallowLeadingWhitespace: true, DisallowTrailingWhitespace: true, InvalidPrefixes: _invalidTagPrefixes, AWSTag: true, }, }, }, { StructField: "SSLCertificateARN", StringPtrValidation: &cr.StringPtrValidation{ AllowExplicitNull: true, }, }, { StructField: "IAMPolicyARNs", StringListValidation: &cr.StringListValidation{ Default: _defaultIAMPolicies, AllowEmpty: true, AllowExplicitNull: true, }, }, { StructField: "AvailabilityZones", StringListValidation: &cr.StringListValidation{ AllowEmpty: true, AllowExplicitNull: true, DisallowDups: true, InvalidLengths: []int{1}, }, }, { StructField: "SubnetVisibility", StringValidation: &cr.StringValidation{ AllowedValues: SubnetVisibilityStrings(), Default: PublicSubnetVisibility.String(), }, Parser: func(str string) (interface{}, error) { return SubnetVisibilityFromString(str), nil }, }, { StructField: "Subnets", StructListValidation: &cr.StructListValidation{ AllowExplicitNull: true, MinLength: 2, StructValidation: &cr.StructValidation{ StructFieldValidations: []*cr.StructFieldValidation{ { StructField: "AvailabilityZone", StringValidation: &cr.StringValidation{}, }, { StructField: "SubnetID", StringValidation: &cr.StringValidation{}, }, }, }, }, }, { StructField: "NATGateway", StringValidation: &cr.StringValidation{ AllowedValues: NATGatewayStrings(), }, Parser: func(str string) (interface{}, error) { return NATGatewayFromString(str), nil }, DefaultDependentFields: []string{"SubnetVisibility", "Subnets"}, DefaultDependentFieldsFunc: func(vals []interface{}) interface{} { subnetVisibility := vals[0].(SubnetVisibility) subnets := vals[1].([]*Subnet) if len(subnets) > 0 { return NoneNATGateway.String() } if subnetVisibility == PublicSubnetVisibility { return NoneNATGateway.String() } return SingleNATGateway.String() }, }, { StructField: "APILoadBalancerType", StringValidation: &cr.StringValidation{ AllowedValues: LoadBalancerTypeStrings(), Default: NLBLoadBalancerType.String(), }, Parser: func(str string) (interface{}, error) { return LoadBalancerTypeFromString(str), nil }, }, { StructField: "APILoadBalancerScheme", StringValidation: &cr.StringValidation{ AllowedValues: LoadBalancerSchemeStrings(), Default: InternetFacingLoadBalancerScheme.String(), }, Parser: func(str string) (interface{}, error) { return LoadBalancerSchemeFromString(str), nil }, }, { StructField: "APILoadBalancerCIDRWhiteList", StringListValidation: &cr.StringListValidation{ Validator: func(addresses []string) ([]string, error) { for i, address := range addresses { _, err := validateCIDR(address) if err != nil { return nil, errors.Wrap(err, fmt.Sprintf("index %d", i)) } } return addresses, nil }, }, }, { StructField: "OperatorLoadBalancerCIDRWhiteList", StringListValidation: &cr.StringListValidation{ Validator: func(addresses []string) ([]string, error) { for i, address := range addresses { _, err := validateCIDR(address) if err != nil { return nil, errors.Wrap(err, fmt.Sprintf("index %d", i)) } } return addresses, nil }, }, }, { StructField: "OperatorLoadBalancerScheme", StringValidation: &cr.StringValidation{ AllowedValues: LoadBalancerSchemeStrings(), Default: InternetFacingLoadBalancerScheme.String(), }, Parser: func(str string) (interface{}, error) { return LoadBalancerSchemeFromString(str), nil }, }, { StructField: "VPCCIDR", StringPtrValidation: &cr.StringPtrValidation{ Validator: validateVPCCIDR, }, }, } var ManagedConfigStructFieldValidations = []*cr.StructFieldValidation{ { StructField: "ClusterUID", StringValidation: &cr.StringValidation{ Default: "", AllowEmpty: true, TreatNullAsEmpty: true, }, }, { StructField: "Bucket", StringValidation: &cr.StringValidation{ Default: "", AllowEmpty: true, TreatNullAsEmpty: true, }, }, { StructField: "CortexPolicyARN", StringValidation: &cr.StringValidation{ Required: false, AllowEmpty: true, TreatNullAsEmpty: true, }, }, { StructField: "AccountID", StringValidation: &cr.StringValidation{ Required: false, AllowEmpty: true, TreatNullAsEmpty: true, }, }, } var nodeGroupsFieldValidation *cr.StructValidation = &cr.StructValidation{ StructFieldValidations: []*cr.StructFieldValidation{ { StructField: "Name", StringValidation: &cr.StringValidation{ Required: true, AlphaNumericDash: true, MaxLength: _maxNodeGroupLength, InvalidSuffixes: []string{"-"}, }, }, { StructField: "InstanceType", StringValidation: &cr.StringValidation{ Required: true, MinLength: 1, Validator: validateInstanceType, }, }, { StructField: "MinInstances", Int64Validation: &cr.Int64Validation{ Default: int64(1), GreaterThanOrEqualTo: pointer.Int64(0), }, }, { StructField: "MaxInstances", Int64Validation: &cr.Int64Validation{ Default: int64(5), GreaterThanOrEqualTo: pointer.Int64(0), // this will be validated to be > 0 during cluster up (can be scaled down later) }, }, { StructField: "Priority", Int64Validation: &cr.Int64Validation{ Default: int64(1), GreaterThanOrEqualTo: pointer.Int64(1), LessThanOrEqualTo: pointer.Int64(100), }, }, { StructField: "InstanceVolumeSize", Int64Validation: &cr.Int64Validation{ Default: 50, GreaterThanOrEqualTo: pointer.Int64(20), // large enough to fit docker images and any other overhead LessThanOrEqualTo: pointer.Int64(16384), }, }, { StructField: "InstanceVolumeType", StringValidation: &cr.StringValidation{ AllowedValues: VolumeTypesStrings(), Default: GP3VolumeType.String(), }, Parser: func(str string) (interface{}, error) { return VolumeTypeFromString(str), nil }, }, { StructField: "InstanceVolumeIOPS", Int64PtrValidation: &cr.Int64PtrValidation{ AllowExplicitNull: true, }, }, { StructField: "InstanceVolumeThroughput", Int64PtrValidation: &cr.Int64PtrValidation{ GreaterThanOrEqualTo: pointer.Int64(125), LessThanOrEqualTo: pointer.Int64(1000), AllowExplicitNull: true, }, }, { StructField: "Spot", BoolValidation: &cr.BoolValidation{ Default: false, }, }, { StructField: "SpotConfig", StructValidation: &cr.StructValidation{ DefaultNil: true, AllowExplicitNull: true, StructFieldValidations: []*cr.StructFieldValidation{ { StructField: "InstanceDistribution", StringListValidation: &cr.StringListValidation{ DisallowDups: true, Validator: validateInstanceDistribution, AllowExplicitNull: true, }, }, { StructField: "OnDemandBaseCapacity", Int64PtrValidation: &cr.Int64PtrValidation{ GreaterThanOrEqualTo: pointer.Int64(0), AllowExplicitNull: true, }, }, { StructField: "OnDemandPercentageAboveBaseCapacity", Int64PtrValidation: &cr.Int64PtrValidation{ GreaterThanOrEqualTo: pointer.Int64(0), LessThanOrEqualTo: pointer.Int64(100), AllowExplicitNull: true, }, }, { StructField: "MaxPrice", Float64PtrValidation: &cr.Float64PtrValidation{ GreaterThan: pointer.Float64(0), AllowExplicitNull: true, }, }, { StructField: "InstancePools", Int64PtrValidation: &cr.Int64PtrValidation{ GreaterThanOrEqualTo: pointer.Int64(1), LessThanOrEqualTo: pointer.Int64(int64(_maxInstancePools)), AllowExplicitNull: true, }, }, }, }, }, }, } var FullConfigValidation = &cr.StructValidation{ Required: true, StructFieldValidations: append(CoreConfigStructFieldValidations, ManagedConfigStructFieldValidations...), AllowExtraFields: false, } var AccessValidation = &cr.StructValidation{ AllowExtraFields: true, StructFieldValidations: []*cr.StructFieldValidation{ { StructField: "ClusterName", StringValidation: &cr.StringValidation{ Default: "cortex", MaxLength: 54, // leaves room for 8 char uniqueness string (and "-") for bucket name (63 chars max) MinLength: 3, Validator: validateClusterName, }, }, { StructField: "Region", StringValidation: &cr.StringValidation{ Required: true, MinLength: 1, Validator: RegionValidator, }, }, { StructField: "ImageManager", StringValidation: &cr.StringValidation{ Default: consts.DefaultRegistry() + "/manager:" + consts.CortexVersion, Validator: validateImageVersion, }, }, }, } func (cc *Config) ToAccessConfig() AccessConfig { return AccessConfig{ ClusterName: cc.ClusterName, Region: cc.Region, ImageManager: cc.ImageManager, } } func SQSNamePrefix(clusterName string) string { // 8 was chosen to make sure that other identifiers can be added to the full queue name before reaching the 80 char SQS name limit return "cx" + SQSQueueDelimiter + libhash.String(clusterName)[:8] + SQSQueueDelimiter } // SQSNamePrefix returns a string with the hash of cluster name and adds trailing "_" e.g. cx_abcd1234_ func (cc *CoreConfig) SQSNamePrefix() string { return SQSNamePrefix(cc.ClusterName) } func (cc *Config) validate(awsClient *aws.Client) error { if cc.APILoadBalancerType == NLBLoadBalancerType { isSupportedByNLB, err := aws.IsInstanceSupportedByNLB(cc.PrometheusInstanceType) if err != nil { return err } if !isSupportedByNLB { return errors.Wrap(ErrorInstanceTypeNotSupportedByCortex(cc.PrometheusInstanceType), PrometheusInstanceTypeKey) } } numNodeGroups := len(cc.NodeGroups) if numNodeGroups > MaxNodeGroups { return ErrorMaxNumOfNodeGroupsReached(MaxNodeGroups) } ngNames := []string{} instances := []aws.InstanceTypeRequests{ { InstanceType: _operatorNodeGroupInstanceType, RequiredOnDemandInstances: 1, }, { InstanceType: cc.PrometheusInstanceType, RequiredOnDemandInstances: 1, }, } for _, nodeGroup := range cc.NodeGroups { if !slices.HasString(ngNames, nodeGroup.Name) { ngNames = append(ngNames, nodeGroup.Name) } else { return errors.Wrap(ErrorDuplicateNodeGroupName(nodeGroup.Name), NodeGroupsKey) } err := nodeGroup.validateNodeGroup(awsClient, cc.Region, cc.APILoadBalancerType) if err != nil { return errors.Wrap(err, NodeGroupsKey, nodeGroup.Name) } instances = append(instances, aws.InstanceTypeRequests{ InstanceType: nodeGroup.InstanceType, RequiredOnDemandInstances: nodeGroup.MaxPossibleOnDemandInstances(), RequiredSpotInstances: nodeGroup.MaxPossibleSpotInstances(), }) } if err := awsClient.VerifyInstanceQuota(instances); err != nil { // Skip AWS errors, since some regions (e.g. eu-north-1) do not support this API if !aws.IsAWSError(err) { return errors.Wrap(err, NodeGroupsKey) } } if len(cc.AvailabilityZones) > 0 && len(cc.Subnets) > 0 { return ErrorSpecifyOneOrNone(AvailabilityZonesKey, SubnetsKey) } if len(cc.Subnets) > 0 && cc.NATGateway != NoneNATGateway { return ErrorNoNATGatewayWithSubnets() } if cc.SubnetVisibility == PrivateSubnetVisibility && cc.NATGateway == NoneNATGateway && len(cc.Subnets) == 0 { return ErrorNATRequiredWithPrivateSubnetVisibility() } accountID, _, err := awsClient.GetCachedAccountID() if err != nil { return err } if cc.AccountID != "" { return ErrorDisallowedField(AccountIDKey) } cc.AccountID = accountID if cc.Bucket != "" { return ErrorDisallowedField(BucketKey) } cc.Bucket = BucketName(accountID, cc.ClusterName, cc.Region) // check if the bucket already exists in a different region for some reason bucketRegion, _ := aws.GetBucketRegion(cc.Bucket) if bucketRegion != "" && bucketRegion != cc.Region { // if the bucket didn't exist, we will create it in the correct region, so there is no error return ErrorS3RegionDiffersFromCluster(cc.Bucket, bucketRegion, cc.Region) } if cc.CortexPolicyARN != "" { return ErrorDisallowedField(CortexPolicyARNKey) } cc.CortexPolicyARN = DefaultPolicyARN(accountID, cc.ClusterName, cc.Region) defaultPoliciesSet := strset.New(_defaultIAMPolicies...) for i := range cc.IAMPolicyARNs { policyARN := cc.IAMPolicyARNs[i] if defaultPoliciesSet.Has(policyARN) { partition := aws.PartitionFromRegion(cc.Region) adjustedPolicyARN := strings.Replace(policyARN, "arn:aws:", fmt.Sprintf("arn:%s:", partition), 1) cc.IAMPolicyARNs[i] = adjustedPolicyARN policyARN = adjustedPolicyARN } _, err := awsClient.IAM().GetPolicy(&iam.GetPolicyInput{ PolicyArn: pointer.String(policyARN), }) if err != nil { if aws.IsErrCode(err, iam.ErrCodeNoSuchEntityException) { return errors.Wrap(ErrorIAMPolicyARNNotFound(policyARN), IAMPolicyARNsKey) } return errors.Wrap(err, IAMPolicyARNsKey) } } if cc.SSLCertificateARN != nil { exists, err := awsClient.DoesCertificateExist(*cc.SSLCertificateARN) if err != nil { return errors.Wrap(err, SSLCertificateARNKey) } if !exists { return errors.Wrap(ErrorSSLCertificateARNNotFound(*cc.SSLCertificateARN, cc.Region), SSLCertificateARNKey) } } for tagName, tagValue := range cc.Tags { if strings.HasPrefix(tagName, "cortex.dev/") { if tagName != ClusterNameTag { return errors.Wrap(cr.ErrorCantHavePrefix(tagName, "cortex.dev/"), TagsKey) } if tagValue != cc.ClusterName { return errors.Wrap(ErrorCantOverrideDefaultTag(), TagsKey) } } } cc.Tags[ClusterNameTag] = cc.ClusterName if len(cc.Subnets) > 0 { if err := cc.validateSubnets(awsClient); err != nil { return errors.Wrap(err, SubnetsKey) } } else { if err := cc.setAvailabilityZones(awsClient); err != nil { return errors.Wrap(err, AvailabilityZonesKey) } } return nil } func (cc *Config) validateTopLevelSectionDiff(oldConfig Config) ([]string, error) { var fieldsToUpdate []string // validate actionable changes newClusterConfigCopy, err := cc.DeepCopy() if err != nil { return nil, err } oldClusterConfigCopy, err := oldConfig.DeepCopy() if err != nil { return nil, err } if libstr.Obj(newClusterConfigCopy.SSLCertificateARN) != libstr.Obj(oldClusterConfigCopy.SSLCertificateARN) { fieldsToUpdate = append(fieldsToUpdate, SSLCertificateARNKey) } if libstr.Obj(newClusterConfigCopy.APILoadBalancerCIDRWhiteList) != libstr.Obj(oldClusterConfigCopy.APILoadBalancerCIDRWhiteList) { fieldsToUpdate = append(fieldsToUpdate, APILoadBalancerCIDRWhiteListKey) } if libstr.Obj(newClusterConfigCopy.OperatorLoadBalancerCIDRWhiteList) != libstr.Obj(oldClusterConfigCopy.OperatorLoadBalancerCIDRWhiteList) { fieldsToUpdate = append(fieldsToUpdate, OperatorLoadBalancerCIDRWhiteListKey) } clearUpdatableFields(&newClusterConfigCopy) clearUpdatableFields(&oldClusterConfigCopy) h1, err := newClusterConfigCopy.Hash() if err != nil { return nil, err } h2, err := oldClusterConfigCopy.Hash() if err != nil { return nil, err } if h1 != h2 { return nil, ErrorConfigCannotBeChangedOnConfigure() } return fieldsToUpdate, nil } func clearUpdatableFields(clusterConfig *Config) { clusterConfig.SSLCertificateARN = nil clusterConfig.APILoadBalancerCIDRWhiteList = nil clusterConfig.OperatorLoadBalancerCIDRWhiteList = nil clusterConfig.NodeGroups = []*NodeGroup{} } func (cc *Config) validateSharedNodeGroupsDiff(oldConfig Config) error { sharedNgsFromNewConfig, sharedNgsFromOldConfig := cc.getCommonNodeGroups(oldConfig) for i := range sharedNgsFromNewConfig { newNgCopy, err := sharedNgsFromNewConfig[i].DeepCopy() if err != nil { return errors.Wrap(err, NodeGroupsKey) } oldNgCopy, err := sharedNgsFromOldConfig[i].DeepCopy() if err != nil { return errors.Wrap(err, NodeGroupsKey) } newNgCopy.MinInstances = 0 newNgCopy.MaxInstances = 0 newNgCopy.Priority = 0 oldNgCopy.MinInstances = 0 oldNgCopy.MaxInstances = 0 oldNgCopy.Priority = 0 newHash, err := newNgCopy.Hash() if err != nil { return errors.Wrap(err, NodeGroupsKey) } oldHash, err := oldNgCopy.Hash() if err != nil { return errors.Wrap(err, NodeGroupsKey) } if newHash != oldHash { return errors.Wrap(ErrorNodeGroupCanOnlyBeScaled(), NodeGroupsKey, newNgCopy.Name) } } return nil } func (cc *Config) validateNodeAdditionRate(k8sClient *k8s.Client) error { workloadNodes, err := k8sClient.ListNodesByLabel("workload", "true") if err != nil { return err } totalCurrentNodes := int64(len(workloadNodes)) totalRequestedNodes := getTotalMinInstances(cc.NodeGroups) if totalRequestedNodes-totalCurrentNodes > MaxNodesToAddOnClusterConfigure { return ErrorMaxNodesToAddOnClusterConfigure(totalRequestedNodes, totalCurrentNodes, MaxNodesToAddOnClusterConfigure) } return nil } // this validates the user-provided cluster config func (cc *Config) ValidateOnInstall(awsClient *aws.Client) error { fmt.Print("verifying your configuration ...\n\n") err := cc.validate(awsClient) if err != nil { return err } requestedTotalMinInstances := getTotalMinInstances(cc.NodeGroups) if requestedTotalMinInstances > MaxNodesToAddOnClusterUp { return errors.Wrap(ErrorMaxNodesToAddOnClusterUp(requestedTotalMinInstances, MaxNodesToAddOnClusterUp), NodeGroupsKey) } // setting max_instances to 0 during cluster creation is not permitted (but scaling max_instances to 0 afterwards is allowed) for _, nodeGroup := range cc.NodeGroups { if nodeGroup != nil && nodeGroup.MaxInstances == 0 { return errors.Wrap(ErrorNodeGroupMaxInstancesIsZero(), NodeGroupsKey, nodeGroup.Name) } } if cc.ClusterUID != "" { return ErrorDisallowedField(ClusterUIDKey) } cc.ClusterUID = strconv.FormatInt(time.Now().Unix(), 10) var requiredVPCs int if len(cc.Subnets) == 0 { requiredVPCs = 1 } longestCIDRWhiteList := libmath.MaxInt(len(cc.APILoadBalancerCIDRWhiteList), len(cc.OperatorLoadBalancerCIDRWhiteList)) if err := VerifyNetworkQuotas(awsClient, 1, cc.NATGateway != NoneNATGateway, cc.NATGateway == HighlyAvailableNATGateway, requiredVPCs, strset.FromSlice(cc.AvailabilityZones), len(cc.NodeGroups), len(cc.NodeGroups), longestCIDRWhiteList, false); err != nil { // Skip AWS errors, since some regions (e.g. eu-north-1) do not support this API if !aws.IsAWSError(err) { return err } } return nil } func (cc *Config) ValidateOnConfigure(awsClient *aws.Client, k8sClient *k8s.Client, oldConfig Config, eksNodeGroupStacks []*cloudformation.StackSummary) (ConfigureChanges, error) { fmt.Print("verifying your configuration ...\n\n") cc.ClusterUID = oldConfig.ClusterUID err := cc.validate(awsClient) if err != nil { return ConfigureChanges{}, err } fieldsToUpdate, err := cc.validateTopLevelSectionDiff(oldConfig) if err != nil { return ConfigureChanges{}, err } err = cc.validateSharedNodeGroupsDiff(oldConfig) if err != nil { return ConfigureChanges{}, err } err = cc.validateNodeAdditionRate(k8sClient) if err != nil { return ConfigureChanges{}, errors.Wrap(err, NodeGroupsKey) } ngsToBeAdded := cc.getNewNodeGroups(oldConfig) ngsToBeRemoved := cc.getRemovedNodeGroups(oldConfig) tempMaxNodeGroupCount := len(cc.NodeGroups) + len(ngsToBeRemoved) tempNetAdditionOfNodeGroupCount := tempMaxNodeGroupCount - len(oldConfig.NodeGroups) longestCIDRWhiteList := libmath.MaxInt(len(cc.APILoadBalancerCIDRWhiteList), len(cc.OperatorLoadBalancerCIDRWhiteList)) if err := VerifyNetworkQuotasOnConfigure(awsClient, strset.FromSlice(cc.AvailabilityZones), tempMaxNodeGroupCount, tempNetAdditionOfNodeGroupCount, longestCIDRWhiteList); err != nil { // Skip AWS errors, since some regions (e.g. eu-north-1) do not support this API if !aws.IsAWSError(err) { return ConfigureChanges{}, errors.Wrap(err, NodeGroupsKey) } } sharedNgsFromNewConfig, sharedNgsFromOldConfig := cc.getCommonNodeGroups(oldConfig) ngsToBeUpdated := []*NodeGroup{} for i := range sharedNgsFromNewConfig { if sharedNgsFromNewConfig[i].HasChanged(sharedNgsFromOldConfig[i]) { ngsToBeUpdated = append(ngsToBeUpdated, sharedNgsFromNewConfig[i]) } } return ConfigureChanges{ NodeGroupsToAdd: GetNodeGroupNames(ngsToBeAdded), NodeGroupsToRemove: GetNodeGroupNames(ngsToBeRemoved), NodeGroupsToUpdate: GetNodeGroupNames(ngsToBeUpdated), EKSNodeGroupsToRemove: getStaleEksNodeGroups(cc.ClusterName, eksNodeGroupStacks, cc.NodeGroups, ngsToBeRemoved), FieldsToUpdate: fieldsToUpdate, }, nil } func (ng *NodeGroup) validateNodeGroup(awsClient *aws.Client, region string, loadBalancerType LoadBalancerType) error { if ng.MinInstances > ng.MaxInstances { return ErrorMinInstancesGreaterThanMax(ng.MinInstances, ng.MaxInstances) } primaryInstanceType := ng.InstanceType if loadBalancerType == NLBLoadBalancerType { isPrimaryInstanceSupportedByNLB, err := aws.IsInstanceSupportedByNLB(primaryInstanceType) if err != nil { return err } if !isPrimaryInstanceSupportedByNLB { return errors.Wrap(ErrorInstanceTypeNotSupportedByCortex(primaryInstanceType), InstanceTypeKey) } } if !aws.InstanceTypes[region].Has(primaryInstanceType) { return errors.Wrap(ErrorInstanceTypeNotSupportedInRegion(primaryInstanceType, region), InstanceTypeKey) } if _, ok := aws.InstanceMetadatas[region][primaryInstanceType]; !ok { return errors.Wrap(ErrorInstanceTypeNotSupportedByCortex(primaryInstanceType), InstanceTypeKey) } // throw error if IOPS defined for other storage than io1/gp3 if ng.InstanceVolumeType != IO1VolumeType && ng.InstanceVolumeType != GP3VolumeType && ng.InstanceVolumeIOPS != nil { return ErrorIOPSNotSupported(ng.InstanceVolumeType) } // throw error if throughput defined for other storage than gp3 if ng.InstanceVolumeType != GP3VolumeType && ng.InstanceVolumeThroughput != nil { return ErrorThroughputNotSupported(ng.InstanceVolumeType) } if ng.InstanceVolumeType == GP3VolumeType && ((ng.InstanceVolumeIOPS != nil && ng.InstanceVolumeThroughput == nil) || (ng.InstanceVolumeIOPS == nil && ng.InstanceVolumeThroughput != nil)) { return ErrorSpecifyTwoOrNone(InstanceVolumeIOPSKey, InstanceVolumeThroughputKey) } if ng.InstanceVolumeIOPS != nil { if ng.InstanceVolumeType == IO1VolumeType { if *ng.InstanceVolumeIOPS < _smallestIOPSForIO1VolumeType { return ErrorIOPSTooSmall(ng.InstanceVolumeType, *ng.InstanceVolumeIOPS, _smallestIOPSForIO1VolumeType) } if *ng.InstanceVolumeIOPS > _highestIOPSForIO1VolumeType { return ErrorIOPSTooLarge(ng.InstanceVolumeType, *ng.InstanceVolumeIOPS, _highestIOPSForIO1VolumeType) } if *ng.InstanceVolumeIOPS > ng.InstanceVolumeSize*_maxIOPSToVolumeSizeRatioForIO1 { return ErrorIOPSToVolumeSizeRatio(ng.InstanceVolumeType, _maxIOPSToVolumeSizeRatioForIO1, *ng.InstanceVolumeIOPS, ng.InstanceVolumeSize) } } else { if *ng.InstanceVolumeIOPS < _smallestIOPSForGP3VolumeType { return ErrorIOPSTooSmall(ng.InstanceVolumeType, *ng.InstanceVolumeIOPS, _smallestIOPSForGP3VolumeType) } if *ng.InstanceVolumeIOPS > _highestIOPSForGP3VolumeType { return ErrorIOPSTooLarge(ng.InstanceVolumeType, *ng.InstanceVolumeIOPS, _highestIOPSForGP3VolumeType) } if *ng.InstanceVolumeIOPS > ng.InstanceVolumeSize*_maxIOPSToVolumeSizeRatioForGP3 { return ErrorIOPSToVolumeSizeRatio(ng.InstanceVolumeType, _maxIOPSToVolumeSizeRatioForGP3, *ng.InstanceVolumeIOPS, ng.InstanceVolumeSize) } iopsToThroughputRatio := float64(*ng.InstanceVolumeIOPS) / float64(*ng.InstanceVolumeThroughput) if iopsToThroughputRatio < float64(_minIOPSToThroughputRatioForGP3) { return ErrorIOPSToThroughputRatio(ng.InstanceVolumeType, _minIOPSToThroughputRatioForGP3, *ng.InstanceVolumeIOPS, *ng.InstanceVolumeThroughput) } } } else if ng.InstanceVolumeType == GP3VolumeType { ng.InstanceVolumeIOPS = pointer.Int64(3000) ng.InstanceVolumeThroughput = pointer.Int64(125) } else if ng.InstanceVolumeType == IO1VolumeType { ng.InstanceVolumeIOPS = pointer.Int64(libmath.MinInt64(ng.InstanceVolumeSize*_maxIOPSToVolumeSizeRatioForIO1, 3000)) } if ng.Spot { ng.FillEmptySpotFields(region) primaryInstance := aws.InstanceMetadatas[region][primaryInstanceType] for _, instanceType := range ng.SpotConfig.InstanceDistribution { if instanceType == primaryInstanceType { continue } if !aws.InstanceTypes[region].Has(instanceType) { return errors.Wrap(ErrorInstanceTypeNotSupportedInRegion(instanceType, region), SpotConfigKey, InstanceDistributionKey) } if loadBalancerType == NLBLoadBalancerType { isSecondaryInstanceSupportedByNLB, err := aws.IsInstanceSupportedByNLB(primaryInstanceType) if err != nil { return err } if !isSecondaryInstanceSupportedByNLB { return errors.Wrap(ErrorInstanceTypeNotSupportedByCortex(primaryInstanceType), SpotConfigKey, InstanceDistributionKey) } } if _, ok := aws.InstanceMetadatas[region][instanceType]; !ok { return errors.Wrap(ErrorInstanceTypeNotSupportedByCortex(instanceType), SpotConfigKey, InstanceDistributionKey) } instanceMetadata := aws.InstanceMetadatas[region][instanceType] err := CheckSpotInstanceCompatibility(primaryInstance, instanceMetadata) if err != nil { return errors.Wrap(err, SpotConfigKey, InstanceDistributionKey) } spotInstancePrice, awsErr := awsClient.SpotInstancePrice(instanceMetadata.Type) if awsErr == nil { if err := CheckSpotInstancePriceCompatibility(primaryInstance, instanceMetadata, ng.SpotConfig.MaxPrice, spotInstancePrice); err != nil { return errors.Wrap(err, SpotConfigKey, InstanceDistributionKey) } } } if ng.SpotConfig.OnDemandBaseCapacity != nil && *ng.SpotConfig.OnDemandBaseCapacity > ng.MaxInstances { return ErrorOnDemandBaseCapacityGreaterThanMax(*ng.SpotConfig.OnDemandBaseCapacity, ng.MaxInstances) } } else { if ng.SpotConfig != nil { return ErrorConfiguredWhenSpotIsNotEnabled(SpotConfigKey) } } return nil } func (cc *Config) GetNodeGroupByName(name string) *NodeGroup { for _, ng := range cc.NodeGroups { if ng.Name == name { return ng } } return nil } func (cc *Config) getNewNodeGroups(oldConfig Config) []*NodeGroup { var newNodeGroups []*NodeGroup for _, updatingNg := range cc.NodeGroups { isNewNg := true for _, previousNg := range oldConfig.NodeGroups { if previousNg.Name == updatingNg.Name { isNewNg = false break } } if isNewNg { ngCopy := *updatingNg newNodeGroups = append(newNodeGroups, &ngCopy) } } return newNodeGroups } func (cc *Config) getRemovedNodeGroups(oldConfig Config) []*NodeGroup { var removedNodeGroups []*NodeGroup for _, previousNg := range oldConfig.NodeGroups { isRemovedNg := true for _, updatingNg := range cc.NodeGroups { if previousNg.Name == updatingNg.Name { isRemovedNg = false break } } if isRemovedNg { ngCopy := *previousNg removedNodeGroups = append(removedNodeGroups, &ngCopy) } } return removedNodeGroups } func (cc *Config) getCommonNodeGroups(oldConfig Config) ([]*NodeGroup, []*NodeGroup) { var commonNewNodeGroups []*NodeGroup var commonOldNodeGroups []*NodeGroup for _, previousNg := range oldConfig.NodeGroups { for _, updatingNg := range cc.NodeGroups { if previousNg.Name == updatingNg.Name { ngNewCopy := *updatingNg ngOldCopy := *previousNg commonNewNodeGroups = append(commonNewNodeGroups, &ngNewCopy) commonOldNodeGroups = append(commonOldNodeGroups, &ngOldCopy) break } } } return commonNewNodeGroups, commonOldNodeGroups } func getTotalMinInstances(nodeGroups []*NodeGroup) int64 { totalMinInstances := int64(0) for _, ng := range nodeGroups { totalMinInstances += ng.MinInstances } return totalMinInstances } func GetNodeGroupNames(nodeGroups []*NodeGroup) []string { ngNames := make([]string, len(nodeGroups)) for i := range nodeGroups { ngNames[i] = nodeGroups[i].Name } return ngNames } func CheckSpotInstanceCompatibility(target aws.InstanceMetadata, suggested aws.InstanceMetadata) error { if target.Inf > 0 && suggested.Inf == 0 { return ErrorIncompatibleSpotInstanceTypeInf(suggested) } if target.GPU > suggested.GPU { return ErrorIncompatibleSpotInstanceTypeGPU(target, suggested) } if target.Memory.Cmp(suggested.Memory) > 0 { return ErrorIncompatibleSpotInstanceTypeMemory(target, suggested) } if target.CPU.Cmp(suggested.CPU) > 0 { return ErrorIncompatibleSpotInstanceTypeCPU(target, suggested) } return nil } func CheckSpotInstancePriceCompatibility(target aws.InstanceMetadata, suggested aws.InstanceMetadata, maxPrice *float64, spotInstancePrice float64) error { if (maxPrice == nil || *maxPrice == target.Price) && target.Price < spotInstancePrice { return ErrorSpotPriceGreaterThanTargetOnDemand(spotInstancePrice, target, suggested) } if maxPrice != nil && *maxPrice < spotInstancePrice { return ErrorSpotPriceGreaterThanMaxPrice(spotInstancePrice, *maxPrice, suggested) } return nil } func AutoGenerateSpotConfig(spotConfig *SpotConfig, region string, instanceType string) { primaryInstance := aws.InstanceMetadatas[region][instanceType] cleanedDistribution := []string{instanceType} for _, spotInstance := range spotConfig.InstanceDistribution { if spotInstance != instanceType { cleanedDistribution = append(cleanedDistribution, spotInstance) } } spotConfig.InstanceDistribution = cleanedDistribution if spotConfig.MaxPrice == nil { spotConfig.MaxPrice = &primaryInstance.Price } if spotConfig.OnDemandBaseCapacity == nil { spotConfig.OnDemandBaseCapacity = pointer.Int64(0) } if spotConfig.OnDemandPercentageAboveBaseCapacity == nil { spotConfig.OnDemandPercentageAboveBaseCapacity = pointer.Int64(0) } if spotConfig.InstancePools == nil { if len(spotConfig.InstanceDistribution) < _maxInstancePools { spotConfig.InstancePools = pointer.Int64(int64(len(spotConfig.InstanceDistribution))) } else { spotConfig.InstancePools = pointer.Int64(int64(_maxInstancePools)) } } } func (ng *NodeGroup) FillEmptySpotFields(region string) { if ng.SpotConfig == nil { ng.SpotConfig = &SpotConfig{} } AutoGenerateSpotConfig(ng.SpotConfig, region, ng.InstanceType) } func validateCIDR(cidr string) (string, error) { _, _, err := net.ParseCIDR(cidr) if err != nil { return "", errors.WithStack(err) } return cidr, nil } func validateVPCCIDR(cidr string) (string, error) { _, network, err := net.ParseCIDR(cidr) if err != nil { return "", errors.WithStack(err) } if network != nil { maskSize, _ := network.Mask.Size() if maskSize < _minSubnetMask || maskSize > _maxSubnetMask { return "", ErrorSubnetMaskOutOfRange(maskSize, _minSubnetMask, _maxSubnetMask) } } return cidr, nil } func validateInstanceType(instanceType string) (string, error) { if err := aws.CheckValidInstanceType(instanceType); err != nil { return "", err } parsedType, err := aws.ParseInstanceType(instanceType) if err != nil { return "", err } if parsedType.Size == "nano" || parsedType.Size == "micro" { return "", ErrorInstanceTypeTooSmall(instanceType) } isAMDGPU, err := aws.IsAMDGPUInstance(instanceType) if err != nil { return "", err } if isAMDGPU { return "", ErrorAMDGPUInstancesNotSupported(instanceType) } isMac, err := aws.IsMacInstance(instanceType) if err != nil { return "", err } if isMac { return "", ErrorMacInstancesNotSupported(instanceType) } isFPGA, err := aws.IsFPGAInstance(instanceType) if err != nil { return "", err } if isFPGA { return "", ErrorFPGAInstancesNotSupported(instanceType) } isAlevo, err := aws.IsAlevoInstance(instanceType) if err != nil { return "", err } if isAlevo { return "", ErrorAlevoInstancesNotSupported(instanceType) } isGaudi, err := aws.IsGaudiInstance(instanceType) if err != nil { return "", err } if isGaudi { return "", ErrorGaudiInstancesNotSupported(instanceType) } isTrainium, err := aws.IsTrainiumInstance(instanceType) if err != nil { return "", err } if isTrainium { return "", ErrorTrainiumInstancesNotSupported(instanceType) } if _, ok := awsutils.InstanceNetworkingLimits[instanceType]; !ok { return "", ErrorInstanceTypeNotSupportedByCortex(instanceType) } return instanceType, nil } func validatePrometheusInstanceType(instanceType string) (string, error) { _, err := validateInstanceType(instanceType) if err != nil { return "", err } isGPU, err := aws.IsGPUInstance(instanceType) if err != nil { return "", err } if isGPU { return "", ErrorGPUInstancesNotSupported(instanceType) } isInf, err := aws.IsInferentiaInstance(instanceType) if err != nil { return "", err } if isInf { return "", ErrorInferentiaInstancesNotSupported(instanceType) } return instanceType, nil } func validateInstanceDistribution(instances []string) ([]string, error) { for _, instance := range instances { _, err := validateInstanceType(instance) if err != nil { return nil, err } } return instances, nil } func (ng *NodeGroup) DeepCopy() (NodeGroup, error) { deepCopied := NodeGroup{} err := structs.DeepCopy(&deepCopied, ng) if err != nil { return NodeGroup{}, err } return deepCopied, nil } func (ng *NodeGroup) Hash() (string, error) { bytes, err := yaml.Marshal(ng) if err != nil { return "", err } hash := sha256.New() hash.Write(bytes) return hex.EncodeToString(hash.Sum(nil)), nil } func (ng *NodeGroup) MaxPossibleOnDemandInstances() int64 { if !ng.Spot || ng.SpotConfig == nil { return ng.MaxInstances } onDemandBaseCap, onDemandPctAboveBaseCap := ng.SpotConfigOnDemandValues() return onDemandBaseCap + int64(math.Ceil(float64(onDemandPctAboveBaseCap)/100*float64(ng.MaxInstances-onDemandBaseCap))) } func (ng *NodeGroup) MaxPossibleSpotInstances() int64 { if !ng.Spot { return 0 } if ng.SpotConfig == nil { return ng.MaxInstances } onDemandBaseCap, onDemandPctAboveBaseCap := ng.SpotConfigOnDemandValues() return ng.MaxInstances - onDemandBaseCap - int64(math.Floor(float64(onDemandPctAboveBaseCap)/100*float64(ng.MaxInstances-onDemandBaseCap))) } func (ng *NodeGroup) SpotConfigOnDemandValues() (int64, int64) { // default OnDemandBaseCapacity is 0 var onDemandBaseCapacity int64 = 0 if ng.SpotConfig.OnDemandBaseCapacity != nil { onDemandBaseCapacity = *ng.SpotConfig.OnDemandBaseCapacity } // default OnDemandPercentageAboveBaseCapacity is 0 var onDemandPercentageAboveBaseCapacity int64 = 0 if ng.SpotConfig.OnDemandPercentageAboveBaseCapacity != nil { onDemandPercentageAboveBaseCapacity = *ng.SpotConfig.OnDemandPercentageAboveBaseCapacity } return onDemandBaseCapacity, onDemandPercentageAboveBaseCapacity } func doesStackExist(stack *cloudformation.StackSummary) bool { if stack == nil || stack.StackName == nil || slices.HasString([]string{ cloudformation.StackStatusDeleteComplete, cloudformation.StackStatusDeleteInProgress, }, *stack.StackStatus) { return false } return true } func getStaleEksNodeGroups(clusterName string, eksNodeGroupStacks []*cloudformation.StackSummary, ngsToExist, ngsMarkedForRemoval []*NodeGroup) []string { eksNodeGroupsToRemove := strset.New() for _, ng := range ngsMarkedForRemoval { lifecycle := "d" if ng.Spot { lifecycle = "s" } eksNgName := fmt.Sprintf("cx-w%s-%s", lifecycle, ng.Name) eksStackName := fmt.Sprintf("eksctl-%s-nodegroup-cx-w%s-%s", clusterName, lifecycle, ng.Name) for _, eksNgStack := range eksNodeGroupStacks { if eksNgStack == nil || eksNgStack.StackName == nil { continue } if *eksNgStack.StackName == eksStackName { eksNodeGroupsToRemove.Add(eksNgName) break } } } for _, eksNgStack := range eksNodeGroupStacks { if !doesStackExist(eksNgStack) { continue } foundNg := false for _, ng := range ngsToExist { lifecycle := "d" if ng.Spot { lifecycle = "s" } eksStackName := fmt.Sprintf("eksctl-%s-nodegroup-cx-w%s-%s", clusterName, lifecycle, ng.Name) if *eksNgStack.StackName == eksStackName { foundNg = true break } } if !foundNg { eksNgName := (*eksNgStack.StackName)[len(fmt.Sprintf("eksctl-%s-nodegroup-", clusterName)):] eksNodeGroupsToRemove.Add(eksNgName) } } return eksNodeGroupsToRemove.Slice() } func (cc *CoreConfig) TelemetryEvent() map[string]interface{} { event := make(map[string]interface{}) if cc.ClusterName != "cortex" { event["cluster_name._is_custom"] = true } event["region"] = cc.Region event["prometheus_instance_type"] = cc.PrometheusInstanceType if !strings.HasPrefix(cc.ImageOperator, "quay.io/cortexlabs/") { event["image_operator._is_custom"] = true } if !strings.HasPrefix(cc.ImageControllerManager, "quay.io/cortexlabs/") { event["image_operator_controller_manager._is_custom"] = true } if !strings.HasPrefix(cc.ImageManager, "quay.io/cortexlabs/") { event["image_manager._is_custom"] = true } if !strings.HasPrefix(cc.ImageKubexit, "quay.io/cortexlabs/") { event["image_kubexit._is_custom"] = true } if !strings.HasPrefix(cc.ImageProxy, "quay.io/cortexlabs/") { event["image_proxy._is_custom"] = true } if !strings.HasPrefix(cc.ImageActivator, "quay.io/cortexlabs/") { event["image_activator._is_custom"] = true } if !strings.HasPrefix(cc.ImageAutoscaler, "quay.io/cortexlabs/") { event["image_autoscaler._is_custom"] = true } if !strings.HasPrefix(cc.ImageAsyncGateway, "quay.io/cortexlabs/") { event["image_async_gateway._is_custom"] = true } if !strings.HasPrefix(cc.ImageEnqueuer, "quay.io/cortexlabs/") { event["image_enqueuer._is_custom"] = true } if !strings.HasPrefix(cc.ImageDequeuer, "quay.io/cortexlabs/") { event["image_dequeuer._is_custom"] = true } if !strings.HasPrefix(cc.ImageClusterAutoscaler, "quay.io/cortexlabs/") { event["image_cluster_autoscaler._is_custom"] = true } if !strings.HasPrefix(cc.ImageMetricsServer, "quay.io/cortexlabs/") { event["image_metrics_server._is_custom"] = true } if !strings.HasPrefix(cc.ImageNvidiaDevicePlugin, "quay.io/cortexlabs/") { event["image_nvidia_device_plugin._is_custom"] = true } if !strings.HasPrefix(cc.ImageNeuronDevicePlugin, "quay.io/cortexlabs/") { event["image_neuron_device_plugin._is_custom"] = true } if !strings.HasPrefix(cc.ImageNeuronScheduler, "quay.io/cortexlabs/") { event["image_neuron_scheduler._is_custom"] = true } if !strings.HasPrefix(cc.ImageFluentBit, "quay.io/cortexlabs/") { event["image_fluent_bit._is_custom"] = true } if !strings.HasPrefix(cc.ImageIstioProxy, "quay.io/cortexlabs/") { event["image_istio_proxy._is_custom"] = true } if !strings.HasPrefix(cc.ImageIstioPilot, "quay.io/cortexlabs/") { event["image_istio_pilot._is_custom"] = true } if strings.HasPrefix(cc.ImagePrometheus, "quay.io/cortexlabs/") { event["image_prometheus._is_custom"] = true } if strings.HasPrefix(cc.ImagePrometheusConfigReloader, "quay.io/cortexlabs/") { event["image_prometheus_config_reloader._is_custom"] = true } if strings.HasPrefix(cc.ImagePrometheusOperator, "quay.io/cortexlabs/") { event["image_prometheus_operator._is_custom"] = true } if strings.HasPrefix(cc.ImagePrometheusStatsDExporter, "quay.io/cortexlabs/") { event["image_prometheus_statsd_exporter._is_custom"] = true } if strings.HasPrefix(cc.ImagePrometheusDCGMExporter, "quay.io/cortexlabs/") { event["image_prometheus_dcgm_exporter._is_custom"] = true } if strings.HasPrefix(cc.ImagePrometheusKubeStateMetrics, "quay.io/cortexlabs/") { event["image_prometheus_kube_state_metrics._is_custom"] = true } if strings.HasPrefix(cc.ImagePrometheusNodeExporter, "quay.io/cortexlabs/") { event["image_prometheus_node_exporter._is_custom"] = true } if strings.HasPrefix(cc.ImageKubeRBACProxy, "quay.io/cortexlabs/") { event["image_kube_rbac_proxy._is_custom"] = true } if strings.HasPrefix(cc.ImageGrafana, "quay.io/cortexlabs/") { event["image_grafana._is_custom"] = true } if strings.HasPrefix(cc.ImageEventExporter, "quay.io/cortexlabs/") { event["image_event_exporter._is_custom"] = true } if len(cc.Tags) > 0 { event["tags._is_defined"] = true event["tags._len"] = len(cc.Tags) } if len(cc.AvailabilityZones) > 0 { event["availability_zones._is_defined"] = true event["availability_zones._len"] = len(cc.AvailabilityZones) event["availability_zones"] = cc.AvailabilityZones } if len(cc.Subnets) > 0 { event["subnets._is_defined"] = true event["subnets._len"] = len(cc.Subnets) event["subnets"] = cc.Subnets } if cc.SSLCertificateARN != nil { event["ssl_certificate_arn._is_defined"] = true } // CortexPolicyARN should be managed by cortex if !strset.New(_defaultIAMPolicies...).IsEqual(strset.New(cc.IAMPolicyARNs...)) { event["iam_policy_arns._is_custom"] = true } event["iam_policy_arns._len"] = len(cc.IAMPolicyARNs) event["subnet_visibility"] = cc.SubnetVisibility event["nat_gateway"] = cc.NATGateway event["api_load_balancer_type"] = cc.APILoadBalancerType event["api_load_balancer_scheme"] = cc.APILoadBalancerScheme event["operator_load_balancer_scheme"] = cc.OperatorLoadBalancerScheme if cc.VPCCIDR != nil { event["vpc_cidr._is_defined"] = true } onDemandInstanceTypes := strset.New() spotInstanceTypes := strset.New() var totalMinSize, totalMaxSize int event["node_groups._len"] = len(cc.NodeGroups) for i, ng := range cc.NodeGroups { nodeGroupKey := func(field string) string { return fmt.Sprintf("node_groups.%d.%s", i, field) } event[nodeGroupKey("_is_defined")] = true event[nodeGroupKey("name")] = ng.Name event[nodeGroupKey("instance_type")] = ng.InstanceType event[nodeGroupKey("min_instances")] = ng.MinInstances event[nodeGroupKey("max_instances")] = ng.MaxInstances event[nodeGroupKey("priority")] = ng.Priority event[nodeGroupKey("instance_volume_size")] = ng.InstanceVolumeSize event[nodeGroupKey("instance_volume_type")] = ng.InstanceVolumeType if ng.InstanceVolumeIOPS != nil { event[nodeGroupKey("instance_volume_iops.is_defined")] = true event[nodeGroupKey("instance_volume_iops")] = *ng.InstanceVolumeIOPS } if ng.InstanceVolumeThroughput != nil { event[nodeGroupKey("instance_volume_throughput.is_defined")] = true event[nodeGroupKey("instance_volume_throughput")] = *ng.InstanceVolumeThroughput } event[nodeGroupKey("spot")] = ng.Spot if !ng.Spot { onDemandInstanceTypes.Add(ng.InstanceType) } else { spotInstanceTypes.Add(ng.InstanceType) } if ng.SpotConfig != nil { event[nodeGroupKey("spot_config._is_defined")] = true if len(ng.SpotConfig.InstanceDistribution) > 0 { event[nodeGroupKey("spot_config.instance_distribution._is_defined")] = true event[nodeGroupKey("spot_config.instance_distribution._len")] = len(ng.SpotConfig.InstanceDistribution) event[nodeGroupKey("spot_config.instance_distribution")] = ng.SpotConfig.InstanceDistribution spotInstanceTypes.Add(ng.SpotConfig.InstanceDistribution...) } if ng.SpotConfig.OnDemandBaseCapacity != nil { event[nodeGroupKey("spot_config.on_demand_base_capacity._is_defined")] = true event[nodeGroupKey("spot_config.on_demand_base_capacity")] = *ng.SpotConfig.OnDemandBaseCapacity } if ng.SpotConfig.OnDemandPercentageAboveBaseCapacity != nil { event[nodeGroupKey("spot_config.on_demand_percentage_above_base_capacity._is_defined")] = true event[nodeGroupKey("spot_config.on_demand_percentage_above_base_capacity")] = *ng.SpotConfig.OnDemandPercentageAboveBaseCapacity } if ng.SpotConfig.MaxPrice != nil { event[nodeGroupKey("spot_config.max_price._is_defined")] = true event[nodeGroupKey("spot_config.max_price")] = *ng.SpotConfig.MaxPrice } if ng.SpotConfig.InstancePools != nil { event[nodeGroupKey("spot_config.instance_pools._is_defined")] = true event[nodeGroupKey("spot_config.instance_pools")] = *ng.SpotConfig.InstancePools } } totalMinSize += int(ng.MinInstances) totalMaxSize += int(ng.MaxInstances) } event["node_groups._total_min_size"] = totalMinSize event["node_groups._total_max_size"] = totalMaxSize event["node_groups._on_demand_instances"] = onDemandInstanceTypes.Slice() event["node_groups._spot_instances"] = spotInstanceTypes.Slice() event["node_groups._instances"] = strset.Union(onDemandInstanceTypes, spotInstanceTypes).Slice() return event } func (cc *CoreConfig) GetNodeGroupByName(name string) *NodeGroup { for _, ng := range cc.NodeGroups { if ng.Name == name { matchedNodeGroup := *ng return &matchedNodeGroup } } return nil } func (cc *CoreConfig) GetNodeGroupNames() []string { allNodeGroupNames := make([]string, len(cc.NodeGroups)) for i := range cc.NodeGroups { allNodeGroupNames[i] = cc.NodeGroups[i].Name } return allNodeGroupNames } func BucketName(accountID, clusterName, region string) string { bucketID := libhash.String(accountID + region)[:8] // this is to "guarantee" a globally unique name return clusterName + "-" + bucketID } func validateClusterName(clusterName string) (string, error) { if !_strictS3BucketRegex.MatchString(clusterName) { return "", errors.Wrap(ErrorDidNotMatchStrictS3Regex(), clusterName) } return clusterName, nil } func validateImageVersion(image string) (string, error) { return cr.ValidateImageVersion(image, consts.CortexVersion) } ================================================ FILE: pkg/types/clusterconfig/config_key.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package clusterconfig const ( BucketKey = "bucket" ClusterUIDKey = "cluster_uid" ClusterNameKey = "cluster_name" RegionKey = "region" PrometheusInstanceTypeKey = "prometheus_instance_type" NodeGroupsKey = "node_groups" InstanceTypeKey = "instance_type" AcceleratorTypeKey = "accelerator_type" AcceleratorsPerInstanceKey = "accelerators_per_instance" MinInstancesKey = "min_instances" MaxInstancesKey = "max_instances" PriorityKey = "priority" SpotKey = "spot" SpotConfigKey = "spot_config" InstanceDistributionKey = "instance_distribution" OnDemandBaseCapacityKey = "on_demand_base_capacity" OnDemandPercentageAboveBaseCapacityKey = "on_demand_percentage_above_base_capacity" InstanceVolumeSizeKey = "instance_volume_size" InstanceVolumeTypeKey = "instance_volume_type" InstanceVolumeIOPSKey = "instance_volume_iops" InstanceVolumeThroughputKey = "instance_volume_throughput" InstancePoolsKey = "instance_pools" MaxPriceKey = "max_price" NetworkKey = "network" SubnetKey = "subnet" TagsKey = "tags" AvailabilityZonesKey = "availability_zones" SubnetsKey = "subnets" AvailabilityZoneKey = "availability_zone" SubnetIDKey = "subnet_id" SSLCertificateARNKey = "ssl_certificate_arn" CortexPolicyARNKey = "cortex_policy_arn" IAMPolicyARNsKey = "iam_policy_arns" SubnetVisibilityKey = "subnet_visibility" NATGatewayKey = "nat_gateway" APILoadBalancerSchemeKey = "api_load_balancer_scheme" OperatorLoadBalancerSchemeKey = "operator_load_balancer_scheme" APILoadBalancerCIDRWhiteListKey = "api_load_balancer_cidr_white_list" OperatorLoadBalancerCIDRWhiteListKey = "operator_load_balancer_cidr_white_list" VPCCIDRKey = "vpc_cidr" AccountIDKey = "account_id" TelemetryKey = "telemetry" ) ================================================ FILE: pkg/types/clusterconfig/errors.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package clusterconfig import ( "fmt" "strings" "github.com/cortexlabs/cortex/pkg/consts" "github.com/cortexlabs/cortex/pkg/lib/aws" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/sets/strset" s "github.com/cortexlabs/cortex/pkg/lib/strings" ) const ( ErrInvalidProvider = "clusterconfig.invalid_provider" ErrInvalidLegacyProvider = "clusterconfig.invalid_legacy_provider" ErrDisallowedField = "clusterconfig.disallowed_field" ErrInvalidRegion = "clusterconfig.invalid_region" ErrNodeGroupMaxInstancesIsZero = "clusterconfig.node_group_max_instances_is_zero" ErrMaxNumOfNodeGroupsReached = "clusterconfig.max_num_of_nodegroups_reached" ErrDuplicateNodeGroupName = "clusterconfig.duplicate_nodegroup_name" ErrMaxNodesToAddOnClusterUp = "clusterconfig.max_nodes_to_add_on_cluster_up" ErrMaxNodesToAddOnClusterConfigure = "clusterconfig.max_nodes_to_add_on_cluster_configure" ErrInstanceTypeTooSmall = "clusterconfig.instance_type_too_small" ErrMinInstancesGreaterThanMax = "clusterconfig.min_instances_greater_than_max" ErrInstanceTypeNotSupportedInRegion = "clusterconfig.instance_type_not_supported_in_region" ErrIncompatibleSpotInstanceTypeMemory = "clusterconfig.incompatible_spot_instance_type_memory" ErrIncompatibleSpotInstanceTypeCPU = "clusterconfig.incompatible_spot_instance_type_cpu" ErrIncompatibleSpotInstanceTypeGPU = "clusterconfig.incompatible_spot_instance_type_gpu" ErrIncompatibleSpotInstanceTypeInf = "clusterconfig.incompatible_spot_instance_type_inf" ErrSpotPriceGreaterThanTargetOnDemand = "clusterconfig.spot_price_greater_than_target_on_demand" ErrSpotPriceGreaterThanMaxPrice = "clusterconfig.spot_price_greater_than_max_price" ErrInstanceTypeNotSupportedByCortex = "clusterconfig.instance_type_not_supported_by_cortex" ErrAMDGPUInstancesNotSupported = "clusterconfig.amd_gpu_instances_not_supported" ErrGPUInstancesNotSupported = "clusterconfig.gpu_instance_not_supported" ErrInferentiaInstancesNotSupported = "clusterconfig.inferentia_instances_not_supported" ErrMacInstancesNotSupported = "clusterconfig.mac_instances_not_supported" ErrFPGAInstancesNotSupported = "clusterconfig.fpga_instances_not_supported" ErrAlevoInstancesNotSupported = "clusterconfig.alevo_instances_not_supported" ErrGaudiInstancesNotSupported = "clusterconfig.gaudi_instances_not_supported" ErrTrainiumInstancesNotSupported = "clusterconfig.trainium_instances_not_supported" ErrAtLeastOneInstanceDistribution = "clusterconfig.at_least_one_instance_distribution" ErrNoCompatibleSpotInstanceFound = "clusterconfig.no_compatible_spot_instance_found" ErrConfiguredWhenSpotIsNotEnabled = "clusterconfig.configured_when_spot_is_not_enabled" ErrOnDemandBaseCapacityGreaterThanMax = "clusterconfig.on_demand_base_capacity_greater_than_max" ErrInvalidAvailabilityZone = "clusterconfig.invalid_availability_zone" ErrAvailabilityZoneSpecifiedTwice = "clusterconfig.availability_zone_specified_twice" ErrUnsupportedAvailabilityZone = "clusterconfig.unsupported_availability_zone" ErrNotEnoughValidDefaultAvailibilityZones = "clusterconfig.not_enough_valid_default_availability_zones" ErrNoNATGatewayWithSubnets = "clusterconfig.no_nat_gateway_with_subnets" ErrSubnetMaskOutOfRange = "clusterconfig.subnet_mask_out_of_range" ErrConfigCannotBeChangedOnConfigure = "clusterconfig.config_cannot_be_changed_on_configure" ErrNodeGroupCanOnlyBeScaled = "clusterconfig.node_group_can_only_be_scaled" ErrSpecifyOneOrNone = "clusterconfig.specify_one_or_none" ErrSpecifyTwoOrNone = "clusterconfig.specify_two_or_none" ErrDependentFieldMustBeSpecified = "clusterconfig.dependent_field_must_be_specified" ErrFieldConfigurationDependentOnCondition = "clusterconfig.field_configuration_dependent_on_condition" ErrDidNotMatchStrictS3Regex = "clusterconfig.did_not_match_strict_s3_regex" ErrNATRequiredWithPrivateSubnetVisibility = "clusterconfig.nat_required_with_private_subnet_visibility" ErrS3RegionDiffersFromCluster = "clusterconfig.s3_region_differs_from_cluster" ErrIOPSNotSupported = "clusterconfig.iops_not_supported" ErrThroughputNotSupported = "clusterconfig.throughput_not_supported" ErrIOPSTooSmall = "clusterconfig.iops_too_small" ErrIOPSTooLarge = "clusterconfig.iops_too_large" ErrIOPSToVolumeSizeRatio = "clusterconfig.iops_to_volume_size_ratio" ErrIOPSToThroughputRatio = "clusterconfig.iops_to_throughput_ratio" ErrCantOverrideDefaultTag = "clusterconfig.cant_override_default_tag" ErrSSLCertificateARNNotFound = "clusterconfig.ssl_certificate_arn_not_found" ErrIAMPolicyARNNotFound = "clusterconfig.iam_policy_arn_not_found" ) func ErrorInvalidProvider(providerStr string) error { return errors.WithStack(&errors.Error{ Kind: ErrInvalidProvider, Message: fmt.Sprintf("\"%s\" is not a supported provider; only aws is supported, so the provider field may be removed from your cluster configuration file", providerStr), }) } func ErrorInvalidLegacyProvider(providerStr string) error { return errors.WithStack(&errors.Error{ Kind: ErrInvalidLegacyProvider, Message: fmt.Sprintf("the %s provider is no longer supported on cortex v%s; only aws is supported, so the provider field may be removed from your cluster configuration file", providerStr, consts.CortexVersionMinor), }) } func ErrorDisallowedField(field string) error { return errors.WithStack(&errors.Error{ Kind: ErrDisallowedField, Message: fmt.Sprintf("the %s field cannot be configured by the user", field), }) } func ErrorInvalidRegion(region string) error { return errors.WithStack(&errors.Error{ Kind: ErrInvalidRegion, Message: fmt.Sprintf("%s is not a valid AWS region, or is an AWS region which is not supported by AWS EKS; please choose one of the following regions: %s", s.UserStr(region), strings.Join(aws.EKSSupportedRegions.SliceSorted(), ", ")), }) } func ErrorNodeGroupMaxInstancesIsZero() error { return errors.WithStack(&errors.Error{ Kind: ErrNodeGroupMaxInstancesIsZero, Message: fmt.Sprintf("nodegroups cannot be created with `%s` set to 0 (but `%s` can be scaled to 0 after the nodegroup has been created)", MaxInstancesKey, MaxInstancesKey), }) } func ErrorMaxNumOfNodeGroupsReached(maxNodeGroups int64) error { return errors.WithStack(&errors.Error{ Kind: ErrMaxNumOfNodeGroupsReached, Message: fmt.Sprintf("cannot have more than %d nodegroups", maxNodeGroups), }) } func ErrorDuplicateNodeGroupName(duplicateNgName string) error { return errors.WithStack(&errors.Error{ Kind: ErrDuplicateNodeGroupName, Message: fmt.Sprintf("cannot have multiple nodegroups with the same name (%s)", duplicateNgName), }) } func ErrorMaxNodesToAddOnClusterUp(requestedNodes, maxNodes int64) error { return errors.WithStack(&errors.Error{ Kind: ErrMaxNodesToAddOnClusterUp, Message: fmt.Sprintf("cannot create a cluster with %d instances (at most %d instances can be created initially); reduce %s for your nodegroups (you may add additional instances via the `cortex cluster configure` command after your cluster has been created)", requestedNodes, maxNodes, MinInstancesKey), }) } func ErrorMaxNodesToAddOnClusterConfigure(requestedNodes, currentNodes, maxNodes int64) error { return errors.WithStack(&errors.Error{ Kind: ErrMaxNodesToAddOnClusterConfigure, Message: fmt.Sprintf("cannot add %d instances to your cluster (you requested %d total instances, but your cluster currently has %d instances); only %d instances can be added at time, so reduce the sum of %s across all nodegroups by %d", requestedNodes-currentNodes, requestedNodes, currentNodes, maxNodes, MinInstancesKey, requestedNodes-currentNodes-maxNodes), }) } func ErrorInstanceTypeTooSmall(instanceType string) error { return errors.WithStack(&errors.Error{ Kind: ErrInstanceTypeTooSmall, Message: fmt.Sprintf("%s: cortex does not support nano or micro instances - please specify a larger instance type", instanceType), }) } func ErrorMinInstancesGreaterThanMax(min int64, max int64) error { return errors.WithStack(&errors.Error{ Kind: ErrMinInstancesGreaterThanMax, Message: fmt.Sprintf("%s cannot be greater than %s (%d > %d)", MinInstancesKey, MaxInstancesKey, min, max), }) } func ErrorInstanceTypeNotSupportedInRegion(instanceType string, region string) error { return errors.WithStack(&errors.Error{ Kind: ErrInstanceTypeNotSupportedInRegion, Message: fmt.Sprintf("%s instances are not available in %s", instanceType, region), }) } func ErrorIncompatibleSpotInstanceTypeMemory(target aws.InstanceMetadata, suggested aws.InstanceMetadata) error { return errors.WithStack(&errors.Error{ Kind: ErrIncompatibleSpotInstanceTypeMemory, Message: fmt.Sprintf("all instances must have at least as much memory as %s (%s has %s memory, but %s only has %s memory)", target.Type, target.Type, target.Memory.String(), suggested.Type, suggested.Memory.String()), }) } func ErrorIncompatibleSpotInstanceTypeCPU(target aws.InstanceMetadata, suggested aws.InstanceMetadata) error { return errors.WithStack(&errors.Error{ Kind: ErrIncompatibleSpotInstanceTypeCPU, Message: fmt.Sprintf("all instances must have at least as much CPU as %s (%s has %s CPU, but %s only has %s CPU)", target.Type, target.Type, target.CPU.String(), suggested.Type, suggested.CPU.String()), }) } func ErrorIncompatibleSpotInstanceTypeGPU(target aws.InstanceMetadata, suggested aws.InstanceMetadata) error { return errors.WithStack(&errors.Error{ Kind: ErrIncompatibleSpotInstanceTypeGPU, Message: fmt.Sprintf("all instances must have at least as much GPU as %s (%s has %d GPU, but %s only has %d GPU)", target.Type, target.Type, target.GPU, suggested.Type, suggested.GPU), }) } func ErrorIncompatibleSpotInstanceTypeInf(suggested aws.InstanceMetadata) error { return errors.WithStack(&errors.Error{ Kind: ErrIncompatibleSpotInstanceTypeInf, Message: fmt.Sprintf("all instances must have at least 1 Inferentia chip, but %s doesn't have any", suggested.Type), }) } func ErrorSpotPriceGreaterThanTargetOnDemand(spotPrice float64, target aws.InstanceMetadata, suggested aws.InstanceMetadata) error { return errors.WithStack(&errors.Error{ Kind: ErrSpotPriceGreaterThanTargetOnDemand, Message: fmt.Sprintf("%s will not be allocated because its current spot price is $%g which is greater than %s's on-demand price of $%g", suggested.Type, spotPrice, target.Type, target.Price), }) } func ErrorSpotPriceGreaterThanMaxPrice(suggestedSpotPrice float64, maxPrice float64, suggested aws.InstanceMetadata) error { return errors.WithStack(&errors.Error{ Kind: ErrSpotPriceGreaterThanMaxPrice, Message: fmt.Sprintf("%s will not be allocated because its current spot price is $%g which is greater than the configured max price $%g", suggested.Type, suggestedSpotPrice, maxPrice), }) } func ErrorInstanceTypeNotSupportedByCortex(instanceType string) error { return errors.WithStack(&errors.Error{ Kind: ErrInstanceTypeNotSupportedByCortex, Message: fmt.Sprintf("instance type %s is not supported by cortex", instanceType), }) } func ErrorAMDGPUInstancesNotSupported(instanceType string) error { return errors.WithStack(&errors.Error{ Kind: ErrAMDGPUInstancesNotSupported, Message: fmt.Sprintf("AMD GPU instances (including %s) are not supported by cortex", instanceType), }) } func ErrorGPUInstancesNotSupported(instanceType string) error { return errors.WithStack(&errors.Error{ Kind: ErrGPUInstancesNotSupported, Message: fmt.Sprintf("GPU instances (including %s) are not supported", instanceType), }) } func ErrorInferentiaInstancesNotSupported(instanceType string) error { return errors.WithStack(&errors.Error{ Kind: ErrInferentiaInstancesNotSupported, Message: fmt.Sprintf("Inferentia instances (including %s) are not supported", instanceType), }) } func ErrorMacInstancesNotSupported(instanceType string) error { return errors.WithStack(&errors.Error{ Kind: ErrMacInstancesNotSupported, Message: fmt.Sprintf("mac instances (including %s) are not supported by cortex", instanceType), }) } func ErrorFPGAInstancesNotSupported(instanceType string) error { return errors.WithStack(&errors.Error{ Kind: ErrFPGAInstancesNotSupported, Message: fmt.Sprintf("FPGA instances (including %s) are not supported by cortex", instanceType), }) } func ErrorAlevoInstancesNotSupported(instanceType string) error { return errors.WithStack(&errors.Error{ Kind: ErrAlevoInstancesNotSupported, Message: fmt.Sprintf("Alevo instances (including %s) are not supported by cortex", instanceType), }) } func ErrorGaudiInstancesNotSupported(instanceType string) error { return errors.WithStack(&errors.Error{ Kind: ErrGaudiInstancesNotSupported, Message: fmt.Sprintf("Gaudi instances (including %s) are not supported by cortex", instanceType), }) } func ErrorTrainiumInstancesNotSupported(instanceType string) error { return errors.WithStack(&errors.Error{ Kind: ErrTrainiumInstancesNotSupported, Message: fmt.Sprintf("Trainium instances (including %s) are not supported by cortex", instanceType), }) } func ErrorConfiguredWhenSpotIsNotEnabled(configKey string) error { return errors.WithStack(&errors.Error{ Kind: ErrConfiguredWhenSpotIsNotEnabled, Message: fmt.Sprintf("%s cannot be specified unless spot is enabled (to enable spot instances, set `%s: true` in your cluster configuration file)", configKey, SpotKey), }) } func ErrorOnDemandBaseCapacityGreaterThanMax(onDemandBaseCapacity int64, max int64) error { return errors.WithStack(&errors.Error{ Kind: ErrOnDemandBaseCapacityGreaterThanMax, Message: fmt.Sprintf("%s cannot be greater than %s (%d > %d)", OnDemandBaseCapacityKey, MaxInstancesKey, onDemandBaseCapacity, max), }) } func ErrorInvalidAvailabilityZone(userZone string, allZones strset.Set, region string) error { return errors.WithStack(&errors.Error{ Kind: ErrInvalidAvailabilityZone, Message: fmt.Sprintf("%s is not an availability zone in %s; please choose from the following availability zones: %s", userZone, region, s.StrsOr(allZones.SliceSorted())), }) } func ErrorAvailabilityZoneSpecifiedTwice(zone string) error { return errors.WithStack(&errors.Error{ Kind: ErrAvailabilityZoneSpecifiedTwice, Message: fmt.Sprintf("availability zone \"%s\" is specified twice", zone), }) } func ErrorUnsupportedAvailabilityZone(userZone string, instanceType string, instanceTypes ...string) error { msg := fmt.Sprintf("the %s availability zone does not support EKS and the %s instance type; please choose a different availability zone, instance type, or region", userZone, instanceType) if len(instanceTypes) > 0 { allInstanceTypes := append([]string{instanceType}, instanceTypes...) msg = fmt.Sprintf("the %s availability zone does not support EKS and %s instance types; please choose a different availability zone, instance types, or region", userZone, s.StrsAnd(allInstanceTypes)) } return errors.WithStack(&errors.Error{ Kind: ErrUnsupportedAvailabilityZone, Message: msg, }) } func ErrorNotEnoughDefaultSupportedZones(region string, validZones strset.Set, instanceType string, instanceTypes ...string) error { areNoStr := "are no" if len(validZones) > 0 { areNoStr = "aren't enough" } msg := fmt.Sprintf("there %s availability zones in %s which support EKS and the %s instance type; please choose a different instance type or a different region", areNoStr, region, instanceType) if len(instanceTypes) > 0 { allInstanceTypes := append([]string{instanceType}, instanceTypes...) msg = fmt.Sprintf("there %s availability zones in %s which support EKS and the %s instance types; please choose different instance types or a different region", areNoStr, region, s.StrsAnd(allInstanceTypes)) } return errors.WithStack(&errors.Error{ Kind: ErrNotEnoughValidDefaultAvailibilityZones, Message: msg, }) } func ErrorNoNATGatewayWithSubnets() error { return errors.WithStack(&errors.Error{ Kind: ErrNoNATGatewayWithSubnets, Message: fmt.Sprintf("nat gateway cannot be automatically created when specifying subnets for your cluster; please unset %s or %s", NATGatewayKey, SubnetsKey), }) } func ErrorSubnetMaskOutOfRange(requestedMaskSize, minMaskSize, maxMaskSize int) error { return errors.WithStack(&errors.Error{ Kind: ErrSubnetMaskOutOfRange, Message: fmt.Sprintf("invalid network size /%d; the network size must be between /%d and /%d", requestedMaskSize, minMaskSize, maxMaskSize), }) } func ErrorConfigCannotBeChangedOnConfigure() error { return errors.WithStack(&errors.Error{ Kind: ErrConfigCannotBeChangedOnConfigure, Message: fmt.Sprintf("in a running cluster, only %s can be modified", s.StrsAnd([]string{NodeGroupsKey, SSLCertificateARNKey, OperatorLoadBalancerCIDRWhiteListKey, APILoadBalancerCIDRWhiteListKey})), }) } func ErrorNodeGroupCanOnlyBeScaled() error { return errors.WithStack(&errors.Error{ Kind: ErrNodeGroupCanOnlyBeScaled, Message: "in a running cluster, nodegroups can only be scaled, added, or deleted", }) } func ErrorSpecifyOneOrNone(fieldName1 string, fieldName2 string, fieldNames ...string) error { fieldNames = append([]string{fieldName1, fieldName2}, fieldNames...) message := fmt.Sprintf("specify exactly one or none of the following fields: %s", s.StrsAnd(fieldNames)) if len(fieldNames) == 2 { message = fmt.Sprintf("cannot specify both %s and %s; specify only one (or neither)", fieldNames[0], fieldNames[1]) } return errors.WithStack(&errors.Error{ Kind: ErrSpecifyOneOrNone, Message: message, }) } func ErrorSpecifyTwoOrNone(fieldName1 string, fieldName2 string, fieldNames ...string) error { fieldNames = append([]string{fieldName1, fieldName2}, fieldNames...) return errors.WithStack(&errors.Error{ Kind: ErrSpecifyTwoOrNone, Message: fmt.Sprintf("specify exactly two or none of the following fields: %s", s.StrsAnd(fieldNames)), }) } func ErrorDependentFieldMustBeSpecified(configuredField string, dependencyField string) error { return errors.WithStack(&errors.Error{ Kind: ErrDependentFieldMustBeSpecified, Message: fmt.Sprintf("%s must be specified when configuring %s", dependencyField, configuredField), }) } func ErrorFieldConfigurationDependentOnCondition(configuredField string, configuredFieldValue string, dependencyField string, dependencyFieldValue string) error { return errors.WithStack(&errors.Error{ Kind: ErrFieldConfigurationDependentOnCondition, Message: fmt.Sprintf("cannot set %s = %s when %s = %s", configuredField, configuredFieldValue, dependencyField, dependencyFieldValue), }) } func ErrorDidNotMatchStrictS3Regex() error { return errors.WithStack(&errors.Error{ Kind: ErrDidNotMatchStrictS3Regex, Message: "only lowercase alphanumeric characters and dashes are allowed, with no consecutive dashes and no leading or trailing dashes", }) } func ErrorNATRequiredWithPrivateSubnetVisibility() error { return errors.WithStack(&errors.Error{ Kind: ErrNATRequiredWithPrivateSubnetVisibility, Message: fmt.Sprintf("a nat gateway is required when `%s: %s` is specified; either set %s to %s or %s, or set %s to %s", SubnetVisibilityKey, PrivateSubnetVisibility, NATGatewayKey, s.UserStr(SingleNATGateway), s.UserStr(HighlyAvailableNATGateway), SubnetVisibilityKey, s.UserStr(PublicSubnetVisibility)), }) } func ErrorS3RegionDiffersFromCluster(bucketName string, bucketRegion string, clusterRegion string) error { return errors.WithStack(&errors.Error{ Kind: ErrS3RegionDiffersFromCluster, Message: fmt.Sprintf("the %s bucket already exists but is in %s (your cluster is in %s); either change the region of your cluster to %s or delete your bucket to allow cortex to create the bucket for you in %s", bucketName, bucketRegion, clusterRegion, bucketRegion, clusterRegion), }) } func ErrorIOPSNotSupported(volumeType VolumeType) error { return errors.WithStack(&errors.Error{ Kind: ErrIOPSNotSupported, Message: fmt.Sprintf("IOPS cannot be configured for volume type %s; set `%s: %s`, `%s: %s`, or remove `%s` from your cluster configuration file", volumeType, InstanceVolumeTypeKey, IO1VolumeType, InstanceVolumeTypeKey, GP3VolumeType, InstanceVolumeIOPSKey), }) } func ErrorThroughputNotSupported(volumeType VolumeType) error { return errors.WithStack(&errors.Error{ Kind: ErrThroughputNotSupported, Message: fmt.Sprintf("throughput cannot be configured for volume type %s; set `%s: %s` or remove `%s` from your cluster configuration file", volumeType, InstanceVolumeTypeKey, GP3VolumeType, InstanceVolumeThroughputKey), }) } func ErrorIOPSTooSmall(volumeType VolumeType, iops, minIOPS int64) error { return errors.WithStack(&errors.Error{ Kind: ErrIOPSTooSmall, Message: fmt.Sprintf("for %s volume type, %s (%d) cannot be smaller than %d", volumeType, InstanceVolumeIOPSKey, iops, minIOPS), }) } func ErrorIOPSTooLarge(volumeType VolumeType, iops, maxIOPS int64) error { return errors.WithStack(&errors.Error{ Kind: ErrIOPSTooLarge, Message: fmt.Sprintf("for %s volume type, %s (%d) cannot be larger than %d", volumeType, InstanceVolumeIOPSKey, iops, maxIOPS), }) } func ErrorIOPSToVolumeSizeRatio(volumeType VolumeType, ratio, iops int64, volumeSize int64) error { return errors.WithStack(&errors.Error{ Kind: ErrIOPSToVolumeSizeRatio, Message: fmt.Sprintf("for %s volume type, %s (%d) cannot be more than %d times larger than %s (%d); increase `%s` or decrease `%s` in your cluster configuration file", volumeType, InstanceVolumeIOPSKey, iops, ratio, InstanceVolumeSizeKey, volumeSize, InstanceVolumeSizeKey, InstanceVolumeIOPSKey), }) } func ErrorIOPSToThroughputRatio(volumeType VolumeType, ratio, iops, throughput int64) error { return errors.WithStack(&errors.Error{ Kind: ErrIOPSToThroughputRatio, Message: fmt.Sprintf("for %s volume type, %s (%d) must be at least %d times larger than %s (%d); decrease `%s` or increase `%s` in your cluster configuration file", volumeType, InstanceVolumeIOPSKey, iops, ratio, InstanceVolumeThroughputKey, throughput, InstanceVolumeThroughputKey, InstanceVolumeIOPSKey), }) } func ErrorCantOverrideDefaultTag() error { return errors.WithStack(&errors.Error{ Kind: ErrCantOverrideDefaultTag, Message: fmt.Sprintf("the \"%s\" tag cannot be overridden (it is set by default, and will always be equal to your cluster name)", ClusterNameTag), }) } func ErrorSSLCertificateARNNotFound(sslCertificateARN string, region string) error { return errors.WithStack(&errors.Error{ Kind: ErrSSLCertificateARNNotFound, Message: fmt.Sprintf("unable to find the specified ssl certificate in %s: %s", region, sslCertificateARN), }) } func ErrorIAMPolicyARNNotFound(policyARN string) error { return errors.WithStack(&errors.Error{ Kind: ErrIAMPolicyARNNotFound, Message: fmt.Sprintf("unable to find iam policy %s", policyARN), }) } ================================================ FILE: pkg/types/clusterconfig/load_balancer_scheme.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package clusterconfig type LoadBalancerScheme int const ( UnknownLoadBalancerScheme LoadBalancerScheme = iota InternetFacingLoadBalancerScheme InternalLoadBalancerScheme ) var _loadBalancerSchemes = []string{ "unknown", "internet-facing", "internal", } func LoadBalancerSchemeFromString(s string) LoadBalancerScheme { for i := 0; i < len(_loadBalancerSchemes); i++ { if s == _loadBalancerSchemes[i] { return LoadBalancerScheme(i) } } return UnknownLoadBalancerScheme } func LoadBalancerSchemeStrings() []string { return _loadBalancerSchemes[1:] } func (t LoadBalancerScheme) String() string { return _loadBalancerSchemes[t] } // MarshalText satisfies TextMarshaler func (t LoadBalancerScheme) MarshalText() ([]byte, error) { return []byte(t.String()), nil } // UnmarshalText satisfies TextUnmarshaler func (t *LoadBalancerScheme) UnmarshalText(text []byte) error { enum := string(text) for i := 0; i < len(_loadBalancerSchemes); i++ { if enum == _loadBalancerSchemes[i] { *t = LoadBalancerScheme(i) return nil } } *t = UnknownLoadBalancerScheme return nil } // UnmarshalBinary satisfies BinaryUnmarshaler // Needed for msgpack func (t *LoadBalancerScheme) UnmarshalBinary(data []byte) error { return t.UnmarshalText(data) } // MarshalBinary satisfies BinaryMarshaler func (t LoadBalancerScheme) MarshalBinary() ([]byte, error) { return []byte(t.String()), nil } ================================================ FILE: pkg/types/clusterconfig/load_balancer_type.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package clusterconfig type LoadBalancerType int const ( UnknownLoadBalancerType LoadBalancerType = iota NLBLoadBalancerType ELBLoadBalancerType ) var _loadBalancerTypes = []string{ "unknown", "nlb", "elb", } func LoadBalancerTypeFromString(s string) LoadBalancerType { for i := 0; i < len(_loadBalancerTypes); i++ { if s == _loadBalancerTypes[i] { return LoadBalancerType(i) } } return UnknownLoadBalancerType } func LoadBalancerTypeStrings() []string { return _loadBalancerTypes[1:] } func (t LoadBalancerType) String() string { return _loadBalancerTypes[t] } // MarshalText satisfies TextMarshaler func (t LoadBalancerType) MarshalText() ([]byte, error) { return []byte(t.String()), nil } // UnmarshalText satisfies TextUnmarshaler func (t *LoadBalancerType) UnmarshalText(text []byte) error { enum := string(text) for i := 0; i < len(_loadBalancerTypes); i++ { if enum == _loadBalancerTypes[i] { *t = LoadBalancerType(i) return nil } } *t = UnknownLoadBalancerType return nil } // UnmarshalBinary satisfies BinaryUnmarshaler // Needed for msgpack func (t *LoadBalancerType) UnmarshalBinary(data []byte) error { return t.UnmarshalText(data) } // MarshalBinary satisfies BinaryMarshaler func (t LoadBalancerType) MarshalBinary() ([]byte, error) { return []byte(t.String()), nil } ================================================ FILE: pkg/types/clusterconfig/nat_gateway_type.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package clusterconfig type NATGateway int const ( UnknownNATGateway NATGateway = iota NoneNATGateway SingleNATGateway HighlyAvailableNATGateway ) var _natGateways = []string{ "unknown", "none", "single", "highly_available", } func NATGatewayFromString(s string) NATGateway { for i := 0; i < len(_natGateways); i++ { if s == _natGateways[i] { return NATGateway(i) } } return UnknownNATGateway } func NATGatewayStrings() []string { return _natGateways[1:] } func (t NATGateway) String() string { return _natGateways[t] } // MarshalText satisfies TextMarshaler func (t NATGateway) MarshalText() ([]byte, error) { return []byte(t.String()), nil } // UnmarshalText satisfies TextUnmarshaler func (t *NATGateway) UnmarshalText(text []byte) error { enum := string(text) for i := 0; i < len(_natGateways); i++ { if enum == _natGateways[i] { *t = NATGateway(i) return nil } } *t = UnknownNATGateway return nil } // UnmarshalBinary satisfies BinaryUnmarshaler // Needed for msgpack func (t *NATGateway) UnmarshalBinary(data []byte) error { return t.UnmarshalText(data) } // MarshalBinary satisfies BinaryMarshaler func (t NATGateway) MarshalBinary() ([]byte, error) { return []byte(t.String()), nil } ================================================ FILE: pkg/types/clusterconfig/network_validations.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package clusterconfig import ( "fmt" "strings" "github.com/cortexlabs/cortex/pkg/lib/aws" "github.com/cortexlabs/cortex/pkg/lib/sets/strset" ) const ( _elasticIPsQuotaCode = "L-0263D0A3" // from EC2 service _internetGatewayQuotaCode = "L-A4707A72" // from EC2 service _natGatewayQuotaCode = "L-FE5A380F" // from EC2 service _vpcQuotaCode = "L-F678F1CE" // from VPC service _securityGroupsQuotaCode = "L-E79EC296" // from VPC service _securityGroupRulesQuotaCode = "L-0EA8095F" // from VPC service ) func VerifyNetworkQuotas( awsClient *aws.Client, requiredInternetGateways int, natGatewayRequired bool, highlyAvailableNATGateway bool, requiredVPCs int, availabilityZones strset.Set, numNodeGroups int, netAdditionOfNodeGroups int, longestCIDRWhiteList int, clusterAlreadyExists bool, ) error { desiredQuotaCodes := []string{ _vpcQuotaCode, _securityGroupsQuotaCode, _securityGroupRulesQuotaCode, } serviceCodes := []string{"vpc"} if !clusterAlreadyExists { desiredQuotaCodes = append(desiredQuotaCodes, _elasticIPsQuotaCode, _internetGatewayQuotaCode, _natGatewayQuotaCode, ) serviceCodes = append(serviceCodes, "ec2") } quotaCodeToValueMap, err := awsClient.ListServiceQuotas(desiredQuotaCodes, serviceCodes) if err != nil { return err } var skippedValidations []string defer func() { if len(skippedValidations) > 0 { fmt.Println(strings.Join(skippedValidations, "\n")) } }() if !clusterAlreadyExists { // check internet GW quota if requiredInternetGateways > 0 { if internetGatewayQuota, found := quotaCodeToValueMap[_internetGatewayQuotaCode]; found { err := awsClient.VerifyInternetGatewayQuota(internetGatewayQuota, requiredInternetGateways) if err != nil { return err } } else { skippedValidations = append(skippedValidations, fmt.Sprintf("skipping internet gateway quota verification: unable to find internet gateway quota (%s)", _internetGatewayQuotaCode)) } } // check nat GW quota if natGatewayRequired { if natGatewayQuota, found := quotaCodeToValueMap[_natGatewayQuotaCode]; found { err := awsClient.VerifyNATGatewayQuota(natGatewayQuota, availabilityZones, highlyAvailableNATGateway) if err != nil { return err } } else { skippedValidations = append(skippedValidations, fmt.Sprintf("skipping nat gateway quota verification: unable to find nat gateway quota (%s)\n", _natGatewayQuotaCode)) } } // check EIP quota if natGatewayRequired { if eipQuota, found := quotaCodeToValueMap[_elasticIPsQuotaCode]; found { err := awsClient.VerifyEIPQuota(eipQuota, availabilityZones, highlyAvailableNATGateway) if err != nil { return err } } else { skippedValidations = append(skippedValidations, fmt.Sprintf("skipping elastic ip quota verification: unable to find elastic ip quota (%s)\n", _elasticIPsQuotaCode)) } } // check required VPC quota if requiredVPCs > 0 { if vpcQuota, found := quotaCodeToValueMap[_vpcQuotaCode]; found { err := awsClient.VerifyVPCQuota(vpcQuota, requiredVPCs) if err != nil { return err } } else { skippedValidations = append(skippedValidations, fmt.Sprintf("skipping vpc quota verification: unable to find vpc quota (%s)\n", _vpcQuotaCode)) } } } if securityGroupRulesQuota, found := quotaCodeToValueMap[_securityGroupRulesQuotaCode]; found { err := awsClient.VerifySecurityGroupRulesQuota(securityGroupRulesQuota, availabilityZones, numNodeGroups, longestCIDRWhiteList) if err != nil { return err } } else { skippedValidations = append(skippedValidations, fmt.Sprintf("skipping security group rules quota verification: unable to find security group rules quota (%s)\n", _securityGroupRulesQuotaCode)) } if securityGroupsQuota, found := quotaCodeToValueMap[_securityGroupsQuotaCode]; found { err := awsClient.VerifySecurityGroupQuota(securityGroupsQuota, netAdditionOfNodeGroups, clusterAlreadyExists) if err != nil { return err } } else { skippedValidations = append(skippedValidations, fmt.Sprintf("skipping security group quota verification: unable to find security group quota (%s)\n", _securityGroupsQuotaCode)) } return nil } func VerifyNetworkQuotasOnConfigure( awsClient *aws.Client, availabilityZones strset.Set, numNodeGroups int, netAdditionOfNodeGroups int, longestCIDRWhiteList int) error { return VerifyNetworkQuotas(awsClient, 0, false, false, 0, availabilityZones, numNodeGroups, netAdditionOfNodeGroups, longestCIDRWhiteList, true) } ================================================ FILE: pkg/types/clusterconfig/subnet_visibility.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package clusterconfig type SubnetVisibility int const ( UnknownSubnetVisibility SubnetVisibility = iota PublicSubnetVisibility PrivateSubnetVisibility ) var _subnetVisibilities = []string{ "unknown", "public", "private", } func SubnetVisibilityFromString(s string) SubnetVisibility { for i := 0; i < len(_subnetVisibilities); i++ { if s == _subnetVisibilities[i] { return SubnetVisibility(i) } } return UnknownSubnetVisibility } func SubnetVisibilityStrings() []string { return _subnetVisibilities[1:] } func (t SubnetVisibility) String() string { return _subnetVisibilities[t] } // MarshalText satisfies TextMarshaler func (t SubnetVisibility) MarshalText() ([]byte, error) { return []byte(t.String()), nil } // UnmarshalText satisfies TextUnmarshaler func (t *SubnetVisibility) UnmarshalText(text []byte) error { enum := string(text) for i := 0; i < len(_subnetVisibilities); i++ { if enum == _subnetVisibilities[i] { *t = SubnetVisibility(i) return nil } } *t = UnknownSubnetVisibility return nil } // UnmarshalBinary satisfies BinaryUnmarshaler // Needed for msgpack func (t *SubnetVisibility) UnmarshalBinary(data []byte) error { return t.UnmarshalText(data) } // MarshalBinary satisfies BinaryMarshaler func (t SubnetVisibility) MarshalBinary() ([]byte, error) { return []byte(t.String()), nil } ================================================ FILE: pkg/types/clusterconfig/volume_types.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package clusterconfig type VolumeType int const ( UnknownVolumeType VolumeType = iota GP2VolumeType GP3VolumeType IO1VolumeType SC1VolumeType ST1VolumeType ) var _availableVolumeTypes = []string{ "unknown", "gp2", "gp3", "io1", "sc1", "st1", } //VolumeTypeFromString turns string into StorageType func VolumeTypeFromString(s string) VolumeType { for i := 0; i < len(_availableVolumeTypes); i++ { if s == _availableVolumeTypes[i] { return VolumeType(i) } } return UnknownVolumeType } func VolumeTypesStrings() []string { return _availableVolumeTypes[1:] } func (t VolumeType) String() string { return _availableVolumeTypes[t] } // MarshalText satisfies TextMarshaler func (t VolumeType) MarshalText() ([]byte, error) { return []byte(t.String()), nil } // UnmarshalText satisfies TextUnmarshaler func (t *VolumeType) UnmarshalText(text []byte) error { enum := string(text) for i := 0; i < len(_availableVolumeTypes); i++ { if enum == _availableVolumeTypes[i] { *t = VolumeType(i) return nil } } *t = UnknownVolumeType return nil } // UnmarshalBinary satisfies BinaryUnmarshaler // Needed for msgpack func (t *VolumeType) UnmarshalBinary(data []byte) error { return t.UnmarshalText(data) } // MarshalBinary satisfies BinaryMarshaler func (t VolumeType) MarshalBinary() ([]byte, error) { return []byte(t.String()), nil } ================================================ FILE: pkg/types/clusterstate/clusterstate.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package clusterstate import ( "fmt" "strings" "github.com/aws/aws-sdk-go/service/cloudformation" "github.com/cortexlabs/cortex/pkg/lib/aws" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/sets/strset" "github.com/cortexlabs/cortex/pkg/lib/slices" "github.com/cortexlabs/cortex/pkg/lib/table" "github.com/cortexlabs/cortex/pkg/types/clusterconfig" ) const ( controlPlaneTemplate = "eksctl-%s-cluster" operatorTemplate = "eksctl-%s-nodegroup-cx-operator" spotTemplatePrefix = "eksctl-%s-nodegroup-cx-ws" onDemandTemplatePrefix = "eksctl-%s-nodegroup-cx-wd" ) type ClusterStacks struct { clusterName string region string ControlPlaneStack *cloudformation.StackSummary OperatorStack *cloudformation.StackSummary NodeGroupsStacks []*cloudformation.StackSummary } func (cs ClusterStacks) TableString() string { numStacks := len(cs.NodeGroupsStacks) if cs.ControlPlaneStack != nil { numStacks++ } if cs.OperatorStack != nil { numStacks++ } idx := 0 rows := make([][]interface{}, numStacks) if cs.ControlPlaneStack != nil { rows[idx] = []interface{}{ *cs.ControlPlaneStack.StackName, *cs.ControlPlaneStack.StackStatus, } idx++ } if cs.OperatorStack != nil { rows[idx] = []interface{}{ *cs.OperatorStack.StackName, *cs.OperatorStack.StackStatus, } idx++ } for _, stack := range cs.NodeGroupsStacks { rows[idx] = []interface{}{*stack.StackName, *stack.StackStatus} idx++ } t := table.Table{ Headers: []table.Header{ { Title: "cloudformation stack name", }, { Title: "status", }, }, Rows: rows, } return t.MustFormat() } func GetClusterStacks(awsClient *aws.Client, accessConfig *clusterconfig.AccessConfig) (ClusterStacks, error) { controlPlaneStackName := fmt.Sprintf(controlPlaneTemplate, accessConfig.ClusterName) operatorStackName := fmt.Sprintf(operatorTemplate, accessConfig.ClusterName) spotStackNamePrefix := fmt.Sprintf(spotTemplatePrefix, accessConfig.ClusterName) onDemandStackNamePrefix := fmt.Sprintf(onDemandTemplatePrefix, accessConfig.ClusterName) nodeGroupStackPrefixesSet := strset.New(operatorStackName, spotStackNamePrefix, onDemandStackNamePrefix) stackSummaries, err := awsClient.ListEKSStacks(controlPlaneStackName, nodeGroupStackPrefixesSet) if err != nil { return ClusterStacks{}, errors.Wrap(err, "unable to get cluster state from cloudformation") } var controlPlaneStack, operatorStack *cloudformation.StackSummary var ngStacks []*cloudformation.StackSummary for _, stack := range stackSummaries { if stack == nil || stack.StackName == nil { continue } if strings.HasPrefix(*stack.StackName, spotStackNamePrefix) || strings.HasPrefix(*stack.StackName, onDemandStackNamePrefix) { ngStacks = append(ngStacks, stack) } else if *stack.StackName == controlPlaneStackName { controlPlaneStack = stack } else if *stack.StackName == operatorStackName { operatorStack = stack } } return ClusterStacks{ clusterName: accessConfig.ClusterName, region: accessConfig.Region, ControlPlaneStack: controlPlaneStack, OperatorStack: operatorStack, NodeGroupsStacks: ngStacks, }, nil } func GetClusterState(stacks ClusterStacks) State { controlPlaneStackName := fmt.Sprintf(controlPlaneTemplate, stacks.clusterName) if stacks.ControlPlaneStack == nil || stacks.ControlPlaneStack.StackName == nil { return StateClusterDoesntExist } if *stacks.ControlPlaneStack.StackName == controlPlaneStackName { controlPlaneStatus := *stacks.ControlPlaneStack.StackStatus if slices.HasString([]string{ cloudformation.StackStatusDeleteComplete, cloudformation.StackStatusDeleteInProgress, }, controlPlaneStatus) { return StateClusterDoesntExist } if slices.HasString([]string{ cloudformation.StackStatusCreateInProgress, cloudformation.StackStatusCreateComplete, cloudformation.StackStatusUpdateInProgress, cloudformation.StackStatusUpdateComplete, cloudformation.StackStatusRollbackInProgress, cloudformation.StackStatusRollbackComplete, cloudformation.StackStatusUpdateRollbackInProgress, cloudformation.StackStatusUpdateRollbackComplete, }, controlPlaneStatus) { return StateClusterExists } return StateClusterInUnexpectedState } return StateClusterDoesntExist } func CloudFormationURL(clusterName string, region string) string { return fmt.Sprintf("https://console.aws.amazon.com/cloudformation/home?region=%s#/stacks?filteringText=eksctl-%s-", region, clusterName) } ================================================ FILE: pkg/types/clusterstate/errors.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package clusterstate import ( "fmt" "github.com/cortexlabs/cortex/pkg/lib/errors" ) const ( ErrClusterDoesNotExist = "clusterstate.cluster_does_not_exist" ErrClusterAlreadyExists = "clusterstate.cluster_already_exists" ErrUnexpectedClusterState = "clusterstate.unexpected_cluster_state" ) func ErrorClusterDoesNotExist(stacks ClusterStacks) error { return errors.WithStack(&errors.Error{ Kind: ErrClusterDoesNotExist, Message: fmt.Sprintf("there is no cluster named \"%s\" in %s", stacks.clusterName, stacks.region), }) } func ErrorClusterAlreadyExists(stacks ClusterStacks) error { return errors.WithStack(&errors.Error{ Kind: ErrClusterAlreadyExists, Message: fmt.Sprintf("a cluster named \"%s\" already exists in %s", stacks.clusterName, stacks.region), }) } func ErrorUnexpectedClusterState(stacks ClusterStacks) error { msg := fmt.Sprintf("cluster named \"%s\" in %s is in an unexpected state; if your CloudFormation stacks are updating, please wait for them to complete. Otherwise, run `cortex cluster down` to delete the cluster\n\n", stacks.clusterName, stacks.region) msg += fmt.Sprintf(stacks.TableString()) return errors.WithStack(&errors.Error{ Kind: ErrUnexpectedClusterState, Message: msg, Metadata: stacks, }) } func AssertClusterState(stacks ClusterStacks, currentState, allowedState State) error { if currentState == allowedState { return nil } switch currentState { case StateClusterDoesntExist: return ErrorClusterDoesNotExist(stacks) case StateClusterExists: return ErrorClusterAlreadyExists(stacks) case StateClusterInUnexpectedState: return ErrorUnexpectedClusterState(stacks) } return nil } ================================================ FILE: pkg/types/clusterstate/state.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package clusterstate type State string const ( StateClusterExists State = "cluster_exists" StateClusterDoesntExist State = "cluster_doesnt_exist" StateClusterInUnexpectedState State = "cluster_in_unexpected_state" ) ================================================ FILE: pkg/types/metrics/batch_metrics.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package metrics import ( "github.com/cortexlabs/cortex/pkg/lib/pointer" "github.com/cortexlabs/cortex/pkg/lib/slices" ) type BatchMetrics struct { Succeeded int `json:"succeeded"` Failed int `json:"failed"` AverageTimePerBatch *float64 `json:"average_time_per_batch"` } func (batchMetrics BatchMetrics) Merge(right BatchMetrics) BatchMetrics { newBatchMetrics := BatchMetrics{} newBatchMetrics.MergeInPlace(batchMetrics) newBatchMetrics.MergeInPlace(right) return newBatchMetrics } func (batchMetrics *BatchMetrics) MergeInPlace(right BatchMetrics) { batchMetrics.AverageTimePerBatch = mergeAvg(batchMetrics.AverageTimePerBatch, batchMetrics.Succeeded, right.AverageTimePerBatch, right.Succeeded) batchMetrics.Succeeded = batchMetrics.Succeeded + right.Succeeded batchMetrics.Failed = batchMetrics.Failed + right.Failed } func mergeAvg(left *float64, leftCount int, right *float64, rightCount int) *float64 { leftCountFloat64Ptr := pointer.Float64(float64(leftCount)) rightCountFloat64Ptr := pointer.Float64(float64(rightCount)) avg, _ := slices.Float64PtrAvg([]*float64{left, right}, []*float64{leftCountFloat64Ptr, rightCountFloat64Ptr}) return avg } ================================================ FILE: pkg/types/metrics/metrics_test.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package metrics import ( "testing" "github.com/cortexlabs/cortex/pkg/lib/pointer" "github.com/stretchr/testify/require" ) func TestMergeAvg(t *testing.T) { floatNilPtr := (*float64)(nil) require.Equal(t, floatNilPtr, mergeAvg(nil, 0, nil, 0)) require.Equal(t, floatNilPtr, mergeAvg(nil, 1, nil, 0)) require.Equal(t, float64(1), *mergeAvg(pointer.Float64(1), 1, nil, 1)) require.Equal(t, pointer.Float64(1), mergeAvg(nil, 1, pointer.Float64(1), 1)) require.Equal(t, floatNilPtr, mergeAvg(pointer.Float64(1), 0, nil, 1)) require.Equal(t, floatNilPtr, mergeAvg(nil, 1, pointer.Float64(1), 0)) require.Equal(t, float64(1.25), *mergeAvg(pointer.Float64(1.25), 5, nil, 0)) require.Equal(t, float64(1.25), *mergeAvg(nil, 0, pointer.Float64(1.25), 5)) require.Equal(t, float64(1.25), *mergeAvg(pointer.Float64(1), 3, pointer.Float64(2), 1)) require.Equal(t, float64(1.25), *mergeAvg(pointer.Float64(2), 1, pointer.Float64(1), 3)) } ================================================ FILE: pkg/types/metrics/queue_metrics.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package metrics // There could be Cortex specific messages in queue type QueueMetrics struct { Visible int `json:"visible"` NotVisible int `json:"not_visible"` } func (q QueueMetrics) IsEmpty() bool { return q.TotalInQueue() == 0 } func (q QueueMetrics) TotalInQueue() int { return q.Visible + q.NotVisible } func (q QueueMetrics) TotalUserMessages() int { total := q.TotalInQueue() if total == 0 { return 0 } return total - 1 // An extra message is added } ================================================ FILE: pkg/types/spec/api.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package spec import ( "bytes" "fmt" "math" "path/filepath" "strconv" "strings" "time" "github.com/cortexlabs/cortex/pkg/consts" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/hash" s "github.com/cortexlabs/cortex/pkg/lib/strings" "github.com/cortexlabs/cortex/pkg/types/userconfig" istioclientnetworking "istio.io/client-go/pkg/apis/networking/v1beta1" kapps "k8s.io/api/apps/v1" ) type API struct { *userconfig.API ID string `json:"id" yaml:"id"` SpecID string `json:"spec_id" yaml:"spec_id"` PodID string `json:"pod_id" yaml:"pod_id"` DeploymentID string `json:"deployment_id" yaml:"deployment_id"` Key string `json:"key" yaml:"key"` InitialDeploymentTime int64 `json:"initial_deployment_time" yaml:"initial_deployment_time"` LastUpdated int64 `json:"last_updated" yaml:"last_updated"` MetadataRoot string `json:"metadata_root" yaml:"metadata_root"` } type Metadata struct { *userconfig.Resource APIID string `json:"id" yaml:"id"` DeploymentID string `json:"deployment_id,omitempty" yaml:"deployment_id,omitempty"` LastUpdated int64 `json:"last_updated" yaml:"last_updated"` } func MetadataFromDeployment(deployment *kapps.Deployment) (*Metadata, error) { lastUpdated, err := TimeFromAPIID(deployment.Labels["apiID"]) if err != nil { return nil, err } return &Metadata{ Resource: &userconfig.Resource{ Name: deployment.Labels["apiName"], Kind: userconfig.KindFromString(deployment.Labels["apiKind"]), }, APIID: deployment.Labels["apiID"], DeploymentID: deployment.Labels["deploymentID"], LastUpdated: lastUpdated.Unix(), }, nil } func MetadataFromVirtualService(vs *istioclientnetworking.VirtualService) (*Metadata, error) { lastUpdated, err := TimeFromAPIID(vs.Labels["apiID"]) if err != nil { return nil, err } return &Metadata{ Resource: &userconfig.Resource{ Name: vs.Labels["apiName"], Kind: userconfig.KindFromString(vs.Labels["apiKind"]), }, APIID: vs.Labels["apiID"], DeploymentID: vs.Labels["deploymentID"], LastUpdated: lastUpdated.Unix(), }, nil } /* * ID (uniquely identifies an api configuration for a given deployment) * DeploymentID (used for refreshing a deployment) * SpecID (uniquely identifies api configuration specified by user) * PodID (an ID representing the pod spec) * Resource * Containers * Compute * Pod * Deployment Strategy * Autoscaling * Networking * APIs initialDeploymentTime is Time.UnixNano() */ func GetAPISpec(apiConfig *userconfig.API, initialDeploymentTime int64, deploymentID string, clusterUID string) *API { var buf bytes.Buffer buf.WriteString(s.Obj(apiConfig.Resource)) buf.WriteString(s.Obj(apiConfig.Pod)) podID := hash.Bytes(buf.Bytes()) buf.Reset() buf.WriteString(podID) buf.WriteString(s.Obj(apiConfig.APIs)) buf.WriteString(s.Obj(apiConfig.Networking)) buf.WriteString(s.Obj(apiConfig.Autoscaling)) buf.WriteString(s.Obj(apiConfig.UpdateStrategy)) buf.WriteString(s.Obj(apiConfig.NodeGroups)) specID := hash.Bytes(buf.Bytes())[:32] apiID := fmt.Sprintf("%s-%s-%s", MonotonicallyDecreasingID(), deploymentID, specID) // should be up to 60 characters long return &API{ API: apiConfig, ID: apiID, SpecID: specID, PodID: podID, Key: Key(apiConfig.Name, apiID, clusterUID), InitialDeploymentTime: initialDeploymentTime, DeploymentID: deploymentID, LastUpdated: time.Now().Unix(), MetadataRoot: MetadataRoot(apiConfig.Name, clusterUID), } } func Key(apiName string, apiID string, clusterUID string) string { return filepath.Join( clusterUID, "apis", apiName, "api", apiID, consts.CortexVersion+"-spec.json", ) } // The path to the directory which contains one subdirectory for each API ID (for its API spec) func KeysPrefix(apiName string, clusterUID string) string { return filepath.Join( clusterUID, "apis", apiName, "api", ) + "/" } func MetadataRoot(apiName string, clusterUID string) string { return filepath.Join( clusterUID, "apis", apiName, "metadata", ) } // Extract the timestamp from an API ID func TimeFromAPIID(apiID string) (time.Time, error) { timeIDStr := strings.Split(apiID, "-")[0] timeID, err := strconv.ParseInt(timeIDStr, 16, 64) if err != nil { return time.Time{}, errors.Wrap(err, fmt.Sprintf("unable to parse API timestamp (%s)", timeIDStr)) } timeNanos := math.MaxInt64 - timeID return time.Unix(0, timeNanos), nil } ================================================ FILE: pkg/types/spec/errors.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package spec import ( "fmt" "regexp" "github.com/cortexlabs/cortex/pkg/consts" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/k8s" "github.com/cortexlabs/cortex/pkg/lib/sets/strset" s "github.com/cortexlabs/cortex/pkg/lib/strings" "github.com/cortexlabs/cortex/pkg/types/userconfig" ) const ( ErrMalformedConfig = "spec.malformed_config" ErrNoAPIs = "spec.no_apis" ErrDuplicateName = "spec.duplicate_name" ErrDuplicateEndpointInOneDeploy = "spec.duplicate_endpoint_in_one_deploy" ErrDuplicateEndpoint = "spec.duplicate_endpoint" ErrDuplicateContainerName = "spec.duplicate_container_name" ErrSpecifyExactlyOneField = "spec.specify_exactly_one_field" ErrSpecifyAllOrNone = "spec.specify_all_or_none" ErrOneOfPrerequisitesNotDefined = "spec.one_of_prerequisites_not_defined" ErrConfigGreaterThanOtherConfig = "spec.config_greater_than_other_config" ErrMinReplicasGreaterThanMax = "spec.min_replicas_greater_than_max" ErrInitReplicasGreaterThanMax = "spec.init_replicas_greater_than_max" ErrInitReplicasLessThanMin = "spec.init_replicas_less_than_min" ErrTargetInFlightLimitReached = "spec.target_in_flight_limit_reached" ErrInvalidSurgeOrUnavailable = "spec.invalid_surge_or_unavailable" ErrSurgeAndUnavailableBothZero = "spec.surge_and_unavailable_both_zero" ErrShmCannotExceedMem = "spec.shm_cannot_exceed_mem" ErrFieldMustBeSpecifiedForKind = "spec.field_must_be_specified_for_kind" ErrFieldIsNotSupportedForKind = "spec.field_is_not_supported_for_kind" ErrCortexPrefixedEnvVarNotAllowed = "spec.cortex_prefixed_env_var_not_allowed" ErrDisallowedEnvVars = "spec.disallowed_env_vars" ErrComputeResourceConflict = "spec.compute_resource_conflict" ErrIncorrectTrafficSplitterWeight = "spec.incorrect_traffic_splitter_weight" ErrTrafficSplitterAPIsNotUnique = "spec.traffic_splitter_apis_not_unique" ErrOneShadowPerTrafficSplitter = "spec.one_shadow_per_traffic_splitter" ErrUnexpectedDockerSecretData = "spec.unexpected_docker_secret_data" ) func ErrorMalformedConfig() error { return errors.WithStack(&errors.Error{ Kind: ErrMalformedConfig, Message: fmt.Sprintf("cortex YAML configuration files must contain a list of maps (see https://docs.cortexlabs.com/v/%s/ for api configuration schema)", consts.CortexVersionMinor), }) } func ErrorNoAPIs() error { return errors.WithStack(&errors.Error{ Kind: ErrNoAPIs, Message: fmt.Sprintf("at least one API must be configured (see https://docs.cortexlabs.com/v/%s/ for api configuration schema)", consts.CortexVersionMinor), }) } func ErrorDuplicateName(apis []userconfig.API) error { filePaths := strset.New() for _, api := range apis { filePaths.Add(api.FileName) } return errors.WithStack(&errors.Error{ Kind: ErrDuplicateName, Message: fmt.Sprintf("name %s must be unique across apis (defined in %s)", s.UserStr(apis[0].Name), s.StrsAnd(filePaths.Slice())), }) } func ErrorDuplicateEndpointInOneDeploy(apis []userconfig.API) error { names := make([]string, len(apis)) for i, api := range apis { names[i] = api.Name } return errors.WithStack(&errors.Error{ Kind: ErrDuplicateEndpointInOneDeploy, Message: fmt.Sprintf("endpoint %s must be unique across apis (defined in %s)", s.UserStr(*apis[0].Networking.Endpoint), s.StrsAnd(names)), }) } func ErrorDuplicateEndpoint(apiName string) error { return errors.WithStack(&errors.Error{ Kind: ErrDuplicateEndpoint, Message: fmt.Sprintf("endpoint is already being used by %s", apiName), }) } func ErrorDuplicateContainerName(containerName string) error { return errors.WithStack(&errors.Error{ Kind: ErrDuplicateContainerName, Message: fmt.Sprintf("container name %s must be unique", containerName), }) } func ErrorSpecifyExactlyOneField(numSpecified int, fields ...string) error { var msg string if len(fields) == 2 { if numSpecified == 0 { msg = fmt.Sprintf("please specify either %s", s.UserStrsOr(fields)) } else { msg = fmt.Sprintf("please specify either %s (both cannot be specified at the same time)", s.UserStrsOr(fields)) } } else { if numSpecified == 0 { msg = fmt.Sprintf("please specify one of the following fields: %s", s.UserStrsOr(fields)) } else { msg = fmt.Sprintf("please specify only one of the following fields: %s", s.UserStrsOr(fields)) } } return errors.WithStack(&errors.Error{ Kind: ErrSpecifyExactlyOneField, Message: msg, }) } func ErrorSpecifyAllOrNone(val string, vals ...string) error { allVals := append([]string{val}, vals...) message := fmt.Sprintf("please specify all or none of %s", s.UserStrsAnd(allVals)) if len(allVals) == 2 { message = fmt.Sprintf("please specify both %s and %s or neither of them", s.UserStr(allVals[0]), s.UserStr(allVals[1])) } return errors.WithStack(&errors.Error{ Kind: ErrSpecifyAllOrNone, Message: message, }) } func ErrorOneOfPrerequisitesNotDefined(argName string, prerequisite string, prerequisites ...string) error { allPrerequisites := append([]string{prerequisite}, prerequisites...) message := fmt.Sprintf("%s specified without specifying %s", s.UserStr(argName), s.UserStrsOr(allPrerequisites)) return errors.WithStack(&errors.Error{ Kind: ErrOneOfPrerequisitesNotDefined, Message: message, }) } func ErrorConfigGreaterThanOtherConfig(tooBigKey string, tooBigVal interface{}, tooSmallKey string, tooSmallVal interface{}) error { return errors.WithStack(&errors.Error{ Kind: ErrConfigGreaterThanOtherConfig, Message: fmt.Sprintf("%s (%s) cannot be greater than %s (%s)", tooBigKey, s.UserStr(tooBigVal), tooSmallKey, s.UserStr(tooSmallVal)), }) } func ErrorMinReplicasGreaterThanMax(min int32, max int32) error { return errors.WithStack(&errors.Error{ Kind: ErrMinReplicasGreaterThanMax, Message: fmt.Sprintf("%s cannot be greater than %s (%d > %d)", userconfig.MinReplicasKey, userconfig.MaxReplicasKey, min, max), }) } func ErrorInitReplicasGreaterThanMax(init int32, max int32) error { return errors.WithStack(&errors.Error{ Kind: ErrInitReplicasGreaterThanMax, Message: fmt.Sprintf("%s cannot be greater than %s (%d > %d)", userconfig.InitReplicasKey, userconfig.MaxReplicasKey, init, max), }) } func ErrorInitReplicasLessThanMin(init int32, min int32) error { return errors.WithStack(&errors.Error{ Kind: ErrInitReplicasLessThanMin, Message: fmt.Sprintf("%s cannot be less than %s (%d < %d)", userconfig.InitReplicasKey, userconfig.MinReplicasKey, init, min), }) } func ErrorTargetInFlightLimitReached(targetInFlight float64, maxConcurrency, maxQueueLength int64) error { return errors.WithStack(&errors.Error{ Kind: ErrTargetInFlightLimitReached, Message: fmt.Sprintf("%s cannot be greater than %s + %s (%f > %d + %d)", userconfig.TargetInFlightKey, userconfig.MaxConcurrencyKey, userconfig.MaxQueueLengthKey, targetInFlight, maxConcurrency, maxQueueLength), }) } func ErrorInvalidSurgeOrUnavailable(val string) error { return errors.WithStack(&errors.Error{ Kind: ErrInvalidSurgeOrUnavailable, Message: fmt.Sprintf("%s is not a valid value - must be an integer percentage (e.g. 25%%, to denote a percentage of desired replicas) or a positive integer (e.g. 5, to denote a number of replicas)", s.UserStr(val)), }) } func ErrorSurgeAndUnavailableBothZero() error { return errors.WithStack(&errors.Error{ Kind: ErrSurgeAndUnavailableBothZero, Message: fmt.Sprintf("%s and %s cannot both be zero", userconfig.MaxSurgeKey, userconfig.MaxUnavailableKey), }) } func ErrorShmCannotExceedMem(shm k8s.Quantity, mem k8s.Quantity) error { return errors.WithStack(&errors.Error{ Kind: ErrShmCannotExceedMem, Message: fmt.Sprintf("%s (%s) cannot exceed total compute %s (%s)", userconfig.ShmKey, shm.UserString, userconfig.MemKey, mem.UserString), }) } func ErrorFieldMustBeSpecifiedForKind(field string, kind userconfig.Kind) error { return errors.WithStack(&errors.Error{ Kind: ErrFieldMustBeSpecifiedForKind, Message: fmt.Sprintf("%s must be specified for %s kind", field, kind.String()), }) } func ErrorFieldIsNotSupportedForKind(field string, kind userconfig.Kind) error { return errors.WithStack(&errors.Error{ Kind: ErrFieldIsNotSupportedForKind, Message: fmt.Sprintf("%s is not supported for %s kind", field, kind.String()), }) } func ErrorCortexPrefixedEnvVarNotAllowed(prefixes ...string) error { return errors.WithStack(&errors.Error{ Kind: ErrCortexPrefixedEnvVarNotAllowed, Message: fmt.Sprintf("environment variables starting with %s are reserved", s.StrsOr(prefixes)), }) } func ErrorDisallowedEnvVars(disallowedValues ...string) error { return errors.WithStack(&errors.Error{ Kind: ErrDisallowedEnvVars, Message: fmt.Sprintf("environment %s %s %s disallowed", s.PluralS("variables", len(disallowedValues)), s.StrsAnd(disallowedValues), s.PluralIs(len(disallowedValues))), }) } func ErrorComputeResourceConflict(resourceA, resourceB string) error { return errors.WithStack(&errors.Error{ Kind: ErrComputeResourceConflict, Message: fmt.Sprintf("%s and %s resources cannot be used together", resourceA, resourceB), }) } func ErrorIncorrectTrafficSplitterWeightTotal(totalWeight int32) error { return errors.WithStack(&errors.Error{ Kind: ErrIncorrectTrafficSplitterWeight, Message: fmt.Sprintf("expected weights of all non-shadow apis to sum to 100 but found %d", totalWeight), }) } func ErrorTrafficSplitterAPIsNotUnique(names []string) error { return errors.WithStack(&errors.Error{ Kind: ErrTrafficSplitterAPIsNotUnique, Message: fmt.Sprintf("%s not unique: %s", s.PluralS("api", len(names)), s.StrsSentence(names, "")), }) } func ErrorOneShadowPerTrafficSplitter() error { return errors.WithStack(&errors.Error{ Kind: ErrOneShadowPerTrafficSplitter, Message: "multiple shadow apis detected; only one api is allowed to be marked as a shadow", }) } var _pwRegex = regexp.MustCompile(`"password":"[^"]+"`) var _authRegex = regexp.MustCompile(`"auth":"[^"]+"`) func ErrorUnexpectedDockerSecretData(reason string, secretData map[string][]byte) error { secretDataStrMap := map[string]string{} for key, value := range secretData { valueStr := string(value) valueStr = _pwRegex.ReplaceAllString(valueStr, `"password":""`) valueStr = _authRegex.ReplaceAllString(valueStr, `"auth":""`) secretDataStrMap[key] = valueStr } return errors.WithStack(&errors.Error{ Kind: ErrUnexpectedDockerSecretData, Message: fmt.Sprintf("docker registry secret named \"%s\" was found, but contains unexpected data (%s); got: %s", _dockerPullSecretName, reason, s.UserStr(secretDataStrMap)), }) } ================================================ FILE: pkg/types/spec/id_gen.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package spec import ( "fmt" "math" "sync" "time" ) var idGenMutex = sync.Mutex{} // ID creation optimized for listing the most recently created jobs in S3. S3 objects are listed in ascending UTF-8 binary order. This should work until the year 2262. func MonotonicallyDecreasingID() string { idGenMutex.Lock() defer idGenMutex.Unlock() i := math.MaxInt64 - time.Now().UnixNano() return fmt.Sprintf("%x", i) } ================================================ FILE: pkg/types/spec/job.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package spec import ( "fmt" "path" "path/filepath" "time" "github.com/cortexlabs/cortex/pkg/consts" s "github.com/cortexlabs/cortex/pkg/lib/strings" "github.com/cortexlabs/cortex/pkg/types/userconfig" ) const ( MetricsFileKey = "metrics.json" ) type JobKey struct { ID string `json:"job_id" yaml:"job_id"` APIName string `json:"api_name" yaml:"api_name"` Kind userconfig.Kind `json:"kind" yaml:"kind"` } func (j JobKey) UserString() string { return fmt.Sprintf("%s (%s api)", j.ID, j.APIName) } // e.g. //jobs/////spec.json func (j JobKey) SpecFilePath(clusterUID string) string { return path.Join(j.Prefix(clusterUID), "spec.json") } // e.g. //jobs//// func (j JobKey) Prefix(clusterUID string) string { return s.EnsureSuffix(path.Join(JobAPIPrefix(clusterUID, j.Kind, j.APIName), j.ID), "/") } func (j JobKey) K8sName() string { return fmt.Sprintf("%s-%s", j.APIName, j.ID) } type SQSDeadLetterQueue struct { ARN string `json:"arn" yaml:"arn"` MaxReceiveCount int `json:"max_receive_count" yaml:"max_receive_count"` } type RuntimeBatchJobConfig struct { Workers int `json:"workers" yaml:"workers"` SQSDeadLetterQueue *SQSDeadLetterQueue `json:"sqs_dead_letter_queue" yaml:"sqs_dead_letter_queue"` Config map[string]interface{} `json:"config" yaml:"config"` Timeout *int `json:"timeout" yaml:"timeout"` } type RuntimeTaskJobConfig struct { Workers int `json:"workers" yaml:"workers"` Config map[string]interface{} `json:"config" yaml:"config"` Timeout *int `json:"timeout" yaml:"timeout"` } type BatchJob struct { JobKey RuntimeBatchJobConfig APIID string `json:"api_id" yaml:"api_id"` SQSUrl string `json:"sqs_url" yaml:"sqs_url"` TotalBatchCount int `json:"total_batch_count,omitempty" yaml:"total_batch_count,omitempty"` StartTime time.Time `json:"start_time,omitempty" yaml:"start_time,omitempty"` } type TaskJob struct { JobKey RuntimeTaskJobConfig APIID string `json:"api_id" yaml:"api_id"` SpecID string `json:"spec_id" yaml:"spec_id"` PodID string `json:"pod_id" yaml:"pod_id"` StartTime time.Time `json:"start_time" yaml:"start_time"` } // e.g. //jobs/// func JobAPIPrefix(clusterUID string, kind userconfig.Kind, apiName string) string { return filepath.Join(clusterUID, "jobs", kind.String(), consts.CortexVersion, apiName) } func JobPayloadKey(clusterUID string, kind userconfig.Kind, apiName string, jobID string) string { return filepath.Join(JobAPIPrefix(clusterUID, kind, apiName), jobID, "payload.json") } func JobBatchCountKey(clusterUID string, kind userconfig.Kind, apiName string, jobID string) string { return filepath.Join(JobAPIPrefix(clusterUID, kind, apiName), jobID, "max_batch_count") } func JobMetricsKey(clusterUID string, kind userconfig.Kind, apiName string, jobID string) string { return filepath.Join(JobAPIPrefix(clusterUID, kind, apiName), jobID, MetricsFileKey) } ================================================ FILE: pkg/types/spec/utils.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package spec import ( "strings" "github.com/cortexlabs/cortex/pkg/lib/errors" s "github.com/cortexlabs/cortex/pkg/lib/strings" "github.com/cortexlabs/cortex/pkg/types/userconfig" ) func FindDuplicateNames(apis []userconfig.API) []userconfig.API { names := make(map[string][]userconfig.API) for _, api := range apis { names[api.Name] = append(names[api.Name], api) } for name := range names { if len(names[name]) > 1 { return names[name] } } return nil } func surgeOrUnavailableValidator(str string) (string, error) { if strings.HasSuffix(str, "%") { parsed, ok := s.ParseInt32(strings.TrimSuffix(str, "%")) if !ok { return "", ErrorInvalidSurgeOrUnavailable(str) } if parsed < 0 || parsed > 100 { return "", ErrorInvalidSurgeOrUnavailable(str) } } else { parsed, ok := s.ParseInt32(str) if !ok { return "", ErrorInvalidSurgeOrUnavailable(str) } if parsed < 0 { return "", ErrorInvalidSurgeOrUnavailable(str) } } return str, nil } func verifyTotalWeight(apis []*userconfig.TrafficSplit) error { totalWeight := int32(0) for _, api := range apis { if !api.Shadow { totalWeight += api.Weight } } if totalWeight == 100 { return nil } return errors.Wrap(ErrorIncorrectTrafficSplitterWeightTotal(totalWeight), userconfig.APIsKey) } // areTrafficSplitterAPIsUnique gives error if the same API is used multiple times in TrafficSplitter func areTrafficSplitterAPIsUnique(apis []*userconfig.TrafficSplit) error { names := make(map[string][]userconfig.TrafficSplit) for _, api := range apis { names[api.Name] = append(names[api.Name], *api) } var notUniqueAPIs []string for name := range names { if len(names[name]) > 1 { notUniqueAPIs = append(notUniqueAPIs, names[name][0].Name) } } if len(notUniqueAPIs) > 0 { return errors.Wrap(ErrorTrafficSplitterAPIsNotUnique(notUniqueAPIs), userconfig.APIsKey) } return nil } ================================================ FILE: pkg/types/spec/validations.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package spec import ( "context" "fmt" "strings" "time" "github.com/cortexlabs/cortex/pkg/consts" "github.com/cortexlabs/cortex/pkg/lib/aws" "github.com/cortexlabs/cortex/pkg/lib/cast" cr "github.com/cortexlabs/cortex/pkg/lib/configreader" "github.com/cortexlabs/cortex/pkg/lib/docker" "github.com/cortexlabs/cortex/pkg/lib/errors" libjson "github.com/cortexlabs/cortex/pkg/lib/json" "github.com/cortexlabs/cortex/pkg/lib/k8s" "github.com/cortexlabs/cortex/pkg/lib/pointer" "github.com/cortexlabs/cortex/pkg/lib/regex" "github.com/cortexlabs/cortex/pkg/lib/slices" s "github.com/cortexlabs/cortex/pkg/lib/strings" libtime "github.com/cortexlabs/cortex/pkg/lib/time" "github.com/cortexlabs/cortex/pkg/lib/urls" "github.com/cortexlabs/cortex/pkg/types/userconfig" dockertypes "github.com/docker/docker/api/types" kresource "k8s.io/apimachinery/pkg/api/resource" ) var AutoscalingTickInterval = 10 * time.Second const _dockerPullSecretName = "registry-credentials" func apiValidation(resource userconfig.Resource) *cr.StructValidation { var structFieldValidations []*cr.StructFieldValidation switch resource.Kind { case userconfig.RealtimeAPIKind: structFieldValidations = append(resourceStructValidations, podValidation(userconfig.RealtimeAPIKind), nodegroupsValidation(), networkingValidation(), autoscalingValidation(), updateStrategyValidation(), ) case userconfig.AsyncAPIKind: structFieldValidations = append(resourceStructValidations, podValidation(userconfig.AsyncAPIKind), nodegroupsValidation(), networkingValidation(), autoscalingValidation(), updateStrategyValidation(), ) case userconfig.BatchAPIKind: structFieldValidations = append(resourceStructValidations, podValidation(userconfig.BatchAPIKind), nodegroupsValidation(), networkingValidation(), ) case userconfig.TaskAPIKind: structFieldValidations = append(resourceStructValidations, podValidation(userconfig.TaskAPIKind), nodegroupsValidation(), networkingValidation(), ) case userconfig.TrafficSplitterKind: structFieldValidations = append(resourceStructValidations, multiAPIsValidation(), networkingValidation(), ) } return &cr.StructValidation{ StructFieldValidations: structFieldValidations, } } var resourceStructValidations = []*cr.StructFieldValidation{ { StructField: "Name", StringValidation: &cr.StringValidation{ Required: true, DNS1035: true, InvalidPrefixes: []string{"b-"}, // collides with our sqs names MaxLength: 42, // k8s adds 21 characters to the pod name, and 63 is the max before it starts to truncate }, }, { StructField: "Kind", StringValidation: &cr.StringValidation{ Required: true, AllowedValues: userconfig.KindStrings(), }, Parser: func(str string) (interface{}, error) { return userconfig.KindFromString(str), nil }, }, } func multiAPIsValidation() *cr.StructFieldValidation { return &cr.StructFieldValidation{ StructField: "APIs", StructListValidation: &cr.StructListValidation{ Required: true, TreatNullAsEmpty: true, StructValidation: &cr.StructValidation{ StructFieldValidations: []*cr.StructFieldValidation{ { StructField: "Name", StringValidation: &cr.StringValidation{ Required: true, AllowEmpty: false, }, }, { StructField: "Weight", Int32Validation: &cr.Int32Validation{ Required: true, GreaterThanOrEqualTo: pointer.Int32(0), LessThanOrEqualTo: pointer.Int32(100), }, }, { StructField: "Shadow", BoolValidation: &cr.BoolValidation{}, }, }, }, }, } } func podValidation(kind userconfig.Kind) *cr.StructFieldValidation { validation := &cr.StructFieldValidation{ StructField: "Pod", StructValidation: &cr.StructValidation{ StructFieldValidations: []*cr.StructFieldValidation{ { StructField: "Port", Int32PtrValidation: &cr.Int32PtrValidation{ Required: false, Default: nil, // it's a pointer because it's not required for the task API AllowExplicitNull: true, GreaterThan: pointer.Int32(0), LessThanOrEqualTo: pointer.Int32(65535), DisallowedValues: consts.ReservedContainerPorts, }, }, containersValidation(kind), }, }, } if kind == userconfig.RealtimeAPIKind { validation.StructValidation.StructFieldValidations = append(validation.StructValidation.StructFieldValidations, &cr.StructFieldValidation{ StructField: "MaxQueueLength", Int64Validation: &cr.Int64Validation{ Default: consts.DefaultMaxQueueLength, GreaterThan: pointer.Int64(0), // the proxy can theoretically accept up to 32768 connections, but during testing, // it has been observed that the number is just slightly lower, so it has been offset by 2678 LessThanOrEqualTo: pointer.Int64(30000), }, }, &cr.StructFieldValidation{ StructField: "MaxConcurrency", Int64Validation: &cr.Int64Validation{ Default: consts.DefaultMaxConcurrency, GreaterThan: pointer.Int64(0), // the proxy can theoretically accept up to 32768 connections, but during testing, // it has been observed that the number is just slightly lower, so it has been offset by 2678 LessThanOrEqualTo: pointer.Int64(30000), }, }, ) } if kind == userconfig.AsyncAPIKind { validation.StructValidation.StructFieldValidations = append(validation.StructValidation.StructFieldValidations, &cr.StructFieldValidation{ StructField: "MaxConcurrency", Int64Validation: &cr.Int64Validation{ Default: consts.DefaultMaxConcurrency, GreaterThan: pointer.Int64(0), LessThanOrEqualTo: pointer.Int64(100), }, }, ) } return validation } func containersValidation(kind userconfig.Kind) *cr.StructFieldValidation { validations := []*cr.StructFieldValidation{ { StructField: "Name", StringValidation: &cr.StringValidation{ Required: true, AllowEmpty: false, DNS1035: true, MaxLength: 63, DisallowedValues: consts.ReservedContainerNames, }, }, { StructField: "Image", StringValidation: &cr.StringValidation{ Required: true, AllowEmpty: false, DockerImage: true, }, }, { StructField: "Env", StringMapValidation: &cr.StringMapValidation{ Required: false, Default: map[string]string{}, AllowEmpty: true, }, }, { StructField: "Command", StringListValidation: &cr.StringListValidation{ Required: false, AllowExplicitNull: true, AllowEmpty: true, }, }, { StructField: "Args", StringListValidation: &cr.StringListValidation{ Required: false, AllowExplicitNull: true, AllowEmpty: true, }, }, computeValidation(), probeValidation("LivenessProbe", true), } if kind == userconfig.RealtimeAPIKind { validations = append(validations, probeValidation("ReadinessProbe", true)) } else if kind == userconfig.AsyncAPIKind || kind == userconfig.BatchAPIKind { validations = append(validations, probeValidation("ReadinessProbe", false)) } if kind == userconfig.RealtimeAPIKind || kind == userconfig.AsyncAPIKind { validations = append(validations, preStopValidation()) } return &cr.StructFieldValidation{ StructField: "Containers", StructListValidation: &cr.StructListValidation{ Required: true, TreatNullAsEmpty: true, MinLength: 1, StructValidation: &cr.StructValidation{ StructFieldValidations: validations, }, }, } } func nodegroupsValidation() *cr.StructFieldValidation { return &cr.StructFieldValidation{ StructField: "NodeGroups", StringListValidation: &cr.StringListValidation{ Required: false, Default: nil, AllowExplicitNull: true, AllowEmpty: false, ElementStringValidation: &cr.StringValidation{ AlphaNumericDashUnderscore: true, }, }, } } func networkingValidation() *cr.StructFieldValidation { return &cr.StructFieldValidation{ StructField: "Networking", StructValidation: &cr.StructValidation{ StructFieldValidations: []*cr.StructFieldValidation{ { StructField: "Endpoint", StringPtrValidation: &cr.StringPtrValidation{ Validator: urls.ValidateEndpoint, MaxLength: 1000, // no particular reason other than it works }, }, }, }, } } func probeValidation(structFieldName string, hasExecProbe bool) *cr.StructFieldValidation { validations := []*cr.StructFieldValidation{ httpGetHandlerValidation(), tcpSocketHandlerValidation(), { StructField: "InitialDelaySeconds", Int32Validation: &cr.Int32Validation{ Default: 0, GreaterThanOrEqualTo: pointer.Int32(0), }, }, { StructField: "TimeoutSeconds", Int32Validation: &cr.Int32Validation{ Default: 1, GreaterThanOrEqualTo: pointer.Int32(0), }, }, { StructField: "PeriodSeconds", Int32Validation: &cr.Int32Validation{ Default: 10, GreaterThanOrEqualTo: pointer.Int32(0), }, }, { StructField: "SuccessThreshold", Int32Validation: &cr.Int32Validation{ Default: 1, GreaterThanOrEqualTo: pointer.Int32(0), }, }, { StructField: "FailureThreshold", Int32Validation: &cr.Int32Validation{ Default: 3, GreaterThanOrEqualTo: pointer.Int32(0), }, }, } if hasExecProbe { validations = append(validations, execHandlerValidation()) } return &cr.StructFieldValidation{ StructField: structFieldName, StructValidation: &cr.StructValidation{ Required: false, AllowExplicitNull: true, DefaultNil: true, StructFieldValidations: validations, }, } } func preStopValidation() *cr.StructFieldValidation { return &cr.StructFieldValidation{ StructField: "PreStop", StructValidation: &cr.StructValidation{ Required: false, AllowExplicitNull: true, DefaultNil: true, StructFieldValidations: []*cr.StructFieldValidation{ httpGetHandlerValidation(), execHandlerValidation(), }, }, } } func httpGetHandlerValidation() *cr.StructFieldValidation { return &cr.StructFieldValidation{ StructField: "HTTPGet", StructValidation: &cr.StructValidation{ Required: false, AllowExplicitNull: true, DefaultNil: true, StructFieldValidations: []*cr.StructFieldValidation{ { StructField: "Path", StringValidation: &cr.StringValidation{ Required: false, Default: "/", Validator: urls.ValidateEndpointAllowEmptyPath, }, }, { StructField: "Port", Int32Validation: &cr.Int32Validation{ Required: true, GreaterThan: pointer.Int32(0), LessThanOrEqualTo: pointer.Int32(65535), DisallowedValues: consts.ReservedContainerPorts, }, }, }, }, } } func tcpSocketHandlerValidation() *cr.StructFieldValidation { return &cr.StructFieldValidation{ StructField: "TCPSocket", StructValidation: &cr.StructValidation{ Required: false, AllowExplicitNull: true, DefaultNil: true, StructFieldValidations: []*cr.StructFieldValidation{ { StructField: "Port", Int32Validation: &cr.Int32Validation{ Required: true, GreaterThan: pointer.Int32(0), LessThanOrEqualTo: pointer.Int32(65535), DisallowedValues: consts.ReservedContainerPorts, }, }, }, }, } } func execHandlerValidation() *cr.StructFieldValidation { return &cr.StructFieldValidation{ StructField: "Exec", StructValidation: &cr.StructValidation{ Required: false, AllowExplicitNull: true, DefaultNil: true, StructFieldValidations: []*cr.StructFieldValidation{ { StructField: "Command", StringListValidation: &cr.StringListValidation{ Required: true, }, }, }, }, } } func computeValidation() *cr.StructFieldValidation { return &cr.StructFieldValidation{ StructField: "Compute", StructValidation: &cr.StructValidation{ StructFieldValidations: []*cr.StructFieldValidation{ { StructField: "CPU", StringPtrValidation: &cr.StringPtrValidation{ Default: pointer.String("200m"), AllowExplicitNull: true, CastNumeric: true, }, Parser: k8s.QuantityParser(&k8s.QuantityValidation{ GreaterThanOrEqualTo: k8s.QuantityPtr(kresource.MustParse("20m")), }), }, { StructField: "Mem", StringPtrValidation: &cr.StringPtrValidation{ Default: nil, AllowExplicitNull: true, }, Parser: k8s.QuantityParser(&k8s.QuantityValidation{ GreaterThanOrEqualTo: k8s.QuantityPtr(kresource.MustParse("20Mi")), }), }, { StructField: "GPU", Int64Validation: &cr.Int64Validation{ Default: 0, GreaterThanOrEqualTo: pointer.Int64(0), }, }, { StructField: "Inf", Int64Validation: &cr.Int64Validation{ Default: 0, GreaterThanOrEqualTo: pointer.Int64(0), }, }, { StructField: "Shm", StringPtrValidation: &cr.StringPtrValidation{ Default: nil, AllowExplicitNull: true, }, Parser: k8s.QuantityParser(&k8s.QuantityValidation{}), }, }, }, } } func autoscalingValidation() *cr.StructFieldValidation { return &cr.StructFieldValidation{ StructField: "Autoscaling", StructValidation: &cr.StructValidation{ StructFieldValidations: []*cr.StructFieldValidation{ { StructField: "MinReplicas", Int32Validation: &cr.Int32Validation{ Default: 1, GreaterThanOrEqualTo: pointer.Int32(0), }, }, { StructField: "MaxReplicas", Int32Validation: &cr.Int32Validation{ Default: 100, GreaterThan: pointer.Int32(0), }, }, { StructField: "InitReplicas", DefaultField: "MinReplicas", Int32Validation: &cr.Int32Validation{ GreaterThanOrEqualTo: pointer.Int32(0), }, }, { StructField: "TargetInFlight", Float64PtrValidation: &cr.Float64PtrValidation{ Default: nil, GreaterThan: pointer.Float64(0), }, }, { StructField: "Window", StringValidation: &cr.StringValidation{ Default: "60s", }, Parser: cr.DurationParser(&cr.DurationValidation{ GreaterThanOrEqualTo: &AutoscalingTickInterval, MultipleOf: &AutoscalingTickInterval, }), }, { StructField: "DownscaleStabilizationPeriod", StringValidation: &cr.StringValidation{ Default: "5m", }, Parser: cr.DurationParser(&cr.DurationValidation{ GreaterThanOrEqualTo: pointer.Duration(libtime.MustParseDuration("0s")), }), }, { StructField: "UpscaleStabilizationPeriod", StringValidation: &cr.StringValidation{ Default: "1m", }, Parser: cr.DurationParser(&cr.DurationValidation{ GreaterThanOrEqualTo: pointer.Duration(libtime.MustParseDuration("0s")), }), }, { StructField: "MaxDownscaleFactor", Float64Validation: &cr.Float64Validation{ Default: 0.75, GreaterThanOrEqualTo: pointer.Float64(0), LessThan: pointer.Float64(1), }, }, { StructField: "MaxUpscaleFactor", Float64Validation: &cr.Float64Validation{ Default: 1.5, GreaterThan: pointer.Float64(1), }, }, { StructField: "DownscaleTolerance", Float64Validation: &cr.Float64Validation{ Default: 0.05, GreaterThanOrEqualTo: pointer.Float64(0), LessThan: pointer.Float64(1), }, }, { StructField: "UpscaleTolerance", Float64Validation: &cr.Float64Validation{ Default: 0.05, GreaterThanOrEqualTo: pointer.Float64(0), }, }, }, }, } } func updateStrategyValidation() *cr.StructFieldValidation { return &cr.StructFieldValidation{ StructField: "UpdateStrategy", StructValidation: &cr.StructValidation{ StructFieldValidations: []*cr.StructFieldValidation{ { StructField: "MaxSurge", StringValidation: &cr.StringValidation{ Default: "25%", CastInt: true, Validator: surgeOrUnavailableValidator, }, }, { StructField: "MaxUnavailable", StringValidation: &cr.StringValidation{ Default: "25%", CastInt: true, Validator: surgeOrUnavailableValidator, }, }, }, }, } } var resourceStructValidation = cr.StructValidation{ AllowExtraFields: true, StructFieldValidations: resourceStructValidations, } func ExtractAPIConfigs(configBytes []byte, configFileName string) ([]userconfig.API, error) { var err error configData, err := cr.ReadYAMLBytes(configBytes) if err != nil { return nil, errors.Wrap(err, configFileName) } configDataSlice, ok := cast.InterfaceToStrInterfaceMapSlice(configData) if !ok { return nil, errors.Wrap(ErrorMalformedConfig(), configFileName) } apis := make([]userconfig.API, len(configDataSlice)) for i, data := range configDataSlice { api := userconfig.API{} var resourceStruct userconfig.Resource errs := cr.Struct(&resourceStruct, data, &resourceStructValidation) if errors.HasError(errs) { name, _ := data[userconfig.NameKey].(string) kindString, _ := data[userconfig.KindKey].(string) kind := userconfig.KindFromString(kindString) err = errors.Wrap(errors.FirstError(errs...), userconfig.IdentifyAPI(configFileName, name, kind, i)) return nil, errors.Append(err, fmt.Sprintf("\n\napi configuration schema can be found at https://docs.cortexlabs.com/v/%s/", consts.CortexVersionMinor)) } errs = cr.Struct(&api, data, apiValidation(resourceStruct)) if errors.HasError(errs) { name, _ := data[userconfig.NameKey].(string) kindString, _ := data[userconfig.KindKey].(string) kind := userconfig.KindFromString(kindString) err = errors.Wrap(errors.FirstError(errs...), userconfig.IdentifyAPI(configFileName, name, kind, i)) return nil, errors.Append(err, fmt.Sprintf("\n\napi configuration schema can be found at https://docs.cortexlabs.com/v/%s/", consts.CortexVersionMinor)) } api.Index = i api.FileName = configFileName interfaceMap, ok := cast.JSONMarshallable(data) if !ok { return nil, errors.ErrorUnexpected("unable to cast api spec to json") // unexpected } api.SubmittedAPISpec = interfaceMap apis[i] = api } return apis, nil } func ValidateAPI( api *userconfig.API, awsClient *aws.Client, k8sClient *k8s.Client, ) error { if api.Networking.Endpoint == nil { api.Networking.Endpoint = pointer.String("/" + api.Name) } if api.Pod != nil { if err := validatePod(api, awsClient, k8sClient); err != nil { return errors.Wrap(err, userconfig.PodKey) } } if api.Autoscaling != nil { if err := validateAutoscaling(api); err != nil { return errors.Wrap(err, userconfig.AutoscalingKey) } } if api.UpdateStrategy != nil { if err := validateUpdateStrategy(api.UpdateStrategy); err != nil { return errors.Wrap(err, userconfig.UpdateStrategyKey) } } return nil } func ValidateTrafficSplitter(api *userconfig.API) error { if api.Networking.Endpoint == nil { api.Networking.Endpoint = pointer.String("/" + api.Name) } if err := verifyTotalWeight(api.APIs); err != nil { return err } if err := areTrafficSplitterAPIsUnique(api.APIs); err != nil { return err } hasShadow := false for _, api := range api.APIs { if api.Shadow { if hasShadow { return ErrorOneShadowPerTrafficSplitter() } hasShadow = true } } return nil } func validatePod( api *userconfig.API, awsClient *aws.Client, k8sClient *k8s.Client, ) error { if api.Pod.Port != nil && api.Kind == userconfig.TaskAPIKind { return ErrorFieldIsNotSupportedForKind(userconfig.PortKey, api.Kind) } if api.Pod.Port == nil && api.Kind != userconfig.TaskAPIKind { api.Pod.Port = pointer.Int32(consts.DefaultUserPodPortInt32) } if err := validateCompute(api); err != nil { return errors.Wrap(err, userconfig.ComputeKey) } if err := validateContainers(api.Pod.Containers, api.Kind, awsClient, k8sClient); err != nil { return errors.Wrap(err, userconfig.ContainersKey) } return nil } func validateContainers( containers []*userconfig.Container, kind userconfig.Kind, awsClient *aws.Client, k8sClient *k8s.Client, ) error { containerNames := []string{} for i, container := range containers { if slices.HasString(containerNames, container.Name) { return errors.Wrap(ErrorDuplicateContainerName(container.Name), s.Index(i), userconfig.ImageKey) } containerNames = append(containerNames, container.Name) if container.Command == nil && (kind == userconfig.BatchAPIKind || kind == userconfig.TaskAPIKind) { return errors.Wrap(ErrorFieldMustBeSpecifiedForKind(userconfig.CommandKey, kind), s.Index(i), userconfig.CommandKey) } if err := validateDockerImagePath(container.Image, awsClient, k8sClient); err != nil { return errors.Wrap(err, s.Index(i), userconfig.ImageKey) } for key := range container.Env { if strings.HasPrefix(key, "CORTEX_") || strings.HasPrefix(key, "KUBEXIT_") { return errors.Wrap(ErrorCortexPrefixedEnvVarNotAllowed("CORTEX_", "KUBEXIT_"), s.Index(i), userconfig.EnvKey, key) } } if kind == userconfig.TaskAPIKind && container.ReadinessProbe != nil { return errors.Wrap(ErrorFieldIsNotSupportedForKind(userconfig.ReadinessProbeKey, kind), s.Index(i), userconfig.ReadinessProbeKey) } if container.ReadinessProbe != nil { supportsExecProbe := kind == userconfig.RealtimeAPIKind if err := validateProbe(*container.ReadinessProbe, supportsExecProbe); err != nil { return errors.Wrap(err, s.Index(i), userconfig.ReadinessProbeKey) } } if container.LivenessProbe != nil { if err := validateProbe(*container.LivenessProbe, true); err != nil { return errors.Wrap(err, s.Index(i), userconfig.LivenessProbeKey) } } if container.PreStop != nil { if err := validatePreStop(*container.PreStop); err != nil { return errors.Wrap(err, s.Index(i), userconfig.PreStopKey) } } compute := container.Compute if compute.Shm != nil && compute.Mem != nil && compute.Shm.Cmp(compute.Mem.Quantity) > 0 { return errors.Wrap(ErrorShmCannotExceedMem(*compute.Shm, *compute.Mem), s.Index(i), userconfig.ComputeKey) } } return nil } func validateProbe(probe userconfig.Probe, supportsExecProbe bool) error { numSpecifiedHandlers := 0 if probe.HTTPGet != nil { numSpecifiedHandlers++ } if probe.TCPSocket != nil { numSpecifiedHandlers++ } if probe.Exec != nil { numSpecifiedHandlers++ } if numSpecifiedHandlers != 1 { validHandlers := []string{userconfig.HTTPGetKey, userconfig.TCPSocketKey} if supportsExecProbe { validHandlers = append(validHandlers, userconfig.ExecKey) } return ErrorSpecifyExactlyOneField(numSpecifiedHandlers, validHandlers...) } return nil } func validatePreStop(preStop userconfig.PreStop) error { numSpecifiedHandlers := 0 if preStop.HTTPGet != nil { numSpecifiedHandlers++ } if preStop.Exec != nil { numSpecifiedHandlers++ } if numSpecifiedHandlers != 1 { return ErrorSpecifyExactlyOneField(numSpecifiedHandlers, userconfig.HTTPGetKey, userconfig.ExecKey) } return nil } func validateAutoscaling(api *userconfig.API) error { autoscaling := api.Autoscaling pod := api.Pod if api.Kind == userconfig.RealtimeAPIKind { if autoscaling.TargetInFlight == nil { autoscaling.TargetInFlight = pointer.Float64(float64(pod.MaxConcurrency)) } if *autoscaling.TargetInFlight > float64(pod.MaxConcurrency)+float64(pod.MaxQueueLength) { return ErrorTargetInFlightLimitReached(*autoscaling.TargetInFlight, pod.MaxConcurrency, pod.MaxQueueLength) } } if api.Kind == userconfig.AsyncAPIKind { if autoscaling.TargetInFlight == nil { autoscaling.TargetInFlight = pointer.Float64(float64(pod.MaxConcurrency)) } } if autoscaling.MinReplicas > autoscaling.MaxReplicas { return ErrorMinReplicasGreaterThanMax(autoscaling.MinReplicas, autoscaling.MaxReplicas) } if autoscaling.InitReplicas > autoscaling.MaxReplicas { return ErrorInitReplicasGreaterThanMax(autoscaling.InitReplicas, autoscaling.MaxReplicas) } if autoscaling.InitReplicas < autoscaling.MinReplicas { return ErrorInitReplicasLessThanMin(autoscaling.InitReplicas, autoscaling.MinReplicas) } return nil } func validateCompute(api *userconfig.API) error { compute := userconfig.GetPodComputeRequest(api) if compute.GPU > 0 && compute.Inf > 0 { return ErrorComputeResourceConflict(userconfig.GPUKey, userconfig.InfKey) } return nil } func validateUpdateStrategy(updateStrategy *userconfig.UpdateStrategy) error { if (updateStrategy.MaxSurge == "0" || updateStrategy.MaxSurge == "0%") && (updateStrategy.MaxUnavailable == "0" || updateStrategy.MaxUnavailable == "0%") { return ErrorSurgeAndUnavailableBothZero() } return nil } func validateDockerImagePath( image string, awsClient *aws.Client, k8sClient *k8s.Client, ) error { dockerClient, err := docker.GetDockerClient() if err != nil { return err } dockerAuthStr := docker.NoAuth if regex.IsValidECRURL(image) { dockerAuthStr, err = docker.AWSAuthConfig(awsClient) if err != nil { return err } } else if k8sClient != nil { dockerAuthStr, err = getDockerAuthStrFromK8s(dockerClient, k8sClient) if err != nil { return err } } if err := docker.CheckImageAccessible(dockerClient, image, dockerAuthStr); err != nil { return err } return nil } func getDockerAuthStrFromK8s(dockerClient *docker.Client, k8sClient *k8s.Client) (string, error) { secretData, err := k8sClient.GetSecretData(_dockerPullSecretName) if err != nil { return "", err } // check if the user provided the registry auth secret if secretData == nil { return docker.NoAuth, nil } authData, ok := secretData[".dockerconfigjson"] if !ok { return "", ErrorUnexpectedDockerSecretData("should contain \".dockerconfigjson\" key", secretData) } var authSecret struct { Auths map[string]struct { Username string `json:"username"` Password string `json:"password"` } `json:"auths"` } err = libjson.Unmarshal(authData, &authSecret) if err != nil { return "", ErrorUnexpectedDockerSecretData(errors.Message(err), secretData) } if len(authSecret.Auths) != 1 { return "", ErrorUnexpectedDockerSecretData("should contain a single set of credentials", secretData) } var dockerAuth dockertypes.AuthConfig for registryAddress, creds := range authSecret.Auths { dockerAuth = dockertypes.AuthConfig{ Username: creds.Username, Password: creds.Password, ServerAddress: registryAddress, } } if dockerAuth.Username == "" { return "", ErrorUnexpectedDockerSecretData("missing username", secretData) } if dockerAuth.Password == "" { return "", ErrorUnexpectedDockerSecretData("missing password", secretData) } if dockerAuth.ServerAddress == "" { return "", ErrorUnexpectedDockerSecretData("missing registry address", secretData) } _, err = dockerClient.RegistryLogin(context.Background(), dockerAuth) if err != nil { return "", err } dockerAuthStr, err := docker.EncodeAuthConfig(dockerAuth) if err != nil { return "", err } return dockerAuthStr, nil } ================================================ FILE: pkg/types/status/job_code.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package status // JobCode is an enum to represent a job status // +kubebuilder:validation:Type=string type JobCode int // Possible values for JobCode const ( JobPending JobCode = iota // pending should be the first status in this list JobEnqueuing JobRunning JobEnqueueFailed JobCompletedWithFailures JobSucceeded JobUnexpectedError JobWorkerError JobWorkerOOM JobTimedOut JobStopped JobUnknown ) var _jobCodes = []string{ "pending", "enqueuing", "running", "enqueue_failed", "completed_with_failures", "succeeded", "unexpected_error", "worker_error", "worker_oom", "timed_out", "stopped", "unknown", } var _ = [1]int{}[int(JobUnknown)-(len(_jobCodes)-1)] // Ensure list length matches var _jobCodeMessages = []string{ "pending", "enqueuing", "running", "failed while enqueuing", "completed with failures", "succeeded", "unexpected error", "worker error", "out of memory", "timed out", "stopped", "unknown", } var _ = [1]int{}[int(JobUnknown)-(len(_jobCodeMessages)-1)] // Ensure list length matches func (code JobCode) IsNotStarted() bool { return code == JobPending || code == JobEnqueuing } func (code JobCode) IsInProgress() bool { return code == JobEnqueuing || code == JobRunning } func (code JobCode) IsCompleted() bool { return code == JobEnqueueFailed || code == JobCompletedWithFailures || code == JobSucceeded || code == JobUnexpectedError || code == JobWorkerError || code == JobWorkerOOM || code == JobStopped || code == JobTimedOut } func (code JobCode) String() string { if int(code) < 0 || int(code) >= len(_jobCodes) { return _jobCodes[JobUnknown] } return _jobCodes[code] } func (code JobCode) Message() string { if int(code) < 0 || int(code) >= len(_jobCodeMessages) { return _jobCodeMessages[JobUnknown] } return _jobCodeMessages[code] } // MarshalText satisfies TextMarshaler func (code JobCode) MarshalText() ([]byte, error) { return []byte(code.String()), nil } // UnmarshalText satisfies TextUnmarshaler func (code *JobCode) UnmarshalText(text []byte) error { enum := string(text) for i := 0; i < len(_jobCodes); i++ { if enum == _jobCodes[i] { *code = JobCode(i) return nil } } *code = JobUnknown return nil } // UnmarshalBinary satisfies BinaryUnmarshaler // Needed for msgpack func (code *JobCode) UnmarshalBinary(data []byte) error { return code.UnmarshalText(data) } // MarshalBinary satisfies BinaryMarshaler func (code JobCode) MarshalBinary() ([]byte, error) { return []byte(code.String()), nil } ================================================ FILE: pkg/types/status/job_status.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package status import ( "time" "github.com/cortexlabs/cortex/pkg/types/spec" ) type BatchJobStatus struct { spec.BatchJob Status JobCode `json:"status" yaml:"status"` EndTime *time.Time `json:"end_time,omitempty" yaml:"end_time,omitempty"` BatchesInQueue int `json:"batches_in_queue" yaml:"batches_in_queue"` WorkerCounts *WorkerCounts `json:"worker_counts,omitempty" yaml:"worker_counts,omitempty"` } type TaskJobStatus struct { spec.TaskJob EndTime *time.Time `json:"end_time,omitempty" yaml:"end_time,omitempty"` Status JobCode `json:"status" yaml:"status"` WorkerCounts *WorkerCounts `json:"worker_counts,omitempty" yaml:"worker_counts,omitempty"` } ================================================ FILE: pkg/types/status/status.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package status import ( kapps "k8s.io/api/apps/v1" ) type Status struct { Ready int32 `json:"ready" yaml:"ready"` // deployment-reported number of ready replicas (latest + out of date) Requested int32 `json:"requested" yaml:"requested"` // deployment-reported number of requested replicas UpToDate int32 `json:"up_to_date" yaml:"up_to_date"` // deployment-reported number of up-to-date replicas (in whichever phase they are found in) ReplicaCounts *ReplicaCounts `json:"replica_counts,omitempty" yaml:"replica_counts,omitempty"` } type ReplicaCountType string const ( ReplicaCountRequested ReplicaCountType = "Requested" // requested number of replicas (for up-to-date pods) ReplicaCountPending ReplicaCountType = "Pending" // pods that are in the pending state (for up-to-date pods) ReplicaCountCreating ReplicaCountType = "Creating" // pods that that have their init/non-init containers in the process of being created (for up-to-date pods) ReplicaCountNotReady ReplicaCountType = "NotReady" // pods that are not passing the readiness checks (for up-to-date pods) ReplicaCountReady ReplicaCountType = "Ready" // pods that are passing the readiness checks (for up-to-date pods) ReplicaCountReadyOutOfDate ReplicaCountType = "ReadyOutOfDate" // pods that are passing the readiness checks (for out-of-date pods) ReplicaCountErrImagePull ReplicaCountType = "ErrImagePull" // pods that couldn't pull the containers' images (for up-to-date pods) ReplicaCountTerminating ReplicaCountType = "Terminating" // pods that are in a terminating state (for up-to-date pods) ReplicaCountFailed ReplicaCountType = "Failed" // pods that have had their containers erroring (for up-to-date pods) ReplicaCountKilled ReplicaCountType = "Killed" // pods that have had their container processes killed (for up-to-date pods) ReplicaCountKilledOOM ReplicaCountType = "KilledOOM" // pods that have had their containers OOM (for up-to-date pods) ReplicaCountStalled ReplicaCountType = "Stalled" // pods that have been in a pending state for more than 15 mins (for up-to-date pods) ReplicaCountUnknown ReplicaCountType = "Unknown" // pods that are in an unknown state (for up-to-date pods) ) var ReplicaCountTypes []ReplicaCountType = []ReplicaCountType{ ReplicaCountRequested, ReplicaCountPending, ReplicaCountCreating, ReplicaCountNotReady, ReplicaCountReady, ReplicaCountReadyOutOfDate, ReplicaCountErrImagePull, ReplicaCountTerminating, ReplicaCountFailed, ReplicaCountKilled, ReplicaCountKilledOOM, ReplicaCountStalled, ReplicaCountUnknown, } type ReplicaCounts struct { Requested int32 `json:"requested" yaml:"requested"` Pending int32 `json:"pending" yaml:"pending"` Creating int32 `json:"creating" yaml:"creating"` NotReady int32 `json:"not_ready" yaml:"not_ready"` Ready int32 `json:"ready" yaml:"ready"` ReadyOutOfDate int32 `json:"ready_out_of_date" yaml:"ready_out_of_date"` ErrImagePull int32 `json:"err_image_pull" yaml:"err_image_pull"` Terminating int32 `json:"terminating" yaml:"terminating"` // includes up-to-date and out-of-date pods Failed int32 `json:"failed" yaml:"failed"` Killed int32 `json:"killed" yaml:"killed"` KilledOOM int32 `json:"killed_oom" yaml:"killed_oom"` Stalled int32 `json:"stalled" yaml:"stalled"` // pending for a long time Unknown int32 `json:"unknown" yaml:"unknown"` } // Worker counts don't have as many failure variations because Jobs clean up dead pods, so counting different failure scenarios isn't interesting type WorkerCounts struct { Pending int32 `json:"pending,omitempty" yaml:"pending,omitempty"` Creating int32 `json:"creating,omitempty" yaml:"creating,omitempty"` NotReady int32 `json:"not_ready,omitempty" yaml:"not_ready,omitempty"` Ready int32 `json:"ready,omitempty" yaml:"ready,omitempty"` Succeeded int32 `json:"succeeded,omitempty" yaml:"succeeded,omitempty"` ErrImagePull int32 `json:"err_image_pull,omitempty" yaml:"err_image_pull,omitempty"` Terminating int32 `json:"terminating,omitempty" yaml:"terminating,omitempty"` Failed int32 `json:"failed,omitempty" yaml:"failed,omitempty"` Killed int32 `json:"killed,omitempty" yaml:"killed,omitempty"` KilledOOM int32 `json:"killed_oom,omitempty" yaml:"killed_oom,omitempty"` Stalled int32 `json:"stalled,omitempty" yaml:"stalled,omitempty"` // pending for a long time Unknown int32 `json:"unknown,omitempty" yaml:"unknown,omitempty"` } func FromDeployment(deployment *kapps.Deployment) *Status { return &Status{ Ready: deployment.Status.ReadyReplicas, Requested: deployment.Status.Replicas, UpToDate: deployment.Status.UpdatedReplicas, } } func (counts *ReplicaCounts) GetCountBy(replicaType ReplicaCountType) int32 { switch replicaType { case ReplicaCountRequested: return counts.Requested case ReplicaCountPending: return counts.Pending case ReplicaCountCreating: return counts.Creating case ReplicaCountNotReady: return counts.NotReady case ReplicaCountReady: return counts.Ready case ReplicaCountReadyOutOfDate: return counts.ReadyOutOfDate case ReplicaCountErrImagePull: return counts.ErrImagePull case ReplicaCountTerminating: return counts.Terminating case ReplicaCountFailed: return counts.Failed case ReplicaCountKilled: return counts.Killed case ReplicaCountKilledOOM: return counts.KilledOOM case ReplicaCountStalled: return counts.Stalled } return counts.Unknown } func (counts *ReplicaCounts) TotalFailed() int32 { return counts.ErrImagePull + counts.Failed + counts.Killed + counts.KilledOOM + counts.Unknown } ================================================ FILE: pkg/types/userconfig/api.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package userconfig import ( "fmt" "strings" "time" "github.com/cortexlabs/cortex/pkg/consts" "github.com/cortexlabs/cortex/pkg/lib/k8s" "github.com/cortexlabs/cortex/pkg/lib/pointer" "github.com/cortexlabs/cortex/pkg/lib/sets/strset" s "github.com/cortexlabs/cortex/pkg/lib/strings" "github.com/cortexlabs/cortex/pkg/lib/urls" "github.com/cortexlabs/yaml" kresource "k8s.io/apimachinery/pkg/api/resource" kmeta "k8s.io/apimachinery/pkg/apis/meta/v1" ) type API struct { Resource Pod *Pod `json:"pod" yaml:"pod"` NodeGroups []string `json:"node_groups" yaml:"node_groups"` APIs []*TrafficSplit `json:"apis" yaml:"apis"` Networking *Networking `json:"networking" yaml:"networking"` Autoscaling *Autoscaling `json:"autoscaling" yaml:"autoscaling"` UpdateStrategy *UpdateStrategy `json:"update_strategy" yaml:"update_strategy"` Index int `json:"index" yaml:"-"` FileName string `json:"file_name" yaml:"-"` SubmittedAPISpec interface{} `json:"submitted_api_spec" yaml:"submitted_api_spec"` } type Pod struct { Port *int32 `json:"port" yaml:"port"` MaxQueueLength int64 `json:"max_queue_length" yaml:"max_queue_length"` MaxConcurrency int64 `json:"max_concurrency" yaml:"max_concurrency"` Containers []*Container `json:"containers" yaml:"containers"` } type Container struct { Name string `json:"name" yaml:"name"` Image string `json:"image" yaml:"image"` Env map[string]string `json:"env" yaml:"env"` Command []string `json:"command" yaml:"command"` Args []string `json:"args" yaml:"args"` ReadinessProbe *Probe `json:"readiness_probe" yaml:"readiness_probe"` LivenessProbe *Probe `json:"liveness_probe" yaml:"liveness_probe"` PreStop *PreStop `json:"pre_stop" yaml:"pre_stop"` Compute *Compute `json:"compute" yaml:"compute"` } type TrafficSplit struct { Name string `json:"name" yaml:"name"` Weight int32 `json:"weight" yaml:"weight"` Shadow bool `json:"shadow" yaml:"shadow"` } type Networking struct { Endpoint *string `json:"endpoint" yaml:"endpoint"` } type Probe struct { HTTPGet *HTTPGetHandler `json:"http_get" yaml:"http_get"` TCPSocket *TCPSocketHandler `json:"tcp_socket" yaml:"tcp_socket"` Exec *ExecHandler `json:"exec" yaml:"exec"` InitialDelaySeconds int32 `json:"initial_delay_seconds" yaml:"initial_delay_seconds"` TimeoutSeconds int32 `json:"timeout_seconds" yaml:"timeout_seconds"` PeriodSeconds int32 `json:"period_seconds" yaml:"period_seconds"` SuccessThreshold int32 `json:"success_threshold" yaml:"success_threshold"` FailureThreshold int32 `json:"failure_threshold" yaml:"failure_threshold"` } type PreStop struct { HTTPGet *HTTPGetHandler `json:"http_get" yaml:"http_get"` Exec *ExecHandler `json:"exec" yaml:"exec"` } type HTTPGetHandler struct { Path string `json:"path" yaml:"path"` Port int32 `json:"port" yaml:"port"` } type TCPSocketHandler struct { Port int32 `json:"port" yaml:"port"` } type ExecHandler struct { Command []string `json:"command" yaml:"command"` } type Compute struct { CPU *k8s.Quantity `json:"cpu" yaml:"cpu"` Mem *k8s.Quantity `json:"mem" yaml:"mem"` GPU int64 `json:"gpu" yaml:"gpu"` Inf int64 `json:"inf" yaml:"inf"` Shm *k8s.Quantity `json:"shm" yaml:"shm"` } type Autoscaling struct { MinReplicas int32 `json:"min_replicas" yaml:"min_replicas"` MaxReplicas int32 `json:"max_replicas" yaml:"max_replicas"` InitReplicas int32 `json:"init_replicas" yaml:"init_replicas"` TargetInFlight *float64 `json:"target_in_flight" yaml:"target_in_flight"` Window time.Duration `json:"window" yaml:"window"` DownscaleStabilizationPeriod time.Duration `json:"downscale_stabilization_period" yaml:"downscale_stabilization_period"` UpscaleStabilizationPeriod time.Duration `json:"upscale_stabilization_period" yaml:"upscale_stabilization_period"` MaxDownscaleFactor float64 `json:"max_downscale_factor" yaml:"max_downscale_factor"` MaxUpscaleFactor float64 `json:"max_upscale_factor" yaml:"max_upscale_factor"` DownscaleTolerance float64 `json:"downscale_tolerance" yaml:"downscale_tolerance"` UpscaleTolerance float64 `json:"upscale_tolerance" yaml:"upscale_tolerance"` } type UpdateStrategy struct { MaxSurge string `json:"max_surge" yaml:"max_surge"` MaxUnavailable string `json:"max_unavailable" yaml:"max_unavailable"` } func (api *API) Identify() string { return IdentifyAPI(api.FileName, api.Name, api.Kind, api.Index) } func IdentifyAPI(filePath string, name string, kind Kind, index int) string { str := "" if filePath != "" { str += filePath + ": " } if name != "" { str += name if kind != UnknownKind { str += " (" + kind.String() + ")" } return str } else if index >= 0 { return str + "resource at " + s.Index(index) } return str + "resource" } // InitReplicas was left out deliberately func (api *API) ToK8sAnnotations() map[string]string { annotations := map[string]string{} if len(api.APIs) > 0 { annotations[NumTrafficSplitterTargetsAnnotationKey] = s.Int32(int32(len(api.APIs))) } if api.Pod != nil && api.Kind == RealtimeAPIKind { annotations[MaxConcurrencyAnnotationKey] = s.Int64(api.Pod.MaxConcurrency) annotations[MaxQueueLengthAnnotationKey] = s.Int64(api.Pod.MaxQueueLength) } if api.Pod != nil && api.Kind == AsyncAPIKind { annotations[MaxConcurrencyAnnotationKey] = s.Int64(api.Pod.MaxConcurrency) } if api.Networking != nil { annotations[EndpointAnnotationKey] = *api.Networking.Endpoint } if api.Autoscaling != nil { annotations[MinReplicasAnnotationKey] = s.Int32(api.Autoscaling.MinReplicas) annotations[MaxReplicasAnnotationKey] = s.Int32(api.Autoscaling.MaxReplicas) annotations[TargetInFlightAnnotationKey] = s.Float64(*api.Autoscaling.TargetInFlight) annotations[WindowAnnotationKey] = api.Autoscaling.Window.String() annotations[DownscaleStabilizationPeriodAnnotationKey] = api.Autoscaling.DownscaleStabilizationPeriod.String() annotations[UpscaleStabilizationPeriodAnnotationKey] = api.Autoscaling.UpscaleStabilizationPeriod.String() annotations[MaxDownscaleFactorAnnotationKey] = s.Float64(api.Autoscaling.MaxDownscaleFactor) annotations[MaxUpscaleFactorAnnotationKey] = s.Float64(api.Autoscaling.MaxUpscaleFactor) annotations[DownscaleToleranceAnnotationKey] = s.Float64(api.Autoscaling.DownscaleTolerance) annotations[UpscaleToleranceAnnotationKey] = s.Float64(api.Autoscaling.UpscaleTolerance) } return annotations } func AutoscalingFromAnnotations(k8sObj kmeta.Object) (*Autoscaling, error) { a := Autoscaling{} minReplicas, err := k8s.ParseInt32Annotation(k8sObj, MinReplicasAnnotationKey) if err != nil { return nil, err } a.MinReplicas = minReplicas maxReplicas, err := k8s.ParseInt32Annotation(k8sObj, MaxReplicasAnnotationKey) if err != nil { return nil, err } a.MaxReplicas = maxReplicas targetInFlight, err := k8s.ParseFloat64Annotation(k8sObj, TargetInFlightAnnotationKey) if err != nil { return nil, err } a.TargetInFlight = pointer.Float64(targetInFlight) window, err := k8s.ParseDurationAnnotation(k8sObj, WindowAnnotationKey) if err != nil { return nil, err } a.Window = window downscaleStabilizationPeriod, err := k8s.ParseDurationAnnotation(k8sObj, DownscaleStabilizationPeriodAnnotationKey) if err != nil { return nil, err } a.DownscaleStabilizationPeriod = downscaleStabilizationPeriod upscaleStabilizationPeriod, err := k8s.ParseDurationAnnotation(k8sObj, UpscaleStabilizationPeriodAnnotationKey) if err != nil { return nil, err } a.UpscaleStabilizationPeriod = upscaleStabilizationPeriod maxDownscaleFactor, err := k8s.ParseFloat64Annotation(k8sObj, MaxDownscaleFactorAnnotationKey) if err != nil { return nil, err } a.MaxDownscaleFactor = maxDownscaleFactor maxUpscaleFactor, err := k8s.ParseFloat64Annotation(k8sObj, MaxUpscaleFactorAnnotationKey) if err != nil { return nil, err } a.MaxUpscaleFactor = maxUpscaleFactor downscaleTolerance, err := k8s.ParseFloat64Annotation(k8sObj, DownscaleToleranceAnnotationKey) if err != nil { return nil, err } a.DownscaleTolerance = downscaleTolerance upscaleTolerance, err := k8s.ParseFloat64Annotation(k8sObj, UpscaleToleranceAnnotationKey) if err != nil { return nil, err } a.UpscaleTolerance = upscaleTolerance return &a, nil } func TrafficSplitterTargetsFromAnnotations(k8sObj kmeta.Object) (int32, error) { targets, err := k8s.ParseInt32Annotation(k8sObj, NumTrafficSplitterTargetsAnnotationKey) if err != nil { return 0, err } return targets, nil } func EndpointFromAnnotation(k8sObj kmeta.Object) (string, error) { endpoint, err := k8s.GetAnnotation(k8sObj, EndpointAnnotationKey) if err != nil { return "", err } return endpoint, nil } func ConcurrencyFromAnnotations(k8sObj kmeta.Object) (int, int, error) { maxQueueLength, err := k8s.ParseIntAnnotation(k8sObj, MaxQueueLengthAnnotationKey) if err != nil { return 0, 0, err } maxConcurrency, err := k8s.ParseIntAnnotation(k8sObj, MaxConcurrencyAnnotationKey) if err != nil { return 0, 0, err } return maxQueueLength, maxConcurrency, nil } func (api *API) UserStr() string { var sb strings.Builder sb.WriteString(fmt.Sprintf("%s: %s\n", NameKey, api.Name)) sb.WriteString(fmt.Sprintf("%s: %s\n", KindKey, api.Kind.String())) if api.Kind == TrafficSplitterKind { sb.WriteString(fmt.Sprintf("%s:\n", APIsKey)) for _, api := range api.APIs { sb.WriteString(s.Indent(api.UserStr(), " ")) } } if api.Pod != nil { sb.WriteString(fmt.Sprintf("%s:\n", PodKey)) sb.WriteString(s.Indent(api.Pod.UserStr(api.Kind), " ")) } if api.Networking != nil { sb.WriteString(fmt.Sprintf("%s:\n", NetworkingKey)) sb.WriteString(s.Indent(api.Networking.UserStr(), " ")) } if api.Autoscaling != nil { sb.WriteString(fmt.Sprintf("%s:\n", AutoscalingKey)) sb.WriteString(s.Indent(api.Autoscaling.UserStr(), " ")) } if api.NodeGroups == nil { sb.WriteString(fmt.Sprintf("%s: null\n", NodeGroupsKey)) } else { sb.WriteString(fmt.Sprintf("%s: %s\n", NodeGroupsKey, s.ObjFlatNoQuotes(api.NodeGroups))) } if api.UpdateStrategy != nil { sb.WriteString(fmt.Sprintf("%s:\n", UpdateStrategyKey)) sb.WriteString(s.Indent(api.UpdateStrategy.UserStr(), " ")) } return sb.String() } func (trafficSplit *TrafficSplit) UserStr() string { var sb strings.Builder sb.WriteString(fmt.Sprintf("%s: %s\n", NameKey, trafficSplit.Name)) sb.WriteString(fmt.Sprintf("%s: %s\n", WeightKey, s.Int32(trafficSplit.Weight))) sb.WriteString(fmt.Sprintf("%s: %s\n", ShadowKey, s.Bool(trafficSplit.Shadow))) return sb.String() } func (pod *Pod) UserStr(kind Kind) string { var sb strings.Builder if pod.Port != nil { sb.WriteString(fmt.Sprintf("%s: %d\n", PortKey, *pod.Port)) } if kind == RealtimeAPIKind { sb.WriteString(fmt.Sprintf("%s: %s\n", MaxConcurrencyKey, s.Int64(pod.MaxConcurrency))) sb.WriteString(fmt.Sprintf("%s: %s\n", MaxQueueLengthKey, s.Int64(pod.MaxQueueLength))) } if kind == AsyncAPIKind { sb.WriteString(fmt.Sprintf("%s: %s\n", MaxConcurrencyKey, s.Int64(pod.MaxConcurrency))) } sb.WriteString(fmt.Sprintf("%s:\n", ContainersKey)) for _, container := range pod.Containers { containerUserStr := s.Indent(container.UserStr(), " ") containerUserStr = containerUserStr[:2] + "-" + containerUserStr[3:] sb.WriteString(containerUserStr) } return sb.String() } func (container *Container) UserStr() string { var sb strings.Builder sb.WriteString(fmt.Sprintf("%s: %s\n", ContainerNameKey, container.Name)) sb.WriteString(fmt.Sprintf("%s: %s\n", ImageKey, container.Image)) if len(container.Env) > 0 { sb.WriteString(fmt.Sprintf("%s:\n", EnvKey)) d, _ := yaml.Marshal(&container.Env) sb.WriteString(s.Indent(string(d), " ")) } if container.Command != nil { sb.WriteString(fmt.Sprintf("%s: %s\n", CommandKey, s.ObjFlatNoQuotes(container.Command))) } if container.Args != nil { sb.WriteString(fmt.Sprintf("%s: %s\n", ArgsKey, s.ObjFlatNoQuotes(container.Args))) } if container.ReadinessProbe != nil { sb.WriteString(fmt.Sprintf("%s:\n", ReadinessProbeKey)) sb.WriteString(s.Indent(container.ReadinessProbe.UserStr(), " ")) } if container.LivenessProbe != nil { sb.WriteString(fmt.Sprintf("%s:\n", LivenessProbeKey)) sb.WriteString(s.Indent(container.LivenessProbe.UserStr(), " ")) } if container.PreStop != nil { sb.WriteString(fmt.Sprintf("%s:\n", PreStopKey)) sb.WriteString(s.Indent(container.PreStop.UserStr(), " ")) } if container.Compute != nil { sb.WriteString(fmt.Sprintf("%s:\n", ComputeKey)) sb.WriteString(s.Indent(container.Compute.UserStr(), " ")) } return sb.String() } func (networking *Networking) UserStr() string { var sb strings.Builder if networking.Endpoint != nil { sb.WriteString(fmt.Sprintf("%s: %s\n", EndpointKey, *networking.Endpoint)) } return sb.String() } func (probe *Probe) UserStr() string { var sb strings.Builder if probe.HTTPGet != nil { sb.WriteString(fmt.Sprintf("%s:\n", HTTPGetKey)) sb.WriteString(s.Indent(probe.HTTPGet.UserStr(), " ")) } if probe.TCPSocket != nil { sb.WriteString(fmt.Sprintf("%s:\n", TCPSocketKey)) sb.WriteString(s.Indent(probe.TCPSocket.UserStr(), " ")) } if probe.Exec != nil { sb.WriteString(fmt.Sprintf("%s:\n", ExecKey)) sb.WriteString(s.Indent(probe.Exec.UserStr(), " ")) } sb.WriteString(fmt.Sprintf("%s: %d\n", InitialDelaySecondsKey, probe.InitialDelaySeconds)) sb.WriteString(fmt.Sprintf("%s: %d\n", TimeoutSecondsKey, probe.TimeoutSeconds)) sb.WriteString(fmt.Sprintf("%s: %d\n", PeriodSecondsKey, probe.PeriodSeconds)) sb.WriteString(fmt.Sprintf("%s: %d\n", SuccessThresholdKey, probe.SuccessThreshold)) sb.WriteString(fmt.Sprintf("%s: %d\n", FailureThresholdKey, probe.FailureThreshold)) return sb.String() } func (preStop *PreStop) UserStr() string { var sb strings.Builder if preStop.HTTPGet != nil { sb.WriteString(fmt.Sprintf("%s:\n", HTTPGetKey)) sb.WriteString(s.Indent(preStop.HTTPGet.UserStr(), " ")) } if preStop.Exec != nil { sb.WriteString(fmt.Sprintf("%s:\n", ExecKey)) sb.WriteString(s.Indent(preStop.Exec.UserStr(), " ")) } return sb.String() } func (httpHandler *HTTPGetHandler) UserStr() string { var sb strings.Builder sb.WriteString(fmt.Sprintf("%s: %s\n", PathKey, httpHandler.Path)) sb.WriteString(fmt.Sprintf("%s: %d\n", PortKey, httpHandler.Port)) return sb.String() } func (tcpSocketHandler *TCPSocketHandler) UserStr() string { var sb strings.Builder sb.WriteString(fmt.Sprintf("%s: %d\n", PortKey, tcpSocketHandler.Port)) return sb.String() } func (execHandler *ExecHandler) UserStr() string { var sb strings.Builder sb.WriteString(fmt.Sprintf("%s: %s\n", CommandKey, s.ObjFlatNoQuotes(execHandler.Command))) return sb.String() } func (compute *Compute) UserStr() string { var sb strings.Builder if compute.CPU == nil { sb.WriteString(fmt.Sprintf("%s: null # no limit\n", CPUKey)) } else { sb.WriteString(fmt.Sprintf("%s: %s\n", CPUKey, compute.CPU.UserString)) } if compute.GPU > 0 { sb.WriteString(fmt.Sprintf("%s: %s\n", GPUKey, s.Int64(compute.GPU))) } if compute.Inf > 0 { sb.WriteString(fmt.Sprintf("%s: %s\n", InfKey, s.Int64(compute.Inf))) } if compute.Mem == nil { sb.WriteString(fmt.Sprintf("%s: null # no limit\n", MemKey)) } else { sb.WriteString(fmt.Sprintf("%s: %s\n", MemKey, compute.Mem.UserString)) } if compute.Shm == nil { sb.WriteString(fmt.Sprintf("%s: null # not configured\n", ShmKey)) } else { sb.WriteString(fmt.Sprintf("%s: %s\n", ShmKey, compute.Shm.UserString)) } return sb.String() } func (autoscaling *Autoscaling) UserStr() string { var sb strings.Builder sb.WriteString(fmt.Sprintf("%s: %s\n", MinReplicasKey, s.Int32(autoscaling.MinReplicas))) sb.WriteString(fmt.Sprintf("%s: %s\n", MaxReplicasKey, s.Int32(autoscaling.MaxReplicas))) sb.WriteString(fmt.Sprintf("%s: %s\n", InitReplicasKey, s.Int32(autoscaling.InitReplicas))) sb.WriteString(fmt.Sprintf("%s: %s\n", TargetInFlightKey, s.Float64(*autoscaling.TargetInFlight))) sb.WriteString(fmt.Sprintf("%s: %s\n", WindowKey, autoscaling.Window.String())) sb.WriteString(fmt.Sprintf("%s: %s\n", DownscaleStabilizationPeriodKey, autoscaling.DownscaleStabilizationPeriod.String())) sb.WriteString(fmt.Sprintf("%s: %s\n", UpscaleStabilizationPeriodKey, autoscaling.UpscaleStabilizationPeriod.String())) sb.WriteString(fmt.Sprintf("%s: %s\n", MaxDownscaleFactorKey, s.Float64(autoscaling.MaxDownscaleFactor))) sb.WriteString(fmt.Sprintf("%s: %s\n", MaxUpscaleFactorKey, s.Float64(autoscaling.MaxUpscaleFactor))) sb.WriteString(fmt.Sprintf("%s: %s\n", DownscaleToleranceKey, s.Float64(autoscaling.DownscaleTolerance))) sb.WriteString(fmt.Sprintf("%s: %s\n", UpscaleToleranceKey, s.Float64(autoscaling.UpscaleTolerance))) return sb.String() } func (updateStrategy *UpdateStrategy) UserStr() string { var sb strings.Builder sb.WriteString(fmt.Sprintf("%s: %s\n", MaxSurgeKey, updateStrategy.MaxSurge)) sb.WriteString(fmt.Sprintf("%s: %s\n", MaxUnavailableKey, updateStrategy.MaxUnavailable)) return sb.String() } func ZeroCompute() Compute { return Compute{ CPU: &k8s.Quantity{}, Mem: &k8s.Quantity{}, GPU: 0, } } func GetPodComputeRequest(api *API) Compute { var cpuQtys []kresource.Quantity var memQtys []kresource.Quantity var shmQtys []kresource.Quantity var totalGPU int64 var totalInf int64 for _, container := range api.Pod.Containers { if container == nil || container.Compute == nil { continue } if container.Compute.CPU != nil { cpuQtys = append(cpuQtys, container.Compute.CPU.Quantity) } if container.Compute.Mem != nil { memQtys = append(memQtys, container.Compute.Mem.Quantity) } if container.Compute.Shm != nil { shmQtys = append(shmQtys, container.Compute.Shm.Quantity) } totalGPU += container.Compute.GPU totalInf += container.Compute.Inf } if api.Kind == RealtimeAPIKind { cpuQtys = append(cpuQtys, consts.CortexProxyCPU) memQtys = append(memQtys, consts.CortexProxyMem) } else if api.Kind == AsyncAPIKind || api.Kind == BatchAPIKind { cpuQtys = append(cpuQtys, consts.CortexDequeuerCPU) memQtys = append(memQtys, consts.CortexDequeuerMem) } return Compute{ CPU: k8s.NewSummed(cpuQtys...), Mem: k8s.NewSummed(memQtys...), Shm: k8s.NewSummed(shmQtys...), GPU: totalGPU, Inf: totalInf, } } func GetContainerNames(containers []*Container) strset.Set { containerNames := strset.New() for _, container := range containers { if container != nil { containerNames.Add(container.Name) } } return containerNames } func (api *API) TelemetryEvent() map[string]interface{} { event := map[string]interface{}{"kind": api.Kind} if len(api.APIs) > 0 { event["apis._is_defined"] = true event["apis._len"] = len(api.APIs) } if api.Networking != nil { event["networking._is_defined"] = true if api.Networking.Endpoint != nil { event["networking.endpoint._is_defined"] = true if urls.CanonicalizeEndpoint(api.Name) != *api.Networking.Endpoint { event["networking.endpoint._is_custom"] = true } } } if api.Pod != nil { event["pod._is_defined"] = true if api.Pod.Port != nil { event["pod.port"] = *api.Pod.Port } event["pod.max_concurrency"] = api.Pod.MaxConcurrency event["pod.max_queue_length"] = api.Pod.MaxQueueLength event["pod.containers._len"] = len(api.Pod.Containers) var numReadinessProbes int var numLivenessProbes int var numPreStops int for _, container := range api.Pod.Containers { if container.ReadinessProbe != nil { numReadinessProbes++ } if container.LivenessProbe != nil { numLivenessProbes++ } if container.PreStop != nil { numPreStops++ } } event["pod.containers._num_readiness_probes"] = numReadinessProbes event["pod.containers._num_liveness_probes"] = numLivenessProbes event["pod.containers._num_pre_stops"] = numPreStops totalCompute := GetPodComputeRequest(api) if totalCompute.CPU != nil { event["pod.containers.compute.cpu._is_defined"] = true event["pod.containers.compute.cpu"] = float64(totalCompute.CPU.MilliValue()) / 1000 } if totalCompute.Mem != nil { event["pod.containers.compute.mem._is_defined"] = true event["pod.containers.compute.mem"] = totalCompute.Mem.Value() } if totalCompute.Shm != nil { event["pod.containers.compute.shm._is_defined"] = true event["pod.containers.compute.shm"] = totalCompute.Shm.Value() } event["pod.containers.compute.gpu"] = totalCompute.GPU event["pod.containers.compute.inf"] = totalCompute.Inf } event["node_groups._len"] = len(api.NodeGroups) if api.UpdateStrategy != nil { event["update_strategy._is_defined"] = true event["update_strategy.max_surge"] = api.UpdateStrategy.MaxSurge event["update_strategy.max_unavailable"] = api.UpdateStrategy.MaxUnavailable } if api.Autoscaling != nil { event["autoscaling._is_defined"] = true event["autoscaling.min_replicas"] = api.Autoscaling.MinReplicas event["autoscaling.max_replicas"] = api.Autoscaling.MaxReplicas event["autoscaling.init_replicas"] = api.Autoscaling.InitReplicas if api.Autoscaling.TargetInFlight != nil { event["autoscaling.target_in_flight._is_defined"] = true event["autoscaling.target_in_flight"] = *api.Autoscaling.TargetInFlight } event["autoscaling.window"] = api.Autoscaling.Window.Seconds() event["autoscaling.downscale_stabilization_period"] = api.Autoscaling.DownscaleStabilizationPeriod.Seconds() event["autoscaling.upscale_stabilization_period"] = api.Autoscaling.UpscaleStabilizationPeriod.Seconds() event["autoscaling.max_downscale_factor"] = api.Autoscaling.MaxDownscaleFactor event["autoscaling.max_upscale_factor"] = api.Autoscaling.MaxUpscaleFactor event["autoscaling.downscale_tolerance"] = api.Autoscaling.DownscaleTolerance event["autoscaling.upscale_tolerance"] = api.Autoscaling.UpscaleTolerance } return event } ================================================ FILE: pkg/types/userconfig/config_key.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package userconfig const ( // API NameKey = "name" KindKey = "kind" NetworkingKey = "networking" ComputeKey = "compute" AutoscalingKey = "autoscaling" UpdateStrategyKey = "update_strategy" // TrafficSplitter APIsKey = "apis" WeightKey = "weight" ShadowKey = "shadow" // Pod PodKey = "pod" NodeGroupsKey = "node_groups" PortKey = "port" MaxConcurrencyKey = "max_concurrency" MaxQueueLengthKey = "max_queue_length" ContainersKey = "containers" // Containers ContainerNameKey = "name" ImageKey = "image" EnvKey = "env" CommandKey = "command" ArgsKey = "args" ReadinessProbeKey = "readiness_probe" LivenessProbeKey = "liveness_probe" PreStopKey = "pre_stop" // Probe HTTPGetKey = "http_get" TCPSocketKey = "tcp_socket" ExecKey = "exec" InitialDelaySecondsKey = "initial_delay_seconds" TimeoutSecondsKey = "timeout_seconds" PeriodSecondsKey = "period_seconds" SuccessThresholdKey = "success_threshold" FailureThresholdKey = "failure_threshold" // Probe types PathKey = "path" // Compute CPUKey = "cpu" MemKey = "mem" GPUKey = "gpu" InfKey = "inf" ShmKey = "shm" // Networking EndpointKey = "endpoint" // Autoscaling MinReplicasKey = "min_replicas" MaxReplicasKey = "max_replicas" InitReplicasKey = "init_replicas" TargetInFlightKey = "target_in_flight" WindowKey = "window" DownscaleStabilizationPeriodKey = "downscale_stabilization_period" UpscaleStabilizationPeriodKey = "upscale_stabilization_period" MaxDownscaleFactorKey = "max_downscale_factor" MaxUpscaleFactorKey = "max_upscale_factor" DownscaleToleranceKey = "downscale_tolerance" UpscaleToleranceKey = "upscale_tolerance" // UpdateStrategy MaxSurgeKey = "max_surge" MaxUnavailableKey = "max_unavailable" // K8s annotation EndpointAnnotationKey = "networking.cortex.dev/endpoint" MaxConcurrencyAnnotationKey = "pod.cortex.dev/max-concurrency" MaxQueueLengthAnnotationKey = "pod.cortex.dev/max-queue-length" NumTrafficSplitterTargetsAnnotationKey = "apis.cortex.dev/traffic-splitter-targets" MinReplicasAnnotationKey = "autoscaling.cortex.dev/min-replicas" MaxReplicasAnnotationKey = "autoscaling.cortex.dev/max-replicas" TargetInFlightAnnotationKey = "autoscaling.cortex.dev/target-in-flight" WindowAnnotationKey = "autoscaling.cortex.dev/window" DownscaleStabilizationPeriodAnnotationKey = "autoscaling.cortex.dev/downscale-stabilization-period" UpscaleStabilizationPeriodAnnotationKey = "autoscaling.cortex.dev/upscale-stabilization-period" MaxDownscaleFactorAnnotationKey = "autoscaling.cortex.dev/max-downscale-factor" MaxUpscaleFactorAnnotationKey = "autoscaling.cortex.dev/max-upscale-factor" DownscaleToleranceAnnotationKey = "autoscaling.cortex.dev/downscale-tolerance" UpscaleToleranceAnnotationKey = "autoscaling.cortex.dev/upscale-tolerance" ) ================================================ FILE: pkg/types/userconfig/kind.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package userconfig type Kind int const ( UnknownKind Kind = iota RealtimeAPIKind BatchAPIKind TrafficSplitterKind TaskAPIKind AsyncAPIKind ) var _kinds = []string{ "unknown", "RealtimeAPI", "BatchAPI", "TrafficSplitter", "TaskAPI", "AsyncAPI", } func KindFromString(s string) Kind { for i := 0; i < len(_kinds); i++ { if s == _kinds[i] { return Kind(i) } } return UnknownKind } func KindStrings() []string { return _kinds[1:] } func (t Kind) String() string { return _kinds[t] } // MarshalText satisfies TextMarshaler func (t Kind) MarshalText() ([]byte, error) { return []byte(t.String()), nil } // UnmarshalText satisfies TextUnmarshaler func (t *Kind) UnmarshalText(text []byte) error { enum := string(text) for i := 0; i < len(_kinds); i++ { if enum == _kinds[i] { *t = Kind(i) return nil } } *t = UnknownKind return nil } // UnmarshalBinary satisfies BinaryUnmarshaler // Needed for msgpack func (t *Kind) UnmarshalBinary(data []byte) error { return t.UnmarshalText(data) } // MarshalBinary satisfies BinaryMarshaler func (t Kind) MarshalBinary() ([]byte, error) { return []byte(t.String()), nil } ================================================ FILE: pkg/types/userconfig/log_level.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package userconfig import "go.uber.org/zap/zapcore" type LogLevel int const ( UnknownLogLevel LogLevel = iota DebugLogLevel InfoLogLevel WarningLogLevel ErrorLogLevel ) var _logLevels = []string{ "unknown", "debug", "info", "warning", "error", } func LogLevelFromString(s string) LogLevel { for i := 0; i < len(_logLevels); i++ { if s == _logLevels[i] { return LogLevel(i) } } return UnknownLogLevel } func LogLevelTypes() []string { return _logLevels[1:] } func (t LogLevel) String() string { return _logLevels[t] } // MarshalText satisfies TextMarshaler func (t LogLevel) MarshalText() ([]byte, error) { return []byte(t.String()), nil } // UnmarshalText satisfies TextUnmarshaler func (t *LogLevel) UnmarshalText(text []byte) error { enum := string(text) for i := 0; i < len(_logLevels); i++ { if enum == _logLevels[i] { *t = LogLevel(i) return nil } } *t = UnknownLogLevel return nil } // UnmarshalBinary satisfies BinaryUnmarshaler // Needed for msgpack func (t *LogLevel) UnmarshalBinary(data []byte) error { return t.UnmarshalText(data) } // MarshalBinary satisfies BinaryMarshaler func (t LogLevel) MarshalBinary() ([]byte, error) { return []byte(t.String()), nil } func ToZapLogLevel(logLevel LogLevel) zapcore.Level { switch logLevel { case InfoLogLevel: return zapcore.InfoLevel case WarningLogLevel: return zapcore.WarnLevel case ErrorLogLevel: return zapcore.ErrorLevel default: return zapcore.DebugLevel } } ================================================ FILE: pkg/types/userconfig/resource.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package userconfig import "fmt" type Resource struct { Name string `json:"name" yaml:"name"` Kind Kind `json:"kind" yaml:"kind"` } func (r Resource) UserString() string { return fmt.Sprintf("%s (%s)", r.Name, r.Kind.String()) } ================================================ FILE: pkg/workloads/configmap.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package workloads import ( libjson "github.com/cortexlabs/cortex/pkg/lib/json" "github.com/cortexlabs/cortex/pkg/types/spec" kcore "k8s.io/api/core/v1" ) type ConfigMapConfig struct { BatchJob *spec.BatchJob TaskJob *spec.TaskJob Probes map[string]kcore.Probe } func (c *ConfigMapConfig) GenerateConfigMapData() (map[string]string, error) { if c == nil { return nil, nil } data := map[string]string{} if len(c.Probes) > 0 { probesEncoded, err := libjson.MarshalIndent(c.Probes) if err != nil { return nil, err } data["probes.json"] = string(probesEncoded) } if c.TaskJob != nil { jobSpecEncoded, err := libjson.MarshalIndent(*c.TaskJob) if err != nil { return nil, err } data["job.json"] = string(jobSpecEncoded) } if c.BatchJob != nil { jobSpecEncoded, err := libjson.MarshalIndent(*c.BatchJob) if err != nil { return nil, err } data["job.json"] = string(jobSpecEncoded) } return data, nil } ================================================ FILE: pkg/workloads/helpers.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package workloads import ( "path" "strings" "github.com/cortexlabs/cortex/pkg/lib/k8s" "github.com/cortexlabs/cortex/pkg/types/userconfig" kcore "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/util/intstr" ) func K8sName(apiName string) string { return "api-" + apiName } func GetProbeSpec(probe *userconfig.Probe) *kcore.Probe { if probe == nil { return nil } var httpGetAction *kcore.HTTPGetAction var tcpSocketAction *kcore.TCPSocketAction var execAction *kcore.ExecAction if probe.HTTPGet != nil { httpGetAction = &kcore.HTTPGetAction{ Path: probe.HTTPGet.Path, Port: intstr.IntOrString{ IntVal: probe.HTTPGet.Port, }, } } if probe.TCPSocket != nil { tcpSocketAction = &kcore.TCPSocketAction{ Port: intstr.IntOrString{ IntVal: probe.TCPSocket.Port, }, } } if probe.Exec != nil { execAction = &kcore.ExecAction{ Command: probe.Exec.Command, } } return &kcore.Probe{ Handler: kcore.Handler{ HTTPGet: httpGetAction, TCPSocket: tcpSocketAction, Exec: execAction, }, InitialDelaySeconds: probe.InitialDelaySeconds, TimeoutSeconds: probe.TimeoutSeconds, PeriodSeconds: probe.PeriodSeconds, SuccessThreshold: probe.SuccessThreshold, FailureThreshold: probe.FailureThreshold, } } func GetLifecycleSpec(preStop *userconfig.PreStop) *kcore.Lifecycle { if preStop == nil { return nil } var httpGetAction *kcore.HTTPGetAction var execAction *kcore.ExecAction if preStop.HTTPGet != nil { httpGetAction = &kcore.HTTPGetAction{ Path: strings.TrimPrefix(preStop.HTTPGet.Path, "/"), // the leading / is automatically added by k8s Port: intstr.IntOrString{ IntVal: preStop.HTTPGet.Port, }, } } if preStop.Exec != nil { execAction = &kcore.ExecAction{ Command: preStop.Exec.Command, } } return &kcore.Lifecycle{ PreStop: &kcore.Handler{ HTTPGet: httpGetAction, Exec: execAction, }, } } func GetReadinessProbesFromContainers(containers []*userconfig.Container) map[string]kcore.Probe { probes := map[string]kcore.Probe{} for _, container := range containers { // this should never happen, it's just a precaution if container == nil { continue } if container.ReadinessProbe != nil { probes[container.Name] = *GetProbeSpec(container.ReadinessProbe) } } return probes } func HasReadinessProbesTargetingPort(containers []*userconfig.Container, targetPort int32) bool { for _, container := range containers { if container == nil || container.ReadinessProbe == nil { continue } probe := container.ReadinessProbe if (probe.TCPSocket != nil && probe.TCPSocket.Port == targetPort) || probe.HTTPGet != nil && probe.HTTPGet.Port == targetPort { return true } } return false } func BaseClusterEnvVars() []kcore.EnvFromSource { envVars := []kcore.EnvFromSource{ { ConfigMapRef: &kcore.ConfigMapEnvSource{ LocalObjectReference: kcore.LocalObjectReference{ Name: "env-vars", }, }, }, } return envVars } func getKubexitEnvVars(containerName string, deathDeps []string, birthDeps []string) []kcore.EnvVar { envVars := []kcore.EnvVar{ { Name: "KUBEXIT_NAME", Value: containerName, }, { Name: "KUBEXIT_GRAVEYARD", Value: _kubexitGraveyardMountPath, }, } if deathDeps != nil { envVars = append(envVars, kcore.EnvVar{ Name: "KUBEXIT_DEATH_DEPS", Value: strings.Join(deathDeps, ","), }, kcore.EnvVar{ Name: "KUBEXIT_IGNORE_CODE_ON_DEATH_DEPS", Value: "true", }, ) } if birthDeps != nil { envVars = append(envVars, kcore.EnvVar{ Name: "KUBEXIT_BIRTH_DEPS", Value: strings.Join(birthDeps, ","), }, kcore.EnvVar{ Name: "KUBEXIT_IGNORE_CODE_ON_DEATH_DEPS", Value: "true", }, ) } return envVars } func MntVolume() kcore.Volume { return k8s.EmptyDirVolume(_emptyDirVolumeName) } func CortexVolume() kcore.Volume { return k8s.EmptyDirVolume(_cortexDirVolumeName) } func APIConfigVolume(name string) kcore.Volume { return kcore.Volume{ Name: name, VolumeSource: kcore.VolumeSource{ ConfigMap: &kcore.ConfigMapVolumeSource{ LocalObjectReference: kcore.LocalObjectReference{ Name: name, }, }, }, } } func ClientConfigVolume() kcore.Volume { return kcore.Volume{ Name: _clientConfigDirVolume, VolumeSource: kcore.VolumeSource{ ConfigMap: &kcore.ConfigMapVolumeSource{ LocalObjectReference: kcore.LocalObjectReference{ Name: _clientConfigConfigMap, }, }, }, } } func ClusterConfigVolume() kcore.Volume { return kcore.Volume{ Name: _clusterConfigDirVolume, VolumeSource: kcore.VolumeSource{ ConfigMap: &kcore.ConfigMapVolumeSource{ LocalObjectReference: kcore.LocalObjectReference{ Name: _clusterConfigConfigMap, }, }, }, } } func ShmVolume(q resource.Quantity, volumeName string) kcore.Volume { return kcore.Volume{ Name: volumeName, VolumeSource: kcore.VolumeSource{ EmptyDir: &kcore.EmptyDirVolumeSource{ Medium: kcore.StorageMediumMemory, SizeLimit: k8s.QuantityPtr(q), }, }, } } func KubexitVolume() kcore.Volume { return k8s.EmptyDirVolume(_kubexitGraveyardName) } func MntMount() kcore.VolumeMount { return k8s.EmptyDirVolumeMount(_emptyDirVolumeName, _emptyDirMountPath) } func CortexMount() kcore.VolumeMount { return k8s.EmptyDirVolumeMount(_cortexDirVolumeName, _cortexDirMountPath) } func APIConfigMount(name string) kcore.VolumeMount { return kcore.VolumeMount{ Name: name, MountPath: path.Join(_cortexDirMountPath, "spec"), } } func ClientConfigMount() kcore.VolumeMount { return kcore.VolumeMount{ Name: _clientConfigDirVolume, MountPath: path.Join(_clientConfigDir, "cli.yaml"), SubPath: "cli.yaml", } } func ClusterConfigMount() kcore.VolumeMount { return kcore.VolumeMount{ Name: _clusterConfigDirVolume, MountPath: path.Join(_clusterConfigDir, "cluster.yaml"), SubPath: "cluster.yaml", } } func ShmMount(volumeName string) kcore.VolumeMount { return k8s.EmptyDirVolumeMount(volumeName, _shmDirMountPath) } func KubexitMount() kcore.VolumeMount { return k8s.EmptyDirVolumeMount(_kubexitGraveyardName, _kubexitGraveyardMountPath) } ================================================ FILE: pkg/workloads/init.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package workloads import ( "github.com/cortexlabs/cortex/pkg/config" kcore "k8s.io/api/core/v1" ) const ( _kubexitInitContainerName = "kubexit" ) func KubexitInitContainer() kcore.Container { return kcore.Container{ Name: _kubexitInitContainerName, Image: config.ClusterConfig.ImageKubexit, ImagePullPolicy: kcore.PullAlways, Command: []string{"cp", "/bin/kubexit", "/cortex/kubexit"}, VolumeMounts: []kcore.VolumeMount{ CortexMount(), }, } } ================================================ FILE: pkg/workloads/k8s.go ================================================ /* Copyright 2022 Cortex Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package workloads import ( "fmt" "path" "sort" "strings" "github.com/cortexlabs/cortex/pkg/config" "github.com/cortexlabs/cortex/pkg/consts" "github.com/cortexlabs/cortex/pkg/lib/k8s" "github.com/cortexlabs/cortex/pkg/lib/pointer" s "github.com/cortexlabs/cortex/pkg/lib/strings" "github.com/cortexlabs/cortex/pkg/types/clusterconfig" "github.com/cortexlabs/cortex/pkg/types/spec" "github.com/cortexlabs/cortex/pkg/types/userconfig" kcore "k8s.io/api/core/v1" kresource "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/util/intstr" ) const ( ServiceAccountName = "default" ) const ( _cortexDirVolumeName = "cortex" _cortexDirMountPath = "/cortex" _clientConfigDir = "/cortex/client" _emptyDirVolumeName = "mnt" _emptyDirMountPath = "/mnt" ProxyContainerName = "proxy" DequeuerContainerName = "dequeuer" GatewayContainerName = "gateway" _kubexitGraveyardName = "graveyard" _kubexitGraveyardMountPath = "/graveyard" _shmDirMountPath = "/dev/shm" _clientConfigDirVolume = "client-config" _clientConfigConfigMap = "client-config" _clusterConfigDirVolume = "cluster-config" _clusterConfigConfigMap = "cluster-config" _clusterConfigDir = "/configs/cluster" ) var ( _statsdAddress = fmt.Sprintf("prometheus-statsd-exporter.%s:9125", consts.PrometheusNamespace) // each Inferentia chip requires 128 HugePages with each HugePage having a size of 2Mi _hugePagesMemPerInf = int64(128 * 2 * 1024 * 1024) // bytes ) func asyncDequeuerProxyContainer(api spec.API, queueURL string) (kcore.Container, kcore.Volume) { return kcore.Container{ Name: DequeuerContainerName, Image: config.ClusterConfig.ImageDequeuer, ImagePullPolicy: kcore.PullAlways, Command: []string{ "/dequeuer", }, Args: []string{ "--cluster-config", consts.DefaultInClusterConfigPath, "--cluster-uid", config.ClusterConfig.ClusterUID, "--probes-path", path.Join(_cortexDirMountPath, "spec", "probes.json"), "--queue", queueURL, "--api-kind", api.Kind.String(), "--api-name", api.Name, "--statsd-address", _statsdAddress, "--user-port", s.Int32(*api.Pod.Port), "--admin-port", consts.AdminPortStr, "--workers", s.Int64(api.Pod.MaxConcurrency), }, Env: BaseEnvVars, EnvFrom: BaseClusterEnvVars(), Ports: []kcore.ContainerPort{ { Name: consts.AdminPortName, ContainerPort: consts.AdminPortInt32, }, }, Resources: kcore.ResourceRequirements{ Requests: kcore.ResourceList{ kcore.ResourceCPU: consts.CortexDequeuerCPU, kcore.ResourceMemory: consts.CortexDequeuerMem, }, }, ReadinessProbe: &kcore.Probe{ Handler: kcore.Handler{ HTTPGet: &kcore.HTTPGetAction{ Path: "/healthz", Port: intstr.FromInt(int(consts.AdminPortInt32)), }, }, InitialDelaySeconds: 1, TimeoutSeconds: 1, PeriodSeconds: 10, SuccessThreshold: 1, FailureThreshold: 1, }, VolumeMounts: []kcore.VolumeMount{ ClusterConfigMount(), }, }, ClusterConfigVolume() } func batchDequeuerProxyContainer(api spec.API, jobID, queueURL string) (kcore.Container, kcore.Volume) { return kcore.Container{ Name: DequeuerContainerName, Image: config.ClusterConfig.ImageDequeuer, ImagePullPolicy: kcore.PullAlways, Command: []string{ "/dequeuer", }, Args: []string{ "--cluster-config", consts.DefaultInClusterConfigPath, "--cluster-uid", config.ClusterConfig.ClusterUID, "--probes-path", path.Join(_cortexDirMountPath, "spec", "probes.json"), "--queue", queueURL, "--api-kind", api.Kind.String(), "--api-name", api.Name, "--job-id", jobID, "--statsd-address", _statsdAddress, "--user-port", s.Int32(*api.Pod.Port), "--admin-port", consts.AdminPortStr, }, Env: BaseEnvVars, EnvFrom: BaseClusterEnvVars(), Resources: kcore.ResourceRequirements{ Requests: kcore.ResourceList{ kcore.ResourceCPU: consts.CortexDequeuerCPU, kcore.ResourceMemory: consts.CortexDequeuerMem, }, }, ReadinessProbe: &kcore.Probe{ Handler: kcore.Handler{ HTTPGet: &kcore.HTTPGetAction{ Path: "/healthz", Port: intstr.FromInt(int(consts.AdminPortInt32)), }, }, InitialDelaySeconds: 1, TimeoutSeconds: 1, PeriodSeconds: 10, SuccessThreshold: 1, FailureThreshold: 1, }, VolumeMounts: []kcore.VolumeMount{ ClusterConfigMount(), CortexMount(), }, }, ClusterConfigVolume() } func realtimeProxyContainer(api spec.API) (kcore.Container, kcore.Volume) { proxyHasTCPProbe := !HasReadinessProbesTargetingPort(api.Pod.Containers, *api.Pod.Port) return kcore.Container{ Name: ProxyContainerName, Image: config.ClusterConfig.ImageProxy, ImagePullPolicy: kcore.PullAlways, Args: []string{ "--cluster-config", consts.DefaultInClusterConfigPath, "--port", consts.ProxyPortStr, "--admin-port", consts.AdminPortStr, "--user-port", s.Int32(*api.Pod.Port), "--max-concurrency", s.Int32(int32(api.Pod.MaxConcurrency)), "--max-queue-length", s.Int32(int32(api.Pod.MaxQueueLength)), "--has-tcp-probe", s.Bool(proxyHasTCPProbe), }, Ports: []kcore.ContainerPort{ {Name: consts.AdminPortName, ContainerPort: consts.AdminPortInt32}, {ContainerPort: consts.ProxyPortInt32}, }, Env: BaseEnvVars, EnvFrom: BaseClusterEnvVars(), VolumeMounts: []kcore.VolumeMount{ ClusterConfigMount(), }, Resources: kcore.ResourceRequirements{ Requests: kcore.ResourceList{ kcore.ResourceCPU: consts.CortexProxyCPU, kcore.ResourceMemory: consts.CortexProxyMem, }, }, ReadinessProbe: &kcore.Probe{ Handler: kcore.Handler{ HTTPGet: &kcore.HTTPGetAction{ Path: "/healthz", Port: intstr.FromInt(int(consts.AdminPortInt32)), }, }, InitialDelaySeconds: 1, TimeoutSeconds: 3, PeriodSeconds: 10, SuccessThreshold: 1, FailureThreshold: 3, }, }, ClusterConfigVolume() } func RealtimeContainers(api spec.API) ([]kcore.Container, []kcore.Volume) { containers, volumes := userPodContainers(api) proxyContainer, proxyVolume := realtimeProxyContainer(api) containers = append(containers, proxyContainer) volumes = append(volumes, proxyVolume) return containers, volumes } func AsyncContainers(api spec.API, queueURL string) ([]kcore.Container, []kcore.Volume) { k8sName := K8sName(api.Name) containers, volumes := userPodContainers(api) dequeuerContainer, dequeuerVolume := asyncDequeuerProxyContainer(api, queueURL) dequeuerContainer.VolumeMounts = append(dequeuerContainer.VolumeMounts, APIConfigMount(k8sName)) containers = append(containers, dequeuerContainer) volumes = append(volumes, dequeuerVolume, APIConfigVolume(k8sName)) return containers, volumes } func TaskContainers(api spec.API, job *spec.JobKey) ([]kcore.Container, []kcore.Volume) { containers, volumes := userPodContainers(api) k8sName := job.K8sName() volumes = append(volumes, KubexitVolume(), APIConfigVolume(k8sName), ) containerNames := userconfig.GetContainerNames(api.Pod.Containers) for i, c := range containers { containers[i].VolumeMounts = append(containers[i].VolumeMounts, KubexitMount(), APIConfigMount(k8sName), ) containerDeathDependencies := containerNames.Copy() containerDeathDependencies.Remove(c.Name) containerDeathEnvVars := getKubexitEnvVars(c.Name, containerDeathDependencies.SliceSorted(), nil) containers[i].Env = append(containers[i].Env, containerDeathEnvVars...) if c.Command[0] != "/cortex/kubexit" { containers[i].Command = append([]string{"/cortex/kubexit"}, c.Command...) } } return containers, volumes } func BatchContainers(api spec.API, job *spec.BatchJob) ([]kcore.Container, []kcore.Volume) { userContainers, userVolumes := userPodContainers(api) dequeuerContainer, dequeuerVolume := batchDequeuerProxyContainer(api, job.ID, job.SQSUrl) // make sure the dequeuer starts first to allow it to start watching the graveyard before user containers begin containers := append([]kcore.Container{dequeuerContainer}, userContainers...) volumes := append([]kcore.Volume{dequeuerVolume}, userVolumes...) k8sName := job.K8sName() volumes = append(volumes, KubexitVolume(), APIConfigVolume(k8sName), ) containerNames := userconfig.GetContainerNames(api.Pod.Containers) containerNames.Add(dequeuerContainer.Name) for i, c := range containers { containers[i].VolumeMounts = append(containers[i].VolumeMounts, KubexitMount(), APIConfigMount(k8sName), ) containerDeathDependencies := containerNames.Copy() containerDeathDependencies.Remove(c.Name) containerDeathEnvVars := getKubexitEnvVars(c.Name, containerDeathDependencies.SliceSorted(), nil) containers[i].Env = append(containers[i].Env, containerDeathEnvVars...) if c.Command[0] != "/cortex/kubexit" { containers[i].Command = append([]string{"/cortex/kubexit"}, c.Command...) } } return containers, volumes } func userPodContainers(api spec.API) ([]kcore.Container, []kcore.Volume) { volumes := []kcore.Volume{ MntVolume(), CortexVolume(), ClientConfigVolume(), } containerMounts := []kcore.VolumeMount{ MntMount(), CortexMount(), ClientConfigMount(), } containers := make([]kcore.Container, len(api.Pod.Containers)) for i, container := range api.Pod.Containers { containerResourceList := kcore.ResourceList{} containerResourceLimitsList := kcore.ResourceList{} securityContext := kcore.SecurityContext{ Privileged: pointer.Bool(true), } var readinessProbe *kcore.Probe if api.Kind == userconfig.RealtimeAPIKind { readinessProbe = GetProbeSpec(container.ReadinessProbe) } if container.Compute.CPU != nil { containerResourceList[kcore.ResourceCPU] = *k8s.QuantityPtr(container.Compute.CPU.Quantity.DeepCopy()) } if container.Compute.Mem != nil { containerResourceList[kcore.ResourceMemory] = *k8s.QuantityPtr(container.Compute.Mem.Quantity.DeepCopy()) } if container.Compute.GPU > 0 { containerResourceList["nvidia.com/gpu"] = *kresource.NewQuantity(container.Compute.GPU, kresource.DecimalSI) containerResourceLimitsList["nvidia.com/gpu"] = *kresource.NewQuantity(container.Compute.GPU, kresource.DecimalSI) } if container.Compute.Inf > 0 { totalHugePages := container.Compute.Inf * _hugePagesMemPerInf containerResourceList["aws.amazon.com/neuron"] = *kresource.NewQuantity(container.Compute.Inf, kresource.DecimalSI) containerResourceList["hugepages-2Mi"] = *kresource.NewQuantity(totalHugePages, kresource.BinarySI) containerResourceLimitsList["aws.amazon.com/neuron"] = *kresource.NewQuantity(container.Compute.Inf, kresource.DecimalSI) containerResourceLimitsList["hugepages-2Mi"] = *kresource.NewQuantity(totalHugePages, kresource.BinarySI) securityContext.Capabilities = &kcore.Capabilities{ Add: []kcore.Capability{ "SYS_ADMIN", "IPC_LOCK", }, } } if container.Compute.Shm != nil { volumes = append(volumes, ShmVolume(container.Compute.Shm.Quantity, "dshm-"+container.Name)) containerMounts = append(containerMounts, ShmMount("dshm-"+container.Name)) } containerEnvVars := BaseEnvVars containerEnvVars = append(containerEnvVars, kcore.EnvVar{ Name: "CORTEX_CLI_CONFIG_DIR", Value: _clientConfigDir, }) if api.Kind != userconfig.TaskAPIKind { containerEnvVars = append(containerEnvVars, kcore.EnvVar{ Name: "CORTEX_PORT", Value: s.Int32(*api.Pod.Port), }) } envVarNames := make([]string, 0, len(container.Env)) for envVarName := range container.Env { envVarNames = append(envVarNames, envVarName) } // k8s deployments will replace pods if env vars are re-ordered sort.Strings(envVarNames) for _, envVarName := range envVarNames { containerEnvVars = append(containerEnvVars, kcore.EnvVar{ Name: envVarName, Value: container.Env[envVarName], }) } containers[i] = kcore.Container{ Name: container.Name, Image: container.Image, Command: container.Command, Args: container.Args, Env: containerEnvVars, VolumeMounts: containerMounts, LivenessProbe: GetProbeSpec(container.LivenessProbe), ReadinessProbe: readinessProbe, Lifecycle: GetLifecycleSpec(container.PreStop), Resources: kcore.ResourceRequirements{ Requests: containerResourceList, Limits: containerResourceLimitsList, }, ImagePullPolicy: kcore.PullAlways, SecurityContext: &securityContext, } } return containers, volumes } func NodeSelectors() map[string]string { return map[string]string{ "workload": "true", } } func GenerateResourceTolerations() []kcore.Toleration { tolerations := []kcore.Toleration{ { Key: "workload", Operator: kcore.TolerationOpEqual, Value: "true", Effect: kcore.TaintEffectNoSchedule, }, { Key: "nvidia.com/gpu", Operator: kcore.TolerationOpExists, Effect: kcore.TaintEffectNoSchedule, }, { Key: "aws.amazon.com/neuron", Operator: kcore.TolerationOpEqual, Value: "true", Effect: kcore.TaintEffectNoSchedule, }, } return tolerations } func GenerateNodeAffinities(apiNodeGroups []string) *kcore.Affinity { var nodeGroups []*clusterconfig.NodeGroup for _, clusterNodeGroup := range config.ClusterConfig.NodeGroups { for _, apiNodeGroupName := range apiNodeGroups { if clusterNodeGroup.Name == apiNodeGroupName { nodeGroups = append(nodeGroups, clusterNodeGroup) } } } if apiNodeGroups == nil { nodeGroups = config.ClusterConfig.NodeGroups } requiredNodeGroups := make([]string, len(nodeGroups)) preferredAffinities := make([]kcore.PreferredSchedulingTerm, len(nodeGroups)) for i, nodeGroup := range nodeGroups { var nodeGroupPrefix string if nodeGroup.Spot { nodeGroupPrefix = "cx-ws-" } else { nodeGroupPrefix = "cx-wd-" } preferredAffinities[i] = kcore.PreferredSchedulingTerm{ Weight: int32(nodeGroup.Priority), Preference: kcore.NodeSelectorTerm{ MatchExpressions: []kcore.NodeSelectorRequirement{ { Key: "alpha.eksctl.io/nodegroup-name", Operator: kcore.NodeSelectorOpIn, Values: []string{nodeGroupPrefix + nodeGroup.Name}, }, }, }, } requiredNodeGroups[i] = nodeGroupPrefix + nodeGroup.Name } var requiredNodeSelector *kcore.NodeSelector if apiNodeGroups != nil { requiredNodeSelector = &kcore.NodeSelector{ NodeSelectorTerms: []kcore.NodeSelectorTerm{ { MatchExpressions: []kcore.NodeSelectorRequirement{ { Key: "alpha.eksctl.io/nodegroup-name", Operator: kcore.NodeSelectorOpIn, Values: requiredNodeGroups, }, }, }, }, } } return &kcore.Affinity{ NodeAffinity: &kcore.NodeAffinity{ PreferredDuringSchedulingIgnoredDuringExecution: preferredAffinities, RequiredDuringSchedulingIgnoredDuringExecution: requiredNodeSelector, }, } } var BaseEnvVars = []kcore.EnvVar{ { Name: "CORTEX_VERSION", Value: consts.CortexVersion, }, { Name: "CORTEX_LOG_LEVEL", Value: strings.ToUpper(userconfig.InfoLogLevel.String()), }, } ================================================ FILE: python/client/README.md ================================================ Cost-effective serverless computing - [docs.cortexlabs.com](https://www.docs.cortexlabs.com) ================================================ FILE: python/client/cortex/__init__.py ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import json from typing import Optional, List from cortex.binary import run_cli from cortex.client import Client from cortex.exceptions import NotFound from cortex.telemetry import sentry_wrapper __version__ = "master" # CORTEX_VERSION @sentry_wrapper def client(env_name: Optional[str] = None) -> Client: """ Initialize a client based on the specified environment. If no environment is specified, it will attempt to use the default environment. Args: env_name: Name of the environment to use. Returns: Cortex client that can be used to deploy and manage APIs in the specified environment. """ environments = env_list() if env_name is None: if not environments.get("default_environment"): raise NotFound("no default environment configured") env_name = environments["default_environment"] found = False for environment in environments["environments"]: if environment["name"] == env_name: found = True break if not found: raise NotFound( f"can't find environment {env_name}, create one by calling `cortex.new_client()`" ) return Client(environment) @sentry_wrapper def new_client( env_name: str, operator_endpoint: str, ) -> Client: """ Create a new environment to connect to an existing cluster, and initialize a client to deploy and manage APIs on that cluster. Args: env_name: Name of the environment to create. operator_endpoint: The endpoint for the operator of your Cortex cluster. You can get this endpoint by running the CLI command `cortex cluster info`. Returns: Cortex client that can be used to deploy and manage APIs on a cluster. """ cli_args = [ "env", "configure", env_name, "--operator-endpoint", operator_endpoint, ] run_cli(cli_args, hide_output=True) return client(env_name) @sentry_wrapper def env_list() -> List: """ List all environments configured on this machine. """ output = run_cli(["env", "list", "--output", "json"], hide_output=True) return json.loads(output.strip()) @sentry_wrapper def env_delete(name: str): """ Delete an environment configured on this machine. Args: name: Name of the environment to delete. """ run_cli(["env", "delete", name], hide_output=True) ================================================ FILE: python/client/cortex/binary/__init__.py ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import subprocess import sys from typing import List from cortex.exceptions import CortexBinaryException def run(): """ Runs the CLI from terminal. """ try: env = os.environ.copy() if "AWS_VPC_K8S_CNI_LOG_FILE" not in env: env["AWS_VPC_K8S_CNI_LOG_FILE"] = "/dev/null" process = subprocess.run([get_cli_path()] + sys.argv[1:], cwd=os.getcwd(), env=env) except KeyboardInterrupt: sys.exit(130) # Ctrl + C sys.exit(process.returncode) def run_cli( args: List[str], hide_output: bool = False, ) -> str: """ Runs the Cortex binary with the specified arguments. Args: args: Arguments to use when invoking the Cortex CLI. hide_output: Flag to prevent streaming CLI output to stdout. Raises: CortexBinaryException: Cortex CLI command returned an error. Returns: The stdout from the Cortex CLI command. """ env = os.environ.copy() env["CORTEX_CLI_INVOKER"] = "python" if "AWS_VPC_K8S_CNI_LOG_FILE" not in env: env["AWS_VPC_K8S_CNI_LOG_FILE"] = "/dev/null" process = subprocess.Popen( [get_cli_path()] + args, stderr=subprocess.PIPE, stdout=subprocess.PIPE, encoding="utf8", env=env, ) output = "" result = "" for c in iter(lambda: process.stdout.read(1), ""): output += c if not hide_output: sys.stdout.write(c) sys.stdout.flush() process.wait() if process.returncode == 0: return output if result != "": raise CortexBinaryException(result + "\n" + process.stderr.read()) raise CortexBinaryException(process.stderr.read()) def get_cli_path() -> str: """ Get the location of the CLI. Default location is the directory containing the `cortex.binary` package. The location can be overridden by setting the `CORTEX_CLI_PATH` environment variable. Raises: Exception: Unable to find the CLI. Returns: str: The location of the CLI in the local filesystem. """ if os.environ.get("CORTEX_CLI_PATH") is not None: cli_path = os.environ["CORTEX_CLI_PATH"] if not os.path.exists(cli_path): raise Exception( f"unable to find cortex binary at {cli_path} as specified in `CORTEX_CLI_PATH` environment variable" ) return cli_path try: import importlib.resources as pkg_resources except ImportError: # Try backported to PY<37 `importlib_resources`. import importlib_resources as pkg_resources from cortex import binary try: with pkg_resources.path(binary, "cli") as p: cli_path = p except FileNotFoundError as e: raise Exception( "unable to find cortex binary, please reinstall the cortex client by running `pip uninstall cortex` and then `pip install cortex`" ) from e return str(cli_path) ================================================ FILE: python/client/cortex/client.py ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import json import os import shutil import time import yaml from typing import List, Dict, Any from cortex import util from cortex.binary import run_cli from cortex.telemetry import sentry_wrapper class Client: @sentry_wrapper def __init__(self, env_config: Dict): """ A client to deploy and manage APIs in the specified environment. This constructor is not meant to be invoked directly. Use `cortex.client()` and `cortex.new_client()` to initialize a new cortex client. Args: env_config: Environment config """ self.env = env_config self.env_name = env_config["name"] # CORTEX_VERSION_MINOR @sentry_wrapper def deploy( self, api_spec: Dict[str, Any], force: bool = True, wait: bool = False, ): """ Deploy or update an API. Args: api_spec: A dictionary defining a single Cortex API. See https://docs.cortexlabs.com/v/master/ for schema. force: Override any in-progress api updates. wait: Block until the API is ready. Returns: Deployment status, API specification, and endpoint for each API. """ temp_deploy_dir = util.cli_config_dir() / "deployments" / api_spec["name"] if temp_deploy_dir.exists(): shutil.rmtree(str(temp_deploy_dir)) temp_deploy_dir.mkdir(parents=True) cortex_yaml_path = os.path.join(temp_deploy_dir, "cortex.yaml") with util.open_temporarily(cortex_yaml_path, "w", delete_parent_if_empty=True) as f: yaml.dump([api_spec], f) # write a list return self.deploy_from_file(cortex_yaml_path, force=force, wait=wait) # CORTEX_VERSION_MINOR @sentry_wrapper def deploy_from_file( self, config_file: str, force: bool = False, wait: bool = False, ) -> Dict: """ Deploy or update APIs specified in a configuration file. Args: config_file: Local path to a yaml file defining Cortex API(s). See https://docs.cortexlabs.com/v/master/ for schema. force: Override any in-progress api updates. wait: Block until the API is ready. Returns: Deployment status, API specification, and endpoint for each API. """ args = [ "deploy", config_file, "--env", self.env_name, "-o", "json", "-y", ] if force: args.append("--force") output = run_cli(args, hide_output=True) deploy_results = json.loads(output.strip()) deploy_result = deploy_results[0] if not wait: return deploy_result api_name = deploy_result["api"]["spec"]["name"] if ( deploy_result["api"]["spec"]["kind"] != "RealtimeAPI" and deploy_result["api"]["spec"]["kind"] != "AsyncAPI" ): return deploy_result while True: time.sleep(5) api = self.get_api(api_name) if api["status"]["status_code"] != "status_updating": break return api @sentry_wrapper def get_api(self, api_name: str) -> Dict: """ Get information about an API. Args: api_name: Name of the API. Returns: Information about the API, including the API specification, endpoint, status, and metrics (if applicable). """ output = run_cli(["get", api_name, "--env", self.env_name, "-o", "json"], hide_output=True) apis = json.loads(output.strip()) return apis[0] @sentry_wrapper def list_apis(self) -> List: """ List all APIs in the environment. Returns: List of APIs, including information such as the API specification, endpoint, status, and metrics (if applicable). """ args = ["get", "-o", "json", "--env", self.env_name] output = run_cli(args, hide_output=True) return json.loads(output.strip()) @sentry_wrapper def get_job(self, api_name: str, job_id: str) -> Dict: """ Get information about a submitted job. Args: api_name: Name of the Batch/Task API. job_id: Job ID. Returns: Information about the job, including the job status, worker status, and job progress. """ args = ["get", api_name, job_id, "--env", self.env_name, "-o", "json"] output = run_cli(args, hide_output=True) return json.loads(output.strip()) @sentry_wrapper def refresh(self, api_name: str, force: bool = False): """ Restart all of the replicas for a Realtime API without downtime. Args: api_name: Name of the API to refresh. force: Override an already in-progress API update. """ args = ["refresh", api_name, "--env", self.env_name, "-o", "json"] if force: args.append("--force") run_cli(args, hide_output=True) @sentry_wrapper def delete(self, api_name: str, keep_cache: bool = False): """ Delete an API. Args: api_name: Name of the API to delete. keep_cache: Whether to retain the cached data for this API. """ args = [ "delete", api_name, "--env", self.env_name, "--force", "-o", "json", ] if keep_cache: args.append("--keep-cache") run_cli(args, hide_output=True) @sentry_wrapper def stop_job(self, api_name: str, job_id: str, keep_cache: bool = False): """ Stop a running job. Args: api_name: Name of the Batch/Task API. job_id: ID of the Job to stop. """ args = [ "delete", api_name, job_id, "--env", self.env_name, "-o", "json", ] run_cli(args) ================================================ FILE: python/client/cortex/consts.py ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. CORTEX_VERSION = "master" # CORTEX_VERSION CORTEX_TELEMETRY_SENTRY_DSN = "https://5cea3d2d67194d028f7191fcc6ebca14@sentry.io/1825326" CORTEX_TELEMETRY_SENTRY_ENVIRONMENT = "client" ================================================ FILE: python/client/cortex/exceptions.py ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. class CortexException(Exception): """ Base class for all Cortex's errors. Each custom exception should be derived from this class. """ pass class CortexBinaryException(CortexException): """ Raise when binary execution returns an unexpected non-zero return code. """ pass class NotFound(CortexException): """ Raise when the specified resource or name is not found. """ pass ================================================ FILE: python/client/cortex/telemetry.py ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os from typing import Dict from uuid import uuid4 import sentry_sdk from cortex import util from cortex.consts import ( CORTEX_VERSION, CORTEX_TELEMETRY_SENTRY_DSN, CORTEX_TELEMETRY_SENTRY_ENVIRONMENT, ) from cortex.exceptions import CortexBinaryException from sentry_sdk.integrations.dedupe import DedupeIntegration from sentry_sdk.integrations.modules import ModulesIntegration from sentry_sdk.integrations.stdlib import StdlibIntegration def _sentry_client( disabled: bool = False, ) -> sentry_sdk.Client: """ Initialize sentry. You can override the default values with the following env vars: 1. CORTEX_TELEMETRY_SENTRY_DSN 2. CORTEX_TELEMETRY_SENTRY_ENVIRONMENT 3. CORTEX_TELEMETRY_DISABLE """ dsn = CORTEX_TELEMETRY_SENTRY_DSN environment = CORTEX_TELEMETRY_SENTRY_ENVIRONMENT if disabled or os.getenv("CORTEX_TELEMETRY_DISABLE", "").lower() == "true": return if os.getenv("CORTEX_TELEMETRY_SENTRY_DSN", "") != "": dsn = os.environ["CORTEX_TELEMETRY_SENTRY_DSN"] if os.getenv("CORTEX_TELEMETRY_SENTRY_ENVIRONMENT", "") != "": environment = os.environ["CORTEX_TELEMETRY_SENTRY_ENVIRONMENT"] client = sentry_sdk.Client( dsn=dsn, environment=environment, release=CORTEX_VERSION, ignore_errors=[CortexBinaryException], # exclude CortexBinaryException exceptions in_app_include=["cortex"], # for better grouping of events in sentry attach_stacktrace=True, default_integrations=False, # disable all default integrations auto_enabling_integrations=False, integrations=[ DedupeIntegration(), # prevent duplication of events StdlibIntegration(), # adds breadcrumbs (aka more info) ModulesIntegration(), # adds info about installed modules ], # debug=True, ) return client def _create_default_scope(optional_tags: Dict = {}) -> sentry_sdk.Scope: """ Creates default scope. Adds user ID as tag to the reported event. Can add optional tags. """ scope = sentry_sdk.Scope() user_id = None client_id_file_path = util.cli_config_dir() / "client-id.txt" if not client_id_file_path.is_file(): client_id_file_path.parent.mkdir(parents=True, exist_ok=True) client_id_file_path.write_text(str(uuid4())) user_id = client_id_file_path.read_text() if user_id: scope.set_user({"id": user_id}) for k, v in optional_tags.items(): scope.set_tag(k, v) return scope # only one instance of this is required hub = sentry_sdk.Hub(_sentry_client(), _create_default_scope()) def sentry_wrapper(func): def wrapper(*args, **kwargs): with hub: try: return func(*args, **kwargs) except: sentry_sdk.capture_exception() sentry_sdk.flush() raise return wrapper ================================================ FILE: python/client/cortex/util.py ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import shutil from contextlib import contextmanager from pathlib import Path def cli_config_dir() -> Path: cli_config_dir = os.environ.get("CORTEX_CLI_CONFIG_DIR", "") if cli_config_dir == "": return Path.home() / ".cortex" return Path(cli_config_dir).expanduser().resolve() @contextmanager def open_temporarily(path, mode, delete_parent_if_empty: bool = False): parentDir = Path(path).parent parentDir.mkdir(parents=True, exist_ok=True) file = open(path, mode) try: yield file finally: file.close() os.remove(path) if delete_parent_if_empty and len(os.listdir(str(parentDir))) == 0: shutil.rmtree(str(parentDir)) @contextmanager def open_tempdir(dir_path): Path(dir_path).mkdir(parents=True) try: yield dir_path finally: shutil.rmtree(dir_path) ================================================ FILE: python/client/setup.py ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import pathlib from setuptools import setup, find_packages from setuptools.command.install import install class InstallBinary(install): def run(self): install.run(self) import requests from pathlib import Path import sys import os import stat import shutil dest_dir = os.path.join(self.install_lib, "cortex", "binary") zip_file_path = os.path.join(dest_dir, "cli.zip") cli_file_path = os.path.join(dest_dir, "cli") if not os.path.exists(cli_file_path): platform = sys.platform # recommended way to check platform: https://docs.python.org/3/library/sys.html#sys.platform if sys.platform.startswith("darwin"): platform = "darwin" if sys.platform.startswith("linux"): platform = "linux" if platform != "darwin" and platform != "linux": raise Exception( f"platform {platform} is not supported; cortex is only supported on mac and linux" ) cortex_version = self.config_vars["dist_version"] if "dev" in cortex_version: cortex_version = "master" download_url = f"https://s3-us-west-2.amazonaws.com/get-cortex/{cortex_version}/cli/{platform}/cortex.zip" print("downloading cortex cli...") with requests.get(download_url, stream=True) as r: with open(zip_file_path, "wb") as f: shutil.copyfileobj(r.raw, f) zip_dir = os.path.join(dest_dir, "cli_dir") print("unzipping cortex cli...") shutil.unpack_archive(zip_file_path, zip_dir) shutil.move(os.path.join(zip_dir, "cortex"), cli_file_path) shutil.rmtree(zip_dir) os.remove(zip_file_path) f = Path(cli_file_path) f.chmod(f.stat().st_mode | stat.S_IEXEC) long_description = "" if pathlib.Path("README.md").is_file(): with open("README.md") as f: long_description = f.read() setup( name="cortex", version="master", # CORTEX_VERSION description="Cost-effective serverless computing", author="cortexlabs.com", author_email="dev@cortexlabs.com", license="Apache License 2.0", long_description_content_type="text/markdown", long_description=long_description, url="https://www.cortexlabs.com", setup_requires=(["setuptools", "requests", "wheel"]), packages=find_packages(), package_data={"cortex.binary": ["cli"]}, entry_points={ "console_scripts": [ "cortex = cortex.binary:run", ], }, install_requires=( [ "importlib-resources; python_version < '3.7'", "pyyaml>=5.4.1", "sentry-sdk>=1.4.3", ] ), python_requires=">=3.6", cmdclass={ "install": InstallBinary, }, classifiers=[ "Operating System :: MacOS", "Operating System :: POSIX :: Linux", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Intended Audience :: Developers", ], project_urls={ "Bug Reports": "https://github.com/cortexlabs/cortex/issues", "Community": "https://gitter.im/cortexlabs/cortex", "Docs": "https://docs.cortexlabs.com", "Source Code": "https://github.com/cortexlabs/cortex", }, ) ================================================ FILE: test/README.md ================================================ # Cortex Tests - [Example APIs](apis) - [End-to-end Tests](e2e) - [Testing Utilities](utils) ================================================ FILE: test/apis/async/hello-world/app/main.py ================================================ import os from fastapi import FastAPI from fastapi.responses import PlainTextResponse app = FastAPI() app.response_str = os.getenv("RESPONSE", "hello world") @app.get("/healthz") def healthz(): return PlainTextResponse("ok") @app.post("/") def handler(): return {"message": app.response_str} ================================================ FILE: test/apis/async/hello-world/app/requirements.txt ================================================ uvicorn[standard] fastapi ================================================ FILE: test/apis/async/hello-world/build-cpu.sh ================================================ #!/usr/bin/env bash # usage: build-cpu.sh [REGISTRY] [--skip-push] # REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test image_name="async-hello-world-cpu" "$(dirname "${BASH_SOURCE[0]}")"/../../../utils/build.sh $(realpath "${BASH_SOURCE[0]}") "$image_name" "$@" ================================================ FILE: test/apis/async/hello-world/cortex_cpu.yaml ================================================ - name: hello-world kind: AsyncAPI pod: port: 8080 containers: - name: api image: quay.io/cortexlabs-test/async-hello-world-cpu:latest readiness_probe: http_get: path: "/healthz" port: 8080 compute: cpu: 200m mem: 128Mi ================================================ FILE: test/apis/async/hello-world/cpu.Dockerfile ================================================ FROM python:3.8-slim ENV PYTHONUNBUFFERED TRUE COPY app/requirements.txt /app/requirements.txt RUN pip install --no-cache-dir -r /app/requirements.txt COPY app /app WORKDIR /app/ ENV PYTHONPATH=/app ENV CORTEX_PORT=8080 CMD uvicorn --workers 1 --host 0.0.0.0 --port $CORTEX_PORT main:app ================================================ FILE: test/apis/async/text-generator/app/main.py ================================================ import os from fastapi import FastAPI, status from fastapi.responses import PlainTextResponse from pydantic import BaseModel from transformers import GPT2Tokenizer, GPT2LMHeadModel app = FastAPI() app.device = os.getenv("TARGET_DEVICE", "cpu") app.ready = False @app.on_event("startup") def startup(): app.tokenizer = GPT2Tokenizer.from_pretrained("gpt2") app.model = GPT2LMHeadModel.from_pretrained("gpt2").to(app.device) app.ready = True @app.get("/healthz") def healthz(): if app.ready: return PlainTextResponse("ok") return PlainTextResponse("service unavailable", status_code=status.HTTP_503_SERVICE_UNAVAILABLE) class Body(BaseModel): text: str @app.post("/") def text_generator(body: Body): input_length = len(body.text.split()) tokens = app.tokenizer.encode(body.text, return_tensors="pt").to(app.device) prediction = app.model.generate(tokens, max_length=input_length + 20, do_sample=True) return {"text": app.tokenizer.decode(prediction[0])} ================================================ FILE: test/apis/async/text-generator/app/requirements-cpu.txt ================================================ uvicorn[standard] fastapi transformers==3.0.* -f https://download.pytorch.org/whl/torch_stable.html torch==1.7.1+cpu ================================================ FILE: test/apis/async/text-generator/app/requirements-gpu.txt ================================================ uvicorn[standard]==0.16.0 sentencepiece==0.1.94 fastapi transformers==3.0.* torch==1.10.2 ================================================ FILE: test/apis/async/text-generator/build-cpu.sh ================================================ #!/usr/bin/env bash # usage: build-cpu.sh [REGISTRY] [--skip-push] # REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test image_name="async-text-generator-cpu" "$(dirname "${BASH_SOURCE[0]}")"/../../../utils/build.sh $(realpath "${BASH_SOURCE[0]}") "$image_name" "$@" ================================================ FILE: test/apis/async/text-generator/build-gpu.sh ================================================ #!/usr/bin/env bash # usage: build-gpu.sh [REGISTRY] [--skip-push] # REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test image_name="async-text-generator-gpu" "$(dirname "${BASH_SOURCE[0]}")"/../../../utils/build.sh $(realpath "${BASH_SOURCE[0]}") "$image_name" "$@" ================================================ FILE: test/apis/async/text-generator/cortex_cpu.yaml ================================================ - name: text-generator kind: AsyncAPI pod: port: 8080 containers: - name: api image: quay.io/cortexlabs-test/async-text-generator-cpu:latest readiness_probe: http_get: path: "/healthz" port: 8080 compute: cpu: 1 mem: 2.5Gi ================================================ FILE: test/apis/async/text-generator/cortex_gpu.yaml ================================================ - name: text-generator kind: AsyncAPI pod: port: 8080 containers: - name: api image: quay.io/cortexlabs-test/async-text-generator-gpu:latest env: TARGET_DEVICE: "cuda" readiness_probe: http_get: path: "/healthz" port: 8080 compute: cpu: 1 gpu: 1 mem: 512Mi ================================================ FILE: test/apis/async/text-generator/cpu.Dockerfile ================================================ FROM python:3.8-slim ENV PYTHONUNBUFFERED TRUE COPY app/requirements-cpu.txt /app/requirements.txt RUN pip install --no-cache-dir -r /app/requirements.txt COPY app /app WORKDIR /app/ ENV PYTHONPATH=/app ENV CORTEX_PORT=8080 CMD uvicorn --workers 1 --host 0.0.0.0 --port $CORTEX_PORT main:app ================================================ FILE: test/apis/async/text-generator/expectations.yaml ================================================ response: content_type: "json" json_schema: type: "object" properties: text: type: "string" required: - "text" ================================================ FILE: test/apis/async/text-generator/gpu.Dockerfile ================================================ FROM nvidia/cuda:11.3.1-cudnn8-runtime-ubuntu18.04 RUN apt-get update \ && apt-get install -y \ python3 \ python3-pip \ pkg-config \ build-essential \ git \ cmake \ && apt-get clean -qq && rm -rf /var/lib/apt/lists/* ENV LC_ALL=C.UTF-8 LANG=C.UTF-8 ENV PYTHONUNBUFFERED TRUE COPY app/requirements-gpu.txt /app/requirements.txt RUN pip3 install \ --no-cache-dir \ --extra-index-url https://download.pytorch.org/whl/cu113 \ -r /app/requirements.txt COPY app /app WORKDIR /app/ ENV PYTHONPATH=/app ENV CORTEX_PORT=8080 CMD uvicorn --workers 1 --host 0.0.0.0 --port $CORTEX_PORT main:app ================================================ FILE: test/apis/async/text-generator/sample.json ================================================ { "text": "machine learning is" } ================================================ FILE: test/apis/batch/image-classifier-alexnet/app/main.py ================================================ import os, json, re import requests import torch import torchvision import boto3 import uuid from typing import List from PIL import Image from io import BytesIO from torchvision import transforms from fastapi import FastAPI, Request, status from fastapi.responses import PlainTextResponse app = FastAPI() app.device = os.getenv("TARGET_DEVICE", "cpu") app.ready = False s3 = boto3.client("s3") @app.on_event("startup") def startup(): # read job spec with open("/cortex/spec/job.json", "r") as f: job_spec = json.load(f) print(json.dumps(job_spec, indent=2)) # get metadata config = job_spec["config"] app.job_id = job_spec["job_id"] if len(config.get("dest_s3_dir", "")) == 0: raise Exception("'dest_s3_dir' field was not provided in job submission") # s3 info app.bucket, app.key = re.match("s3://(.+?)/(.+)", config["dest_s3_dir"]).groups() app.key = os.path.join(app.key, app.job_id) # loading model normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) app.preprocess = transforms.Compose( [transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor(), normalize] ) app.labels = requests.get( "https://storage.googleapis.com/download.tensorflow.org/data/ImageNetLabels.txt" ).text.split("\n")[1:] app.model = torchvision.models.alexnet(pretrained=True).eval().to(app.device) app.ready = True @app.get("/healthz") def healthz(): if app.ready: return PlainTextResponse("ok") return PlainTextResponse("service unavailable", status_code=status.HTTP_503_SERVICE_UNAVAILABLE) @app.post("/") def handle_batch(image_urls: List[str]): tensor_list = [] # download and preprocess each image for image_url in image_urls: if image_url.startswith("s3://"): bucket, image_key = re.match("s3://(.+?)/(.+)", image_url).groups() image_bytes = s3.get_object(Bucket=bucket, Key=image_key)["Body"].read() elif image_url.startswith("http"): image_bytes = requests.get(image_url).content else: raise RuntimeError(f"{image_url}: invalid image url") img_pil = Image.open(BytesIO(image_bytes)) tensor_list.append(app.preprocess(img_pil)) # classify the batch of images img_tensor = torch.stack(tensor_list).to(app.device) with torch.no_grad(): prediction = app.model(img_tensor) _, indices = prediction.max(1) # extract predicted classes results = [ {"url": image_urls[i], "class": app.labels[class_idx]} for i, class_idx in enumerate(indices) ] json_output = json.dumps(results) # save results prediction_id = uuid.uuid4() s3.put_object(Bucket=app.bucket, Key=f"{app.key}/{prediction_id}.json", Body=json_output) @app.post("/on-job-complete") def on_job_complete(): all_results = [] # aggregate all classifications paginator = s3.get_paginator("list_objects_v2") for page in paginator.paginate(Bucket=app.bucket, Prefix=app.key): if "Contents" not in page: continue for obj in page["Contents"]: body = s3.get_object(Bucket=app.bucket, Key=obj["Key"])["Body"] all_results += json.loads(body.read().decode("utf8")) # save single file containing aggregated classifications s3.put_object( Bucket=app.bucket, Key=os.path.join(app.key, "aggregated_results.json"), Body=json.dumps(all_results), ) ================================================ FILE: test/apis/batch/image-classifier-alexnet/app/requirements-cpu.txt ================================================ uvicorn[standard] fastapi requests boto3==1.17.* -f https://download.pytorch.org/whl/torch_stable.html torch==1.7.1+cpu torchvision==0.8.2+cpu ================================================ FILE: test/apis/batch/image-classifier-alexnet/app/requirements-gpu.txt ================================================ uvicorn[standard]==0.16.0 sentencepiece==0.1.94 fastapi requests boto3==1.17.* torch==1.10.2 torchvision==0.8.* ================================================ FILE: test/apis/batch/image-classifier-alexnet/build-cpu.sh ================================================ #!/usr/bin/env bash # usage: build-cpu.sh [REGISTRY] [--skip-push] # REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test image_name="batch-image-classifier-alexnet-cpu" "$(dirname "${BASH_SOURCE[0]}")"/../../../utils/build.sh $(realpath "${BASH_SOURCE[0]}") "$image_name" "$@" ================================================ FILE: test/apis/batch/image-classifier-alexnet/build-gpu.sh ================================================ #!/usr/bin/env bash # usage: build-gpu.sh [REGISTRY] [--skip-push] # REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test image_name="batch-image-classifier-alexnet-gpu" "$(dirname "${BASH_SOURCE[0]}")"/../../../utils/build.sh $(realpath "${BASH_SOURCE[0]}") "$image_name" "$@" ================================================ FILE: test/apis/batch/image-classifier-alexnet/cortex_cpu.yaml ================================================ - name: image-classifier-alexnet kind: BatchAPI pod: containers: - name: api image: quay.io/cortexlabs-test/batch-image-classifier-alexnet-cpu:latest command: ["uvicorn", "--workers", "1", "--host", "0.0.0.0", "--port", "$(CORTEX_PORT)", "main:app"] readiness_probe: http_get: path: "/healthz" port: 8080 compute: cpu: 1 mem: 2Gi ================================================ FILE: test/apis/batch/image-classifier-alexnet/cortex_gpu.yaml ================================================ - name: image-classifier-alexnet kind: BatchAPI pod: containers: - name: api image: quay.io/cortexlabs-test/batch-image-classifier-alexnet-gpu:latest command: ["uvicorn", "--workers", "1", "--host", "0.0.0.0", "--port", "$(CORTEX_PORT)", "main:app"] env: TARGET_DEVICE: "cuda" readiness_probe: http_get: path: "/healthz" port: 8080 compute: cpu: 200m gpu: 1 mem: 512Mi ================================================ FILE: test/apis/batch/image-classifier-alexnet/cpu.Dockerfile ================================================ FROM python:3.8-slim ENV PYTHONUNBUFFERED TRUE COPY app/requirements-cpu.txt /app/requirements.txt RUN pip install --no-cache-dir -r /app/requirements.txt COPY app /app WORKDIR /app/ ENV PYTHONPATH=/app ENV CORTEX_PORT=8080 CMD uvicorn --workers 1 --host 0.0.0.0 --port $CORTEX_PORT main:app ================================================ FILE: test/apis/batch/image-classifier-alexnet/gpu.Dockerfile ================================================ FROM nvidia/cuda:11.3.1-cudnn8-runtime-ubuntu18.04 RUN apt-get update \ && apt-get install -y \ python3 \ python3-pip \ pkg-config \ build-essential \ git \ cmake \ libjpeg8-dev \ zlib1g-dev \ && apt-get clean -qq && rm -rf /var/lib/apt/lists/* ENV LC_ALL=C.UTF-8 LANG=C.UTF-8 ENV PYTHONUNBUFFERED TRUE COPY app/requirements-gpu.txt /app/requirements.txt RUN pip3 install \ --no-cache-dir \ --extra-index-url https://download.pytorch.org/whl/cu113 \ -r /app/requirements.txt COPY app /app WORKDIR /app/ ENV PYTHONPATH=/app ENV CORTEX_PORT=8080 CMD uvicorn --workers 1 --host 0.0.0.0 --port $CORTEX_PORT main:app ================================================ FILE: test/apis/batch/image-classifier-alexnet/sample.json ================================================ [ "https://i.imgur.com/PzXprwl.jpg", "https://i.imgur.com/a5djbnv.jpeg", "https://i.imgur.com/jDimNTZ.jpg", "https://i.imgur.com/WqeovVj.jpg" ] ================================================ FILE: test/apis/batch/image-classifier-alexnet/submit.py ================================================ """ Typical usage example: python submit.py """ from typing import List import sys import json import requests import cortex def main(): # parse args if len(sys.argv) != 3: print("usage: python submit.py ") sys.exit(1) env_name = sys.argv[1] dest_s3_dir = sys.argv[2] # read sample file with open("sample.json") as f: sample_items: List[str] = json.load(f) # get batch endpoint cx = cortex.client(env_name) batch_endpoint = cx.get_api("image-classifier-alexnet")["endpoint"] # submit job job_spec = { "workers": 1, "item_list": {"items": sample_items, "batch_size": 1}, "config": {"dest_s3_dir": dest_s3_dir}, } response = requests.post(batch_endpoint, json=job_spec) print(json.dumps(response.json(), indent=2)) if __name__ == "__main__": main() ================================================ FILE: test/apis/batch/sum/app/main.py ================================================ import os import boto3 import json import re from typing import List from fastapi import FastAPI, status from fastapi.responses import PlainTextResponse app = FastAPI() app.ready = False app.numbers_list = [] s3 = boto3.client("s3") @app.on_event("startup") def startup(): # read job spec with open("/cortex/spec/job.json", "r") as f: job_spec = json.load(f) print(json.dumps(job_spec, indent=2)) # get metadata config = job_spec["config"] job_id = job_spec["job_id"] if len(config.get("dest_s3_dir", "")) == 0: raise Exception("'dest_s3_dir' field was not provided in job submission") # s3 info app.bucket, app.key = re.match("s3://(.+?)/(.+)", config["dest_s3_dir"]).groups() app.key = os.path.join(app.key, job_id) app.ready = True @app.get("/healthz") def healthz(): if app.ready: return PlainTextResponse("ok") return PlainTextResponse("service unavailable", status_code=status.HTTP_503_SERVICE_UNAVAILABLE) @app.post("/") def handle_batch(batches: List[List[int]]): for numbers_list in batches: app.numbers_list.append(sum(numbers_list)) @app.post("/on-job-complete") def on_job_complete(): # this is only intended to work if 1 worker is used (since on-job-complete runs once across all workers) json_output = json.dumps(app.numbers_list) s3.put_object(Bucket=app.bucket, Key=f"{app.key}.json", Body=json_output) ================================================ FILE: test/apis/batch/sum/app/requirements.txt ================================================ uvicorn[standard] fastapi boto3==1.17.* ================================================ FILE: test/apis/batch/sum/build-cpu.sh ================================================ #!/usr/bin/env bash # usage: build-cpu.sh [REGISTRY] [--skip-push] # REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test image_name="batch-sum-cpu" "$(dirname "${BASH_SOURCE[0]}")"/../../../utils/build.sh $(realpath "${BASH_SOURCE[0]}") "$image_name" "$@" ================================================ FILE: test/apis/batch/sum/cortex_cpu.yaml ================================================ # this API is only meant to run with 1-worker jobs - name: sum kind: BatchAPI pod: containers: - name: api image: quay.io/cortexlabs-test/batch-sum-cpu:latest command: ["uvicorn", "--workers", "1", "--host", "0.0.0.0", "--port", "$(CORTEX_PORT)", "main:app"] readiness_probe: http_get: path: "/healthz" port: 8080 compute: cpu: 200m mem: 256Mi ================================================ FILE: test/apis/batch/sum/cpu.Dockerfile ================================================ FROM python:3.8-slim ENV PYTHONUNBUFFERED TRUE COPY app/requirements.txt /app/requirements.txt RUN pip install --no-cache-dir -r /app/requirements.txt COPY app /app WORKDIR /app/ ENV PYTHONPATH=/app ENV CORTEX_PORT=8080 CMD uvicorn --workers 1 --host 0.0.0.0 --port $CORTEX_PORT main:app ================================================ FILE: test/apis/batch/sum/sample.json ================================================ [ [1, 2, 67, -2, 43], [9, 0, 1, 0, 0, -1] ] ================================================ FILE: test/apis/batch/sum/sample_generator.py ================================================ from typing import List from random import sample RANGE = 10 ** 12 LENGTH = 5 def generate_sample() -> List[int]: return sample(range(RANGE), LENGTH) ================================================ FILE: test/apis/batch/sum/submit.py ================================================ """ Typical usage example: python submit.py """ from typing import List import sys import json import requests import cortex def main(): # parse args if len(sys.argv) != 3: print("usage: python submit.py ") sys.exit(1) env_name = sys.argv[1] dest_s3_dir = sys.argv[2] # read sample file with open("sample.json") as f: sample_items: List[str] = json.load(f) # get batch endpoint cx = cortex.client(env_name) batch_endpoint = cx.get_api("sum")["endpoint"] # submit job job_spec = { "workers": 1, "item_list": {"items": sample_items, "batch_size": 1}, "config": {"dest_s3_dir": dest_s3_dir}, } response = requests.post(batch_endpoint, json=job_spec) print(json.dumps(response.json(), indent=2)) if __name__ == "__main__": main() ================================================ FILE: test/apis/realtime/hello-world/app/main.py ================================================ import os from fastapi import FastAPI from fastapi.responses import PlainTextResponse app = FastAPI() app.response_str = os.getenv("RESPONSE", "hello world") @app.get("/healthz") def healthz(): return PlainTextResponse("ok") @app.post("/") def post_handler(): return PlainTextResponse(app.response_str) ================================================ FILE: test/apis/realtime/hello-world/app/requirements.txt ================================================ uvicorn[standard] fastapi ================================================ FILE: test/apis/realtime/hello-world/build-cpu.sh ================================================ #!/usr/bin/env bash # usage: build-cpu.sh [REGISTRY] [--skip-push] # REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test image_name="realtime-hello-world-cpu" "$(dirname "${BASH_SOURCE[0]}")"/../../../utils/build.sh $(realpath "${BASH_SOURCE[0]}") "$image_name" "$@" ================================================ FILE: test/apis/realtime/hello-world/cortex_cpu.yaml ================================================ - name: hello-world kind: RealtimeAPI pod: port: 8080 max_concurrency: 1 containers: - name: api image: quay.io/cortexlabs-test/realtime-hello-world-cpu:latest readiness_probe: http_get: path: "/healthz" port: 8080 compute: cpu: 200m mem: 128Mi ================================================ FILE: test/apis/realtime/hello-world/cortex_cpu_arm64.yaml ================================================ - name: hello-world kind: RealtimeAPI pod: port: 8080 max_concurrency: 1 containers: - name: api image: infrastructureascode/hello-world readiness_probe: http_get: path: "/health" port: 8080 compute: cpu: 200m mem: 128Mi ================================================ FILE: test/apis/realtime/hello-world/cortex_scale_to_zero.yaml ================================================ - name: hello-world kind: RealtimeAPI pod: port: 8080 max_concurrency: 1 max_queue_length: 999 containers: - name: api image: quay.io/cortexlabs-test/realtime-hello-world-cpu:latest readiness_probe: http_get: path: "/healthz" port: 8080 timeout_seconds: 3 compute: cpu: 200m mem: 128Mi autoscaling: min_replicas: 0 max_replicas: 1 downscale_stabilization_period: 30s ================================================ FILE: test/apis/realtime/hello-world/cpu.Dockerfile ================================================ FROM python:3.8-slim ENV PYTHONUNBUFFERED TRUE COPY app/requirements.txt /app/requirements.txt RUN pip install --no-cache-dir -r /app/requirements.txt COPY app /app WORKDIR /app/ ENV PYTHONPATH=/app ENV CORTEX_PORT=8080 CMD uvicorn --workers 1 --host 0.0.0.0 --port $CORTEX_PORT main:app ================================================ FILE: test/apis/realtime/hello-world/sample.json ================================================ {} ================================================ FILE: test/apis/realtime/image-classifier-resnet50/README.md ================================================ # Running an example ## CPU/GPU Get the image classifier endpoint: ```bash cortex get image-classifier-resnet50 ``` ```bash python client.py http://.elb..amazonaws.com/image-classifier-resnet50 ``` Or alternatively: ```bash curl "http://.elb..amazonaws.com/image-classifier-resnet50/v1/models/resnet50:predict" -X POST -H "Content-type: application/json" -d @sample.json ``` ## Inferentia ### HTTP Get the image classifier endpoint: ```bash cortex get image-classifier-resnet50 ``` ```bash python client_inf.py http://.elb..amazonaws.com/image-classifier-resnet50 ``` ### gRPC The Inferentia examples were inspired by [this tutorial](https://awsdocs-neuron.readthedocs-hosted.com/en/latest/neuron-deploy/tutorials/k8s_rn50_demo.html). This guide shows how to exec into a pod, check the Neuron runtime, and make inferences using gRPC for testing purposes (exposing gRPC endpoints outside of the cluster is not currently supported by Cortex, but is on the roadmap). #### Making an inference Exec into the TensorFlow Serving pod: ```bash kubectl exec api-image-classifier-resnet50-5b6df59b9b-p4s2h -c api -it -- /bin/bash ``` Create this `tensorflow-model-server-infer.py` with this contents (e.g. using `vim tensorflow-model-server-infer.py`): ```python import numpy as np import grpc import tensorflow as tf from tensorflow.keras.preprocessing import image from tensorflow.keras.applications.resnet50 import preprocess_input from tensorflow_serving.apis import predict_pb2 from tensorflow_serving.apis import prediction_service_pb2_grpc from tensorflow.keras.applications.resnet50 import decode_predictions if __name__ == '__main__': channel = grpc.insecure_channel('localhost:8500') stub = prediction_service_pb2_grpc.PredictionServiceStub(channel) img_file = tf.keras.utils.get_file( "./kitten_small.jpg", "https://raw.githubusercontent.com/awslabs/mxnet-model-server/master/docs/images/kitten_small.jpg") img = image.load_img(img_file, target_size=(224, 224)) img_array = preprocess_input(image.img_to_array(img)[None, ...]) request = predict_pb2.PredictRequest() request.model_spec.name = 'resnet50_neuron' request.inputs['input'].CopyFrom( tf.make_tensor_proto(img_array, shape=img_array.shape)) result = stub.Predict(request) prediction = tf.make_ndarray(result.outputs['output']) print(decode_predictions(prediction)) ``` Run an inference: ```bash python tensorflow-model-server-infer.py ``` #### Inspecting the neuron runtime Exec into the TensorFlow Serving pod (the `rtd` pod will also work for the example which uses the neuron-rtd sidecar): ```bash kubectl exec api-image-classifier-resnet50-5b6df59b9b-p4s2h -c api -it -- /bin/bash ``` Install dependencies: ```bash apt-get update && apt-get install -y aws-neuron-dkms aws-neuron-runtime-base aws-neuron-runtime aws-neuron-tools PATH="/opt/aws/neuron/bin:${PATH}" ``` Run CLI commands (described [here](https://awsdocs-neuron.readthedocs-hosted.com/en/latest/neuron-guide/neuron-tools/basic.html)): ```bash neuron-ls neuron-cli list-model neuron-top ``` ================================================ FILE: test/apis/realtime/image-classifier-resnet50/build-cpu.sh ================================================ #!/usr/bin/env bash # usage: build-cpu.sh [REGISTRY] [--skip-push] # REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test image_name="realtime-image-classifier-resnet50-cpu" "$(dirname "${BASH_SOURCE[0]}")"/../../../utils/build.sh $(realpath "${BASH_SOURCE[0]}") "$image_name" "$@" ================================================ FILE: test/apis/realtime/image-classifier-resnet50/build-gpu.sh ================================================ #!/usr/bin/env bash # usage: build-gpu.sh [REGISTRY] [--skip-push] # REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test image_name="realtime-image-classifier-resnet50-gpu" "$(dirname "${BASH_SOURCE[0]}")"/../../../utils/build.sh $(realpath "${BASH_SOURCE[0]}") "$image_name" "$@" ================================================ FILE: test/apis/realtime/image-classifier-resnet50/build-neuron-rtd.sh ================================================ #!/usr/bin/env bash # usage: build-neuron-rtd.sh [REGISTRY] [--skip-push] # REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test image_name="neuron-rtd" aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 790709498068.dkr.ecr.us-east-1.amazonaws.com "$(dirname "${BASH_SOURCE[0]}")"/../../../utils/build.sh $(realpath "${BASH_SOURCE[0]}") "$image_name" "$@" ================================================ FILE: test/apis/realtime/image-classifier-resnet50/build-neuron-tf-serving.sh ================================================ #!/usr/bin/env bash # usage: build-neuron-tf-serving.sh [REGISTRY] [--skip-push] # REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test image_name="neuron-tf-serving" aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 763104351884.dkr.ecr.us-east-1.amazonaws.com "$(dirname "${BASH_SOURCE[0]}")"/../../../utils/build.sh $(realpath "${BASH_SOURCE[0]}") "$image_name" "$@" ================================================ FILE: test/apis/realtime/image-classifier-resnet50/client.py ================================================ """A client that performs inferences on a ResNet model using the REST API. The client downloads a test image of a cat, queries the server over the REST API with the test image repeatedly and measures how long it takes to respond. The client expects a TensorFlow Serving ModelServer running a ResNet SavedModel from: https://github.com/tensorflow/models/tree/master/official/resnet#pre-trained-model The SavedModel must be one that can take JPEG images as inputs. Typical usage example: python client.py """ import sys import base64 import requests # the image URL is the location of the image we should send to the server IMAGE_URL = "https://tensorflow.org/images/blogs/serving/cat.jpg" def main(): # parse arg if len(sys.argv) != 2: print("usage: python client.py ") sys.exit(1) address = sys.argv[1] server_url = f"{address}/v1/models/resnet50:predict" # download labels labels = requests.get( "https://storage.googleapis.com/download.tensorflow.org/data/ImageNetLabels.txt" ).text.split("\n")[1:] # download the image dl_request = requests.get(IMAGE_URL, stream=True) dl_request.raise_for_status() # compose a JSON Predict request (send JPEG image in base64). jpeg_bytes = base64.b64encode(dl_request.content).decode("utf-8") predict_request = '{"instances" : [{"b64": "%s"}]}' % jpeg_bytes # send few requests to warm-up the model. for _ in range(3): response = requests.post(server_url, data=predict_request) response.raise_for_status() # send few actual requests and report average latency. total_time = 0 num_requests = 10 for _ in range(num_requests): response = requests.post(server_url, data=predict_request) response.raise_for_status() total_time += response.elapsed.total_seconds() prediction = labels[response.json()["predictions"][0]["classes"]] print( "Prediction class: {}, avg latency: {} ms".format( prediction, (total_time * 1000) / num_requests ) ) if __name__ == "__main__": main() ================================================ FILE: test/apis/realtime/image-classifier-resnet50/client_inf.py ================================================ """A client that performs inferences on a ResNet model using the REST API. The client downloads a test image of a cat, queries the server over the REST API with the test image repeatedly and measures how long it takes to respond. The client expects a TensorFlow Serving ModelServer running a ResNet SavedModel from: https://github.com/tensorflow/models/tree/master/official/resnet#pre-trained-model The SavedModel must be one that can take JPEG images as inputs. Typical usage example: python client.py """ import sys import json import io from tensorflow.keras.preprocessing import image from tensorflow.keras.applications import resnet50 from PIL import Image import requests import numpy as np # the image URL is the location of the image we should send to the server IMAGE_URL = "https://tensorflow.org/images/blogs/serving/cat.jpg" def main(): # parse arg if len(sys.argv) != 2: print("usage: python client.py ") sys.exit(1) address = sys.argv[1] server_url = f"{address}/v1/models/resnet50_neuron:predict" # download labels labels = requests.get( "https://storage.googleapis.com/download.tensorflow.org/data/ImageNetLabels.txt" ).text.split("\n")[1:] # download the image response = requests.get(IMAGE_URL, stream=True) img = Image.open(io.BytesIO(response.content)) img = img.resize((224, 224)) # process the image img_arr = image.img_to_array(img) img_arr2 = np.expand_dims(img_arr, axis=0) img_arr3 = resnet50.preprocess_input(np.repeat(img_arr2, 1, axis=0)) img_list = img_arr3.tolist() request_payload = {"signature_name": "serving_default", "inputs": img_list} # send few requests to warm-up the model. for _ in range(3): response = requests.post( server_url, data=json.dumps(request_payload), headers={"content-type": "application/json"}, ) response.raise_for_status() # send few actual requests and report average latency. total_time = 0 num_requests = 10 for _ in range(num_requests): response = requests.post( server_url, data=json.dumps(request_payload), headers={"content-type": "application/json"}, ) response.raise_for_status() total_time += response.elapsed.total_seconds() label_idx = np.argmax(response.json()["outputs"][0]) prediction = labels[label_idx] print( "Prediction class: {}, avg latency: {} ms".format( prediction, (total_time * 1000) / num_requests ) ) if __name__ == "__main__": main() ================================================ FILE: test/apis/realtime/image-classifier-resnet50/cortex_cpu.yaml ================================================ - name: image-classifier-resnet50 kind: RealtimeAPI pod: port: 8501 max_concurrency: 8 containers: - name: api image: quay.io/cortexlabs-test/realtime-image-classifier-resnet50-cpu:latest readiness_probe: exec: command: ["tfs_model_status_probe", "-addr", "localhost:8500", "-model-name", "resnet50"] compute: cpu: 1 mem: 2Gi ================================================ FILE: test/apis/realtime/image-classifier-resnet50/cortex_gpu.yaml ================================================ - name: image-classifier-resnet50 kind: RealtimeAPI pod: port: 8501 max_concurrency: 8 containers: - name: api image: quay.io/cortexlabs-test/realtime-image-classifier-resnet50-gpu:latest readiness_probe: exec: command: ["tfs_model_status_probe", "-addr", "localhost:8500", "-model-name", "resnet50"] compute: cpu: 200m gpu: 1 mem: 512Mi ================================================ FILE: test/apis/realtime/image-classifier-resnet50/cortex_inf.yaml ================================================ - name: image-classifier-resnet50 kind: RealtimeAPI pod: port: 9000 max_concurrency: 8 containers: - name: api image: quay.io/cortexlabs-test/neuron-tf-serving:latest command: ["/usr/local/bin/entrypoint.sh"] args: - --port=8500 - --rest_api_port=9000 - --model_name=resnet50_neuron - --model_base_path=s3://cortex-examples/resnet50_neuron/ env: AWS_REGION: us-west-2 S3_USE_HTTPS: "1" S3_VERIFY_SSL: "0" S3_ENDPOINT: s3.us-west-2.amazonaws.com AWS_LOG_LEVEL: "3" compute: inf: 1 ================================================ FILE: test/apis/realtime/image-classifier-resnet50/cortex_inf_rtd.yaml ================================================ - name: image-classifier-resnet50 kind: RealtimeAPI pod: port: 9000 max_concurrency: 8 containers: - name: api image: quay.io/cortexlabs-test/neuron-tf-serving:latest command: ["/usr/local/bin/tensorflow_model_server_neuron"] args: - --port=8500 - --rest_api_port=9000 - --model_name=resnet50_neuron - --model_base_path=s3://cortex-examples/resnet50_neuron/ env: AWS_REGION: us-west-2 S3_USE_HTTPS: "1" S3_VERIFY_SSL: "0" S3_ENDPOINT: s3.us-west-2.amazonaws.com AWS_LOG_LEVEL: "3" NEURON_RTD_ADDRESS: unix:/mnt/neuron.sock - name: rtd image: quay.io/cortexlabs-test/neuron-rtd:latest command: ["neuron-rtd", "-g", "$(NEURON_RTD_ADDRESS)", "--log-console"] compute: inf: 1 env: NEURON_RTD_ADDRESS: unix:/mnt/neuron.sock ================================================ FILE: test/apis/realtime/image-classifier-resnet50/cpu.Dockerfile ================================================ FROM tensorflow/serving:2.3.0 RUN apt-get update -qq && apt-get install -y --no-install-recommends -q \ wget \ && apt-get clean -qq && rm -rf /var/lib/apt/lists/* RUN TFS_PROBE_VERSION=1.0.1 \ && wget -qO /bin/tfs_model_status_probe https://github.com/codycollier/tfs-model-status-probe/releases/download/v${TFS_PROBE_VERSION}/tfs_model_status_probe_${TFS_PROBE_VERSION}_linux_amd64 \ && chmod +x /bin/tfs_model_status_probe RUN mkdir -p /model/resnet50/ \ && wget -qO- http://download.tensorflow.org/models/official/20181001_resnet/savedmodels/resnet_v2_fp32_savedmodel_NHWC_jpg.tar.gz | \ tar --strip-components=2 -C /model/resnet50 -xvz ENV CORTEX_PORT 8501 ENTRYPOINT tensorflow_model_server --rest_api_port=$CORTEX_PORT --rest_api_num_threads=8 --model_name="resnet50" --model_base_path="/model/resnet50" ================================================ FILE: test/apis/realtime/image-classifier-resnet50/gpu.Dockerfile ================================================ FROM tensorflow/serving:2.8.2-gpu RUN apt-get update -qq && apt-get install -y --no-install-recommends -q \ wget \ && apt-get clean -qq && rm -rf /var/lib/apt/lists/* RUN TFS_PROBE_VERSION=1.0.1 \ && wget -qO /bin/tfs_model_status_probe https://github.com/codycollier/tfs-model-status-probe/releases/download/v${TFS_PROBE_VERSION}/tfs_model_status_probe_${TFS_PROBE_VERSION}_linux_amd64 \ && chmod +x /bin/tfs_model_status_probe RUN mkdir -p /model/resnet50/ \ && wget -qO- http://download.tensorflow.org/models/official/20181001_resnet/savedmodels/resnet_v2_fp32_savedmodel_NHWC_jpg.tar.gz | \ tar --strip-components=2 -C /model/resnet50 -xvz ENV CORTEX_PORT 8501 ENTRYPOINT tensorflow_model_server --rest_api_port=$CORTEX_PORT --rest_api_num_threads=8 --model_name="resnet50" --model_base_path="/model/resnet50" ================================================ FILE: test/apis/realtime/image-classifier-resnet50/neuron-rtd.Dockerfile ================================================ # LIST VERSIONS: aws ecr list-images --region us-east-1 --registry-id 790709498068 --repository-name neuron-rtd FROM 790709498068.dkr.ecr.us-east-1.amazonaws.com/neuron-rtd:1.5.0.0 ================================================ FILE: test/apis/realtime/image-classifier-resnet50/neuron-tf-serving.Dockerfile ================================================ # LIST VERSIONS: aws ecr list-images --region us-east-1 --registry-id 763104351884 --repository-name tensorflow-inference-neuron FROM 763104351884.dkr.ecr.us-east-1.amazonaws.com/tensorflow-inference-neuron:1.15.5-neuron-py37-ubuntu18.04-v1.3 ================================================ FILE: test/apis/realtime/image-classifier-resnet50/sample.json ================================================ {"instances": [{"b64": "/9j/4AAQSkZJRgABAQAASABIAAD/4QBYRXhpZgAATU0AKgAAAAgAAgESAAMAAAABAAEAAIdpAAQAAAABAAAAJgAAAAAAA6ABAAMAAAABAAEAAKACAAQAAAABAAAB8qADAAQAAAABAAAC0AAAAAD/7QA4UGhvdG9zaG9wIDMuMAA4QklNBAQAAAAAAAA4QklNBCUAAAAAABDUHYzZjwCyBOmACZjs+EJ+/8AAEQgC0AHyAwEiAAIRAQMRAf/EAB8AAAEFAQEBAQEBAAAAAAAAAAABAgMEBQYHCAkKC//EALUQAAIBAwMCBAMFBQQEAAABfQECAwAEEQUSITFBBhNRYQcicRQygZGhCCNCscEVUtHwJDNicoIJChYXGBkaJSYnKCkqNDU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6g4SFhoeIiYqSk5SVlpeYmZqio6Slpqeoqaqys7S1tre4ubrCw8TFxsfIycrS09TV1tfY2drh4uPk5ebn6Onq8fLz9PX29/j5+v/EAB8BAAMBAQEBAQEBAQEAAAAAAAABAgMEBQYHCAkKC//EALURAAIBAgQEAwQHBQQEAAECdwABAgMRBAUhMQYSQVEHYXETIjKBCBRCkaGxwQkjM1LwFWJy0QoWJDThJfEXGBkaJicoKSo1Njc4OTpDREVGR0hJSlNUVVZXWFlaY2RlZmdoaWpzdHV2d3h5eoKDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uLj5OXm5+jp6vLz9PX29/j5+v/bAEMAAwMDAwMDBQMDBQgFBQUICggICAgKDQoKCgoKDRANDQ0NDQ0QEBAQEBAQEBMTExMTExYWFhYWGRkZGRkZGRkZGf/bAEMBBAQEBgYGCwYGCxoSDxIaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGv/dAAQAIP/aAAwDAQACEQMRAD8A/VKiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooA//Q/VKiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAphkQHBPWs7Vr5LG2aZ2CADqelfM3iT4xi0vXtIiQAcHaCxz9elceKxsKHxHVh8JOt8J9VLIjZ2nOKfkV8i6V8b4knRL2TAPXcMY/Kvb/AA/4+0fVYg8UwZ26DPSs6GZUqul7F1sDVp6tHpdFU7W9huVzG24eo6VcrvTT1RxtWCiiimIKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAoopuD3oAdkVl32pR2ti94CPlBYA9wOuPwHFc/4v15NHsRzhp38gexbBB/Lj8a+aPiH8W4reF47YM7iR2WNTk7DwFbHQDnP1rzsZmFOgmnud2FwM6zVloeqal8Wb20AkhgjdWJBGGyp4ODz6Zrn5fjjqILeTbwELgc56/8AfX6V8Z6h4y1fW5XK+YkR9Rhc+u3/ABrn5by+d1zcyZHXIxn8hXzM85xDekrH0MMopJaxPvzSfjeJ5fL1KKNOM5QHqO3U8+le36Vrdhq8CTWkqy7gCdhyBkZxn2zX4/XOtX1vOWW4kVsjGCAK+hPg38aDp1/FpWqShI5mAB7474PGM4ruwWcT5kqzuu5x4zKYqPNTWp+itFZul6pbarbLc2Z3Ie46frjNaVfURkmro+daadmFFFFMQUUUUAFFFFABRRRQAUUUUAf/0f1SooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooA4/xtDI/h68aIciNjnr2r8utQv531CRrmYxMWPJYknnoF7nH1xX6y6pBHPYzRydGUjn3Ffkx4306TSfEt3DJIUjikYOTgMQW4A47181nsWnGZ9Fkck+aJZj1CRiY7aJ5PlyzyyYzjuccL+Jq9p2q6pA3m2l2I3B4CMxH8h+dYlnkxCSVSsQ5CHvju3cn0/lW2keoIu52FujDqw+Zh6hBjj/eP4GvmozbZ9BOmtj3LwZ8aNS0t1steAZV4DKePxr6s8MePNH123VracSuwyQO31r85vsqy5gjk3Njuo499qrx+JzWhomuax4Vu0ubGYFMglfu5/4CTXq4XNa1F2lqjy8TllOqrx0Z+pCOHUMO9Pr53+HnxbsNZjWC/kw+Bwa99gv7S4UNFIG3dPxr6vDYunWjzQZ8zXw06UuWSLdFFFdRzhRRRQAUUUUAFFFFABRRRQAUUUUAFFFGaACikzmloAa7pGpdzgDqTXLa14ostNRkDZfkfQ/5IrD8ba69hauiHBfIX6pn+or5V8ZeOJrW22pIXuZR8q9SCTliR+tePmOZ+x9yO56eCy91tXsT/FD4jy3M32dJMTh2AABbGM44Hfnqa+ZWubi9uDNdF23MfmYDBPf5T0xV6WC5vZ5JxcebI3UyDGS3Tv07ZqKJ7QpJBe+bbmNgkgJyFbOMjPI/zmvjqs51ZOTZ9hRoRpQ5USh/Kb94+z0dfusPTjpWZe3kYXEcy7v9ruPqOP0FW5rKaydWeQz2sp271OQM8ZYdRn6cGubuLNhMd4G8NnrjnoQQeDms+SyNVqZdyp37ojkEHK9SOf5Vzhml068W4jJVkbPHGD6iupLRxn7m3HYjke1c5evBNHjvmrpysxTjc/TL4AePm8QeHbeEy5dBiRCAACOMhgO/v+dfT6uGAI71+SHwO1+5stYexkuxbFhuUkkKSp54APX0NfqV4V1JdT0uGYAkgAFuxPtnkfSvr8nxTnD2T6Hx+a4b2dTmXU6eiiivbPICiiigAooooAKKKKACiiigD//S/VKiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigCC5iE0JQnFfmv+0J4fl07xS9zApWOUhg20s2f5Amv0vPSvl79ojwi2qaA2oR5DQDPy9ffHvXk5xQ56N10PVyiv7Ouk+p8RaFKZSGUDcvCknIGO/px+prrBHa3MwMjltzcsG+8fQcZ/wA+p48z0+5fTfOtlOZIuCc9MkKFX/PauwtdRt4Ll45kCBBtCgbpAmdqr14Z/wA+TniviIxaZ9pNX2N+NJL2V47MMtsh+YJ8qlvfux9yfoO9Wl0y2CmVNgGcEhdxOOvPP88UttN9uzPOSYrcfJbxjChv9oggMfY8Dqa0IbD7SRc34aTLAJFEOOO+TyRnjI6npXbGHMjkk7GGI7vT5vtGns6MDleAP0zXq/hv4p6lb+RFNKUkiK5DcZ2nIOPQ+1ctDopkcrdxxwq3IQ8yH6ryR/wIk+wqld6GmP3TrGQQV4BK/iD39KpRnT1g7GM1CppJH2L4T+KNnf7LW/YRyM4BJPGDx1/WvXbXULW8j82Bww27vwr80rXUr/S5fKmw4HI7AjvjPevYNA8e6lBEpilYRbSrZ7Dtn6GvVwudSh7tZXPIxOUp+9TPtqivDdL+J4ljTzhyuC3uB1x/Ot6++JWmQGRYX3so+THfJ6/lXsxzTDOPNznlPA1k+XlPVKK8Yt/ilar5T3KkCQHcPQr0H41txfEzRzaLM5+dskr6c/4VdPMMPP4ZEywVaO8T0yivMW+KGjZUxqSGIHX1qhf/ABU0y2nMYIKhSMj+/jj8Kc8ww8VdzQRwdZuyieu0V4Jc/F+1D7IFLDGfyOKzZvjI0pWRIsIeMZ6H/wCtXLLOsIvtm8crxD+yfRpIHBPWjINfM6/FG/vWhKpho95Y5659PTAqW1+JWsW0i+eodYwfYc5Az+FT/beH8ynlVc+k6QjNfPTfFe+a4K+ViIALx16/4V0Nj8VLctJ9tjKc/KB2Hv8AWrhnGFk7cxnLLa8Vex7KBiqOo6hbabavdXLbVQE1wl58R9Hghyr5YxhsDrls4H4da8N8b+P7i6hlW4YqnJRRz2AyfypYrNqNOD5HdlYfLqtSSUlZGZ8SviFDcRsYBtCOzxjOT8x6/QE8V81Sy3erXr3quxZWxgLgbu+Wz3PrV67nuNdvJWY7Vc5ZsYCrxhSzFRnn7q+tdlpuhTmBIuYLhMKjkAowPOGA6D3zXyM3Urz5p7s+upU4UIcqONWynnjYMuZAM/NwSAc9cDp7ipbyx+2Wkd9kTb8RS9x8vC7vRsYB9cda7e50OaH968DQujZUlSQrezDsagbTZpnOQrrMMSJ1U+uVPOPQg5FbwoeRLrLdHm8dkIYJbG53NbnBIccqp4PPp/LrXIalbPZ3DWty2UJIVsg5HY5Feo39lLHGEVpIymcB1J+XoMZ+8OOevFcNr1g1xBHKEQZG3IUEe3XGQR+NTUpWNYVNTz28MxbfuJA7/TiuPmkKuyKeSTj2rq7u3uY2KBc44+XpWDHbEy7nxjmudRsbPU6bwHdG38TWrruBzggduMEmv1C+DF/LcWjISHC4B24GPr3/ADr8r/D8LQ65bvuGwMDk4GB6/j2r9K/gzaTiU3UROw8Ntxlc9m/pXrZTJrEK3Y8TN4r2ep9Q0Ui7sfNyaWvsT5IKKKKACiiigAooooAKKKKAP//T/VKiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigArm/FmkR6zodxZv1ZDg/4V0lNdd6MvqMVFSCnFxfUqEnGSkj8f/F2hS+HNcubMsEkaXIB5IAPUDux7ZrNtWkH+qbY0kgz/E7ueg49SfwAr6T/AGivBH9mXY1vywFcEs69c+nqa+dvDTQXN5bxheAS20nbt/vM30A/OvgsTRlCq0z9AwtZVaKmj0S3vLWxaHQ5ys05BeTcV2KiDJkdR1GegPYZPpWzavJf3X7p5pJJMFtmRJkj5Qo528dAMBRySSad4e8OWc1vPeXETAXZDyuW+fDEFRkA47bVAJ79a9M0v+z9PtfskSLawAliu3YxyernJJJ68kk9zXXSo3V5M5q1VR2V2c/a6HevIdyeUH6gEuxx/fbOAB2A789aZc6SY02i4YOpxmPA5/HNdW+rW8qljF/o4bHHG4kdgOWJ9P6VgXN9eSJhLTyEH3Y1KbiPQ9Pxxj8a1lGFvdOVTm3qcldaSxVvMJcDoXbJz9SBWXZXkukuAGDd/LOAduMHBXIIre1OPWGbJjVARyGdePbHeuD1a0ZFErSYxzsXJ5Hc9h9RXBVp9UdlN3VmdvbawJleS1PlvHnfHnOR2K9609N12OZgCPmJPH5153o199kuEMhyoBHI5I6j8Aenaqs0503UmhaQssjZXnpk7v8A9Vc3mVKNtD3KW8gnRuh24P6cn9awLi/lSXybT5s+/Q1TsUM9ivk/MXbj8OT+FaenaIrYuI2yAH3HPJOf6VnOHOKKUdytObhnSKFjgthsfn+FP8iWFkE55UYck/3uldlBFaQHeuCxJI47471z+oRG4eaViFVSBnHH/wCvrzU1KSUUy4TV7WOVUSQCUoCzb8H/AHTip4Z1Nx5bkKWclfQL1/QVfk8hbBVkz5s4Ue55Y/4Uy5tWEalX8oAFR77yeOPQCsFS6o3c11NbTtTtvLkkiZCisB1HDNyf8KfcXMUmYGbndycjB5yao+H7SEWRjH3gxY+vJ4HuaTUYoIvk4aSVhs/LJNdCbsYNLmNl7qKGJHj4XHGevPGapLcPLI6xEbR8xP8AeOM4/Ws77OLpke4yIyOcjHAOf1qzDdQSXQsLfDHPJ9Mdf0q4tt6ktJEmta6ukwiOMLJcnacP0QHA5xXmk+vRzFri/Ly7i2BnGQoyff0/DjioPF9/LNqsrQuFEQAweCQMgH8OTXnWoXi2phiacR7V643MQPuqB2GeSc8nsTSd5Ssb06aUbnsGmSSSItxdSLHGRjyiAx555Y4x9AK3YruRz5cdrgx9PMlXaR6ADJ5+grx3TtWluLtLSJpGj2/KOUwM8n5DuP4kn1Ir0drY3BEH2m3tQw/1cyI+Rj2yxJ9ya9CjHQ5q2j1OhXW2tI2W/KwAHOULTAD68kfyqI+J7KchRcx3IYk7RlHx9M9q5G802SHMGnOINgYnyZUDAdN2HC4H+6vHrXkWvapqKnypLiKeTJB2ljKdvcruBb6iutRdrr+v69TKNNSPoz+3NGv8wzE5OCBsJAx7jpz61gXvh/Tpo3Nmcbzltvzqx/2kJyD6EV8yNrF6wVDcSW/ozAx5/wCA9fx/OrVvql1A/wA7NvxjdnaxH1HUfnSkpdTWOHtrFnaeI/D89shkjkBzn7vLH/H8ea4J7dbaMgnB756/WtSTVJ4fmmlabfx8xz/kiueur0yEq/8AEOorhnG7OuN0rMvaO5l1OIICQrLkgDPXtX6l/BuK/ist0ixNGRgOOXHsSD2+lfm58P8ATln1qxVxyzMVXqXKgnGPUYr9TPhppj2OkRMzhgw3Kdmxh6qw45B9RXp5PC9e66Hh51P3LHqlFFFfWHyoUUUUAFFFFABRRRQAUUUUAf/U/VKiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooA83+J3hBPFvhqezUDzVUsp75HSvzh0zRprHxSumyr5cibotxUEljnJGRwBk5Jr9ZWUMpU9DXxD8evAw0vUW17TP3P2gYZunzH0Pb/wCvXz2d4e1q8fmfQZLi7N0X12POb/xvo2kXC6dYzbvJHGw/Jk9SXwcsxznsPWuevPGFvdsIJtszkjCCXaozjoGA/Mj6ZrzO1DbxBOxjRWB4xufHbkdOO/15NdJo+leJ9Vui5/dKAW3EAIvP3iT8zEA8KPxwK8SNSU9UfRewhBXZ041vUpZozY2kS2iZVXaRgm0dcAAPKxPXbhc9TXULq18BstoFVtvL9CPXAJ2gZ4yWPPSsx47LS2MWLjUrhFziMMQCP78uOT6IgAH6muJfFtwoWxs0sYF4KFtr8c/OcnH4kn8a6InPJJ7IsTm38wGSHa8g5JVdzfTHzY/AA1mTW7pCIleSRGPIZiu38+g9K1LWC581luctITk4c4YdOCeMenHPtTrmwulbcibQPlYpHgj+7yTyB34rOqnYSZwxRAmE+c9VOfTkr2zXUnRxrFxBeoMIzKzDqPlAOM/h+lXLbRYpmzcLhkOCQMZHXP41uSkROYoV2L1BH94Zz+vNeXVnym8feNG3uYNLtTASDtPy/jWnp2qCbCR8bwTjtnArza/mmBKseuR9R/8AWp2g6hcy35hGVVAQSeg61zKtJs1dFWbO81XUngkBhPrkY5ABxx+lJqF5K9qlsowZdvPf5jgViTKJpHM5wWT5eeucevYVrpIvmJLs3CMgDPP3ec/hVqV3YmySRjW7qdXs4JwzKu8/98L/APXq/NcRy3cUZbKld+PTOaclmtnJNkbnV2MfHODjI9qyrSOZdR+2yYC54HYg8H8qPhuh76o20uUsrgup3ALnA9c9/wADWet+txqEczfdQAjPoetYOtXpt7mNs4GMH3wcf04pLOVrmRJ8AbwFyOePvfn2qOdrYpQT1Z2mryzSWx8oHZIOo+nFcppV5FpP2m7vOJEUlWbrkgcV0lzdPFagDO1FLEfgAP5153rcAv4JZnk2JGhLhOCSeAg+tbueqMEtLHmWt3893qJjjcgznIUDcSc5UAe55yeABUR0hUjI1GF3zwsioRIT/sYzwO5OM9qZdaVeQOivCZbuddwWMnESMcBSw/iYDoOceldhpPhVrC3e41mMWkBA3BZGWSUsOR1DBR+v05r0KdLRMcqtkc/bamNIjWxtlmSCUnMsICuD6qzEkkd8rjsOma0P7WdZ49M1icTIqlw32hCxQ9CyNGdwPvyM1pS6Hf3iS6hYyqlo67VGI0TYv91eWIXuWbnrXOP4VttUVPmAgiG9jA2ckdxJwD14AyPYV1xgk9TGUkzvdP1Tw00iaV9vktcHPBUoB2wRkD2+7TtX8C2OrRC7s75Z8qSwOwMew3K0bkfUH8689/4Rm5uLOTTVaG/hGfLS5BhuIixzkSxg8Z/vDHeoYtI8W+Hbdbq8nlEFuxAZVacbCuWVigb046D8a3gu39fqZtdmZV14V1HTrloEI6FsR4lcD1OX3gfVMVjtaGzVpZJBKD0KqU/MN/StDVdZ8RWNmTfPb6jazcxMr7Jx7eW4Xkd8AE9a4aXV/td0oVTCwAypGD+NOdzopyfU6B4JWAdzkFQ351SSMySIHIALZ/AV1+nXUVpYxSSxrKOQc44H41mazZzG8NxbHdDtGwk889vY15dSerSOhHpXwaistb+LemQt8lpZW8i84wXZTnuB37mv1Y0WC3tYEhtSSrYIznIUDGTnnntmvzA/Zq0wf8JHqWsMq5tIQi7hn5pGx+PAxj3r9MvDVwJrfibzmJ+dlA5I498KOg/HHFfRZLFKnfqfK53L97yrojrKKKK908MKKKKACiiigAooooAKKKKAP//V/VKiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAK8b+Nmiw6x4Ou0lG4hDt9jXslch42s3vdBuIkA+4SfpiuXGw5qE15HRhJ8taMvM/M3w5pVtBE1/rJZvJ4ZQpy/PyhfX8P5V0Fxr8d5GYEjZWX/V2sJB5HTzJCcE/wCyOB7VX8R2V7bWT2gDD5yAFbaCDy2cdP8ACvNrtDaQhbEhHJ+Z1Hf+6uece/U+1fBRqSi7H6AqaqK53hn1eaF4YpILEZBMSSh5WbOPmbDhRnk4Vjx8oHWrNtbLczeUZJ9WnQ/6qFGggAzjAbLSN/tO5rzBIbnUn+wm7MbjmZtwwB3XAIOAPTqeOgr6X+Hfhq2to47u2mVkyMtyXkx03N3Yf/Wx3r1MOubY5MTJU1dnR6J4T/0NDqlqLfI/1ceWVc9QW6tn1ySadfeH7axZltgie0WM7fqRn8K9MWa3iGRhAOuRgj3OOv5VzeszQGNWjQHPQriuzFKEaZ5FKrOU9TzEwyjdDJ8yqT+HuPp3FQG3eQHI6de45HWti6LS/v0BHPJPb61T89YSUbAyOCOf84r5mtFN3PZpt2OM1aJvJZh1Ukfh2NP8PKx8ySRQCxA9+oyao6vdCa58tMk87ucdOePwrpdKgNnaAznceCG7HP8ASuOK1OmWkR968ZkMZ+Xym2luvy4J/nTlnFtKNykEl8egz09ua57ULlpdQFuPus4BI5OScHPoKtIbi8iljAOYuFA9evHf1qkm3oTolqb32oTTWzZw0ikEf7QIPP1qtfI9vGFQFsKWH45IGPxpLVo474goQSVb6ZGD/wCPU/W3jVGlZsED5cHp3FW/MjrZHmt9ema4EUvEhJAHbJ6fqDXY6Y2YUAOVb5go65HX8K4XUvKub1rqFtz5JwO3A4H410miXJa3Dn5cJkL3z6VLWxq9jrr25RN6hs8bfXjI/WuUhs/tEjXtwB9nViFT+8x/ngUs140hECnLs3bng5q2NrQwo+T5QBKg9c54+projHqzmbsUkjaO5a4tTvu7gqMjkxqOMIF9up4rrLTSpLmNWuGfc5O0OFI/Uc+5zgdBz0qWJSHdPMDGeXcrxwBgD8+n09q6SwuBLsuCvlr0T+8wHoTyBXfRmc1S/Qzp/DEUtu1xf3DyTKQI0hXa68cBWGCPyrgPEWi3VjCUsZ3jveHEarnzQOPnAzggf3fvHrnmvclmhwGuQHC9IxyXJ7HP/wBYV5d4xb+1LecpMXbOEgsEyPlPCNJuTdz15Ar0lytaHPGcr2Z4fp+s6nsktLpEuIELYN0vkFW4GEZXRcnPYqT6VvweKYbRZ7LxNo84gVUJmhUncT0AAJB7DGeMHr1rkNe8B+I9de2lvYpWkhZjKGCyO5GNigoNg6c88VDDpniLSk/sy9jmSNidgh3s1u548wN8oJx2J55JxxVtNf1/X5G6SZJr7eF/EF3KtlfzQ26H5zDM2BIv8DwzAj5Rx8revXNefNZ6le3git5/tMY+6NzDGPUOFwPpmqOv6b4ntbhIDPIVA2RRxOWURk/fZ/VvQ5JPWruk2N/aWbwvM5dyCzbiQB6c1MnZXOinE0JGnhBgJOQe/tXo3hm0ku4Htb0gIvzKeB06/rwM1wGkaPPPqii5dnjjBJQcZOeM8c8V7Vodt5LF3G6V+igcIteZiLJnQ37p1PwKsZbHUNetHwcGE7RzySf5Dgmv0T8HuY9MjRsDPAAAHA47dq+KPhpo8lq2pXcW7N1NGuR97CDn8MnrX274TsWtNPiLDkqOTnP0yf6D8TX0uUp8iZ8jm81Ks3/Wx1tFFFe0eQFFFFABRRRQAUUUUAFFFFAH/9b9UqKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigArN1gE6ZcADOUNaVMlQSRtGejDFTNXi0OLs0z88vGVlc/aZUAQRyk7lPcH615RqOjyGDChgwPynHIx0zzz9a+r/H3hwW95NEqfeywJxjB9civD7/7TZrsVRDg/eYjGB1I718BVpuM2mffYTEKUE0ct4N+E19rFyl/ffJDncMFhyPY4yP5V9P2OmQaPbpAIlBUDnuQOnP/ANavC9E8SalARFFfROwzgyPhf/QT/jXfQa9qzw7tQVSOzRnK/h616FCtCENFqceMjVqT956HT3WpurMqsR7Hkfga527upZPnBJ6npVmZopFE44DDr15rEu3Cjfnp+ePauLFVm9bhQppFGa5kQtkfK/A56Vi38rpGVjGCuCe9SzSmRiScZ/DNZ91cMVBJ+YHaccH/ACa8mc+Y9GMbHEXErJfgOdpkJBPp2Ga9EtHd9PRZOrDcc84H/wCuvOdTtZ7md2QYCr25OPX8D60nh7xLPcf8S28wOMehOKmETSeqOg0h5H8RtDL8+1xJgf3CwBJ/OvSYdMe1vLdI03ASukh9QjHj8M5rndDhgi8Sw3T8R3KyWjccBiMrz/wEYr3HStOWZPLbq+JQf9vaAx/EivZwGEVSNzzcXieRnid3pstlrkk3mfuWYoR/dPpz+B/GuU8U3TNB6ZHr6V6F46dbXU5geEnCHPTDL0P9Pwrz3UtNudQs/LY4ffvHsrZx+o/WvNxELVWkdlGV4qTOTsrYzwQmOTa27nHOQxH8q6jyYEt3uIyBGgCrjj5geSf89ax4YJIE2YP7pOe3OT/LpUP2nMUaxL8jMWAHPAJz+FZJO5pJ3NolVVJARuZCp9RkY/PrWjaQbcM5wEQkY7EjjPuaxrhJ2hDRrgF+P93j+dbTyC3eEEgAHzGHqccCuqN2kYSLVvIZBlvkXCrgjOMgA/jitV7hkCSxnduAC5+uOf8ACsCBWnMSFuZCWznso5PsOa0fMY+SsRG4jIHXqeP0NbRRjIuyTkxuZwWydiqM5dj16f5x6VesoNPhVVvz9ocEnltsYPZQq4zgfhWXEhXYCcKg69yT7e9UrmS6t5d6IT2T6nk9a6YTcdjJxvoegPf26lS3k2qAYCooU4zxjqf0Fc1repaS0DxmYIeeMAtyepDAj865yLzZiFQh5DycEkD1J/8Ar8V5/wCInmgcwxzF5Dn5t2FXngLxyffpW3tW9yYUlc89+IPmxyhhcquTu2vguw6A4AJHtnGfSuU8PsLkEzyZwdp962da0eW4AjjnSVi/ziMlsE+pbGD6muet7SfRrloG+bzeVx7d/pTc7xstzvp6aM9U064toPkiABAH1J7Zr0PSrOTyxc3TMo4bkYGR+PWvNvCemSzyCeToSG3YBP4civbfszqYY1UmMkZyOw7V5zjedhVqtlZHtXw40hp7aETzmNHbeRk7j+A/rX13aQQwQokK7VAGM8nH1r5y8C6c8t8hi2QqiAAsM7R32jp+J+g5NfScQ2xqASeOp619tgKajTsj4rFzcptj6KKK7zkCiiigAooooAKKKKACiiigD//X/VKiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKAPKfiDp0NzEZM4dOlfKHivTmZXEIChucjk5/Gvr7xrcIITgD5e9eBXNtBcS/v1LDnv/AEr5HMYp1pJH0mXTcaabPn/RPCV3dXzNcR7UXHTaCQfUY/8Ar16la6YumL5MUxkjxnnqB6HqDXXeVEkey1QKMcN7/WsKfadyP949RXmzXKen7RzepWkdVUiPkeg6evSs2SP7WV2dR07H/wCvVi4gkHMeRnv9arWl3HFNtvwSgP8AnHFcsnzOzNYqyujEu7G6hzuXjpz6ensRXPXSSRgrjGBnvyM17W9lp2p2vmwsA6jB75x2OK881pIU3RNwQMH29/pSxGF5FzJ6F0K/O7MoaXYx3Nv5pXJIK9DjB9a8k1G2GneLolRDsZwv4k//AFq+idDt/L0/KL0z+R615zrOird649xHzLBJEyDvtLAZ/AmiNPReY4z95o6uKwk/cQIuZYphIo9SQT+lfTFjZbY45gu1imcemR/SuTsvDcb6nBdYG2PqMZHzL/8AXr1CO2VIxH/dGPwr6vLMG6cW5HzuPxSnZI+cfFWmLPrUcc3zKcqdwyOexrnLnSzbIsMg5RcZHoOnP1Fe3a9pGZvOKkrnt2/ya4PxDCFjYxjDFe/9K8fGYVwlKTPSw+J5oxijwnUdkczxIu3JOfoxzUWm6W0ske/O1R2HQY5znvz+vtVi5KRXCMVIIyST04zzXe+HrMXarwAOO2R+NeZShzTsejUfLC5CmmgwRxui713O2TkjP3R+Vc7PDFLcOWj3IoHDcLx1JPU5xx7V3OtXttaZsrI7mA5GOWPcn1/lXCCK8lmlMrhjkBUXJA+uOv0reslB2iY0m5K7IFZpZGf5VVvvED+HsBWnE6uJGjXHXpycYx1p5sblgzyFE42/Pldv0Azz60FZLYeXbOX3/edRjP8Au9h7miMhyQIRDtHKhRk45JPHAqtLatesvnDG4846Adl+vrWgkiWq8KC/dmO7HsoFTRN55DsvA5Abj8hWikZNHPXljLCBb2rAxkcgZ+b6nso9PxrDuvCd5qakxys+7KkopX6jcckD3HJ9q9CnEPE0cSs7nuSefZafG91G224+RjjIPX8AKnnaY13PKn+HtpC6iNN+zqAensM+vfvW7aeEoY49pgQ5xuyM/wAxXp0ckbMFEeAPoD9eelWGtzjMakg9iarfW4nVZwlhoFnAxMW1JFz2B5+hrbjsZVnUyNnBHTp+lSzsUzwAyk85B/wqG3vvOuY7flixA4/+tSh8SIm21c+o/h5DboTLkyucHaoyAAOCT0Htk17WpyAa8t8D2XlqsTsWWJRhc8A9yQPy5r1OvusOrQR8hWd5MKKKK3MgooooAKKKKACiiigAooooA//Q/VKiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKa5+U/0p1Nf7pHrQwPIvGkB+zs0ORg5OTzXkJaJE8x8jPYn9a9h8YoUh/dsx7c9PevEJpI8iEpsKnORXx2NdqzPpMGr00X5HMdqHDAHtz1z71zN0yZzL+Z4/Wr107Ngswb0A5rIlkAmUuCM9Aozn8zXmVp3dj0aUeo6GESMZfMDBeuWwB75I/lSXdudm5tkg7YGT+dOutesNLtztT7RNgEKCAB6lm+6oFeL+Ivj3oNlcyQxtDdzxn5vKVpEX6udqfln2pqMbcq1fkVzS3PdbKRIbc7pSoboCp25/L+tec+I5gbxRy3bjkfpXzndftPRxT7LiJSvIzCsikj3JO38hiuj0n4xeDvFUyQtcGGRgB8wxg1rXw9XkXuu3oKjUipN3PrLw9AF0yNV+YngZ9DU+l+E2u9Zhu3Xd5TgPnuByPzPFc94U1QwwLaTkbVI2kdCuOCD+Ne+aBGrW4mGCWPUdwK78Dho1Gr9DixWIlTu11Ny1tFjxgY4H51rEYFMjUdae5ABHoK+nSsj55u7MXUY1eMqehrxbxLhQcjgZB/CvXdRuQiNXh3izUFjZmBPPbqTntXh5tZxPWy6/MfP3iS/EFywzhcE57ACu48K6ysOmhwTGAPmY8du3px+NeY6+yWxn1G5OYUIK7j0HXHPB54r5v8AEnxR1R7trDSpCQn8QJ2qT1wP6mvCo4ac5e4e7Uqx5bM+1J9b0yS5Ms1/HFnKhWcA7h1ODk/nV7StX0GZALa7EjgckFd2fYEg498V+b0fjrxa12sFjI00ruMAKDlugA/wrpP+Fn+KtLums9bsofPhO1w0YRlI90wf1rteWVl7yVzk+uU/hufopE+mToxEm/Pdcbvw3Lj9TVG9u9Hj2xrJI5XtINwyP9wgV8Y2Xxe/drJfQSxsoBDwyFW+pz8rfQj8a9c8N/EfUvFkQV4vMQHIm3YbA6cco30IrCVGaTvE1hJN6M9be/lllyJMjsuCn6d66K0UvGC7Mm7nIAHH868vj1u1j+a4wpHLYUJU6+OLSBglpl29Sefzx1rzlLleh0yg2j2W1gtLL98AzOc43N/QVJK9tg7l2N3APLfXH8s15Gnji7aQbtoJHAyCR7nFbEHiKW9GHcIWHRVOfxOMfzrdVU9DB05J3OnS/EUhiBGOe3f8P8avxXSsDucrnuAST9BjAriWX5PO3YXH/LPPftzjn8atW+pRsnlCXjcQATnIHHP/AOupUmnqOUU0dVcp5illJB7bufzrK0G1kGsxyEkhTxsGD+tRtLbRqv2gk7uyOo/qa6Lw5brJqUY6AHIBJJA/CunDrmqRsc9V2gz6t8EbxbKGQru+bGPvH3PfH4CvRq4XwlZxwQiT5tzgdeM/hgH88/hXdV91S+FHyU9wooorQgKKKKACiiigAooooAKKKKAP/9H9UqKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAqKVgqnNS1RvCQhpPYaPMvGE37rhTg8DsK+etUlMMjuCcLzkAnmvoDxax8n5evc9/oK8F1kGMSSK21hkgDknNfH5qrVHY+ky5+7Y8r1DXL2SVxZZ3DgK2efyBx+NQofFd+BGJYokyAxDHdj64/pUltaveXTLGzAk8vnHPsAea9d0pEjVWKscDBbA/XjivGoUFOWp7NSqoLRHlvijwrP8A8I0yzMzs+Nw5A54/zmvhPVrA2Wi3c7KN6yuv0IcqfyAGK/VbXtIj1TQrlLXhvLJXB6lecHqa/PnxxoYF5caPGnli93Tw9MOxP7xf94da9dxVBW6bnHCbrRfc+W9Y1PRbm10+LSjOZfs4a9aZEQC5LNlYdrNmMJtwzYYtngACqV3rdrNo9jpsFjDBc2sk0kl4hfz5vM27EfLbQsYX5QoBySSTVbWdIm0i+e2uFIwTjjH4VVt4TcSLBCuWbgDHNfUxrRac4bNfgeH7Np2e6P0G/Zh8ay+K9Pk8Ma45e7sFEsMjHl4iQMH1Kn9K++/CrNDD9mfPydPoTxX5P/BWZvCnxJ8MxEjN27wSgH+CRD1/ECv1k0oYKOo68V5uD5XNyhsb4xNQXMdyjAZx6VUuptqE5qVG+WsTVpmSAgd816knZHlxV2cPr2pPGvB5NeR6tDLqFyRuwqjn26/4V3mqt506555rH1S0WGylMY2syHn265z9ea8WvT57tnsUHyWSPir47619hthptp8rPmRgP7qD+XpXzHptqtrpM2oOvmsE3nPGSeTzXs3xavDqniy4UEyIuIx6Y6kD615RpEJkt59LkP7xASAehTtiudR5KN13VzumnzW8jzSHUJYrmO8G1pI3DgMoK/Kc4Kngj1HcVY1vXNR8Rarc61qjK1xdSNK+xFiQFyWIVEAVVGeFUAAcCnanpU+lXBEiHYeVbHY1TgilvJlgt1MjscADmvejVi4Xi9NzxnT967Wp6N4UtZtSsH/eiIFWUk8ZXByCfQ19ifBbwzBF4Tt5TFucRiU7x0LuSMjPPH5V8++EfD0sKw6JEP8ASZ1BkI5Ecf8AExHqei+pPtX6N+APCsei+G1S5hbdJ8zKMfKB90fUAV4Ln7VyUdm/wPUS9nFSZwkvhC3u5WkMeR1BGCgz6Z//AF1xureDdPthuhiecrnCkhFH5DNen63fxqzERkKp6ucDr0HTNcjJrGZvsu1W3Y3KcHj0AGa8ao4KVkd0JTtc8ourMRNssYVjY5zsO4D23HFZUV5fWcxWQtuHT5gT19icV7zfaakluWFkhBGdxHA9MDOa8p1p49PhkkW02c5O05J/76/pSdEuNZPQ6uwvWmjRpR5hA53nJH41biMD5SEqSpyVPHXtgf415dpHia3mfyYIPJz1YMOD7g8A/Su9h1iNgInnWTH96Rdw/LNZvTRiaOhY3NvGXhQomMcbf55zXY+CLmd9Sg4KZ7g5z+Irzldc0+NgId0znjBZiP0BFen+CtUN1qEZiVYsdTg9fxArqwlnVic2IuqbPsTwpBdiDzHhMUbfxEjc34dh9ea7ivPvCchlAWSYOV/hXJ/E+leg191T+E+QnuFFFFaEhRRRQAUUUUAFFFFABRRRQB//0v1SooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACoZoxIuKmooBHnviawtorKWaUc9vrXzn4g3xK46s2Tg4HPv3r601ezjuIw0kfmFfuj0z/WvnbxhpgSZlZPlYfd/wAa+Zzmm17yPcy2or2Z4DpVxGdUeyaWISMdwVVzge7cAfmc1ozXN5pF6YjIyK2MED5SPqKzpNIgsdZW4s1CEn5mAzj+f6YrotTto71BJICCvdh/9fvXz0oNw03R7vMlLyZ0ejeKnKm01FswNwGZgMH8eo9a8e+JHgCHURJdQJ9ps5X3hocmSGTu8Z7/AExzW1dWk0CEwKsnfHXP4VvaV4guLVRbztuTHzRvjb9ARjFaUsW2lGp06h7LlblDr0PirxF4VaVjDqFmNRU8CaEgOf8AejcrtP0JFYeneHLTThjStCnaUnBeYxxqD6El+BX6Fz2fgjVjuvbUI0hxnaCD+IGKuwfDrwJMyz2llE7j5i5Ab68dM1301zLlg9Oyb/I55ezT5pJ/cfDHgnwlqw8Z6d4jvmR/s0yuwj+4o6ABj1xnsOa/UXQZhNYxyDuq9K+XfFOmm21QfZbdUSE4wvAAH06/hX1R4aiEGj2qydREufrjn8q9LK6rnUd1a2hzZrCMaUXHqdQvMYx1xWBrOfL2ryK0ROVYqvPFZl5i4jIfo1e1PVHiQVnc8uuIZftrOQSAQM/WszxbdTQaTK6DjB5HOPpXX6jGEO7bjHX1rhtdt5L3TJIZG+RyowBggE/0/WvMrxtFpHpUmnOLZ8P+IPDM+satcy7uGcbRz1x2NcrdfDrUopvtjC4t3QfJMib1Ge5wMlfUYr6DsNLkXxHdaXKdht5sgHjj6fka+j9HtojpyNfxgov3tuDn9OteZSqTb5EexiHBK7Vz8210XVArWGo21vqHP31doz+TIDW5oXw91a6n8vRtOW2aT+PmVh7rwoB9zn6V+gV1Po1smYbVbl1ySCgDAE8HGMn69qoPr1vbWYWygigYnnK7frjI5NYVJ04XX+f5bGMUpaqL/A8v8B/Dix8JL9t1cqkmd7b23M7gdXbufQDAHau68S/EG28kWOlkxbcgnzC2c9RtX5fzNc/qcySsJ76XMg6AuCPqQv8AKsBIbSJPPt4DcEgYbb8uT2BbmvOq42esYM6VQi2pTIb7Ubq5jEkvzSDuw5x7AZ/MmptEtY7MC4chmc5Cqfm/EVQFvd3VyxkOxF6jAOOenr+dWr+5i06MzK6wxqPvTBgTn0xWNODfvMc5fZR15mSYGWdTx93c24fp0/OuQ16zt5o2a8yARxiQjj22jj8TVDS/EUd4+UkQM2MZUnOO4yMYrpp4ru8GQx5HHOAffAwRXbGSasc3K4s8UvdDWc+dZsIgTnEyqc+nzAg8/SrVho2oqMm3THTOQBj1BzXpF3pdykS+c25vYlwB6kHmmSzW9vAkUywXB/uSRl+fbjioku5upvoZljoMi7ZLqTYAMBQRn9OtezeCrZ7e5TycjaO/615rpkFu0vmrbR2xY56dvrgY/HNe3eG1MKggbzj1H9K3wME6qZy4ubUGj27wvrrC+WzVERRj7uMlvfuTXs6klQTXiHhKWOG7yyrvbAyw5H+78pGT6mvbYyCgI6e9faUG2tT5SokmPooorczCiiigAooooAKKKKACiiigD//T/VKiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKAI5V3IR3Irxjxnp0bltmWYjOa9rIzXMaxpC3VvI7tl3HX0HtXn5jh3Vp2R14SryTufEes2Est2FDnZnpjjjtUes6jp1hCiXVwVAHOR0Hpn/AArvPF2kz2F08XK9SCK8E8RW8kiMJGCDoDyTkemOc/TH1r4uUnFOJ9ZTSnaR0X9oW0A3xuXB5UxnqPfioIb61uX25ckHJG0nv2JAFcTp93qj2eLqFo4Y/uyNjc/bkc/1+tdboiRXimSAYQfebG0k+xwM1xuMuax06JXOu0/THusRwOAp6BnzjPsB1H1r2Xw94afSrUyR8u464JH4lu35D2rM8F6fYReXIRMzOMBucYPuGIr2eCzjZf3mTgY5Oa+qyzL0o+0lueDj8a78i2PLJ/Dh1K4V7lck8qMdh3P1Pau8hiSIR2yfKFHStmeFI4zjgjpXOfaAt2hA7f5FezSoRpXa6nnTrSq2T6G5thjQ4A9v6VkXTxmPeuOmc/Wpb2VlXCYySOPY1iXNztUkEHcfm7Be38q0lImEOpyWrSckAkfz/nXN+V9ttDEVww5/Kn6/eiPLyHksOO+O2fQVT0557q4SS3kGwfe759sDv9K85zvOx6Cg1C50vh/wJp9tC018guJ3eRw7gbgJCCV3d8dB3xW7NoqxM4t1+VhjB4+tb1ioSNS2V6fnWuio3TJ+tdP1eFrJHE8RO+rPD77w9eQXHmQLuLD+LnPGM4PPHf1rzzU4VhkxeReU6EjLIXUjPHfAz9BX1NeRRsCEC5HUEc15D4nlTzNlzbouPlEgYbwPUgbTj868LHZfGKcos9XCY1ydmjwm/klhQzRQMwOdoSNdpwepPWuZfxDdTstsYwijqCPmXHoWIH4ium1qaztGB84sjEgHC5/DKgiuTuLVp22wRGUdc7tpGeeg6/nXz86bjLU9hSTRv/bJbaEH7SuOuFAc59toGK8+11b7VJTPCAXTplXf8SckA/pXW2WmXBA89iR/dXgjPsf/AK9dFaaQQplSPJHOR8vH4E/pXRTTMHJJniNvdahYF47mEKc8tna36Cu60vWdRt1T908i5xw+7j24/pXeTabHIux7VAPVuSPwbIrKXwuZXzEvkr0ygUNj/eHP60Sg90UqkXuE1ydQVAeQTn/aX8QMit/TdJgVFe7m3dcCT5m/PH9auaT4TtbZ/PJdjnJJxn8a6wW6uBHERj0OK1hSk9zGdVLSJmWemws4+zKeec7q9B0WEwyKjgD6c1iWsFvEvzIQ3+fStvTnKSBuQB2r1MJTUZJnnYio5Kx32m3z2eoKs0m1X4AGBn8e34V7/p063NokqABSOMcivmWZBI8cx6gjI45r3nwzqUl1ZxiX5R0HGBx2HTOPYYHrX0OHlq0zxa0TrqKKK7DmCiiigAooooAKKKKACiiigD//1P1SooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACijpyaj81T9z5vpzQBJUcqb1Ipu6Y/dUL9T/Qf40hSYj5pAPoP8c0mNHkfjnw1LcQNdRgNIK+atR8OW89yJZ0JYnlR/WvuG6to54yjSySZ6gf/AFhXinijQGs7g3UUDbepJC8fzr5nMcvjGftY7dT3MDjXy+zZ4M+jozYkzsThU7n8O31NWrewRZFaCIjtjOE/kK71zZcyrD5hPoq8/iQKz3vNJhP/ACD8sDnIfbz9Oc/lXN9Xho7nV7eW1juvCumrHbiWYgluuzAH6V6IsqRKB0FefeFrp75SII5ERf7xXA/EAGu5XMQEcq7s+2a+jwyXIuU8bENubuWpgWTPSuLnZINRVXYbSCfxNdSZxKWReNvWvDPiJ4hufD2rRRDG25GYy3+z1FViKihHmZpgqDq1PZxPQr3UIWYDdgk8DPXFYl5eQxQku4AH3vXjjrXk58cGQZlGGTpj1PWuW1nxhcyxeXEcJzwOp/8ArV588bHc92nk9XRMteJ9Ua71FDbEKm4/e4HzDGTz/OvUvCdnYQRBrYqWflmBzyB1z/gK+LvEHxEnSX7LprxXN2x4UfPtwOWOz09K+qfgzb3Nv4UtW1An7RIN77jk5IyefXP0x0rDDTbnfuLMKPs6fL2PdI2JQZ4HWq8+rR28ywMj/OeMLkfy4qit+8kzQwI+U6tt4P0J61JFNrRkYzIqx/wkcnHvnvXq8x4HL3NJ7pGXjPPrXKaxBbX8DxnI4wGTBI9xnNWjpl3fD/Trggbs7UAXIznHU1g63Y6nEc6dcYRR9w/Ln8a5MQ3yu6NqSSejPNNW0vVbJ3CXTXELdFZeMehU8Vy7WkjPtNnsHfAwPy5FdPc3lyNxuZYgwJ48xzz68cViGYufkeDPfnP8xXzdRRvorHrxlKwyKwES5ZGAPOMFR+gpzTKM+VG4PuDUkMNzMR+8gAGeyd/wrZt9NuOu1MjnKhMH8qhR7Dcu5nW2JF/eRjjqc8/rWvHDC46KfXjB/MVOsMqDcVwfoMfhUbPMGG5VQ9N23GfrxWsVYhyuNKjA8uQg+mOv5VIkQdMrnI74qSOIOf3hGT3FX1gfdlX/AMa2jEzlIzhJIMCVmA+hre02SEMOSfqaqbvL+WQBj7Zq3Es24OB8p9RXVSVncxm7o7Zo1nth0yBXongeDAy6KCeN2WLN7AHov0A+teb6ZcsqeXMCVP6V3Hh/XrfSrr7LP9x/utnHJ7EngfXrXs0mrqTPNqReqR7TRVe2m86MOOnr0z9ParFeicIUUUUAFFFFABRRRQAUUUUAf//V/VKiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigBCQKQ7j04p2AKKAGeWv8AF831pdwHAGfpS7c9aWgYzDt32/qaaIh1PJ9+alqBpiWMcI3MOueg+v8AhSduoIkJVFyxAArC1a1+327xJGcMOrfKP6k/lW0kIB3uS7ep7fQdqSSUHcqY+Xqx6ConHmVpFRlZ3R88y6Z/Z9xJB5atzxkkj8uKoTWtwp3RmKL0wij9SP8AGvTNd0txIblUwGPU8Mf8B7VxU8e0kdTXluioLlPSVXm1JdCe+iRlMoK+u3H+Gfyrak1S7hUgRbj6+v4Vx7PcQsohJDdgP5nsBV6LUruMbCPNbHJHU1rTq2XKROF3c3v7dBYpNCyYGc//AKq8k+Jnhm08Z6dhXMdxFny2BIK56ke/pXXXPibYVEsDBDxwOCa53VNesJi1ugdGQZYnj/PH4Ac1OIqRlBps1wynCanFHxmkuteGrWbStWleaS2P7uST7zJnGCe+PWuR1zX5NX0maxWUxmQbXKnBI7j8a978bJo2twrFJukaQlUYAjb7k8cCvmiLTLDS/FUcV7KZLZmJG7uU5BNeGopyufZUcdenaS1PUfh18ONNtbJLqeIok+Nx7EA5x6g5HXNfWelXrxR+d5LCNR0UAuw/+vXgtprUjuHtYWKxxqVGVCtnnp0H6V3Np4yvliRI4DuYgbEC4Hbk5/lWlOryy5pHi4pSqHqsfiDxDdxILHTtqscFpHwVA9VI5NWZW8Ru4InWNOpHUflXmv8AwlHiKdlaxljhA+8r8tj2we1XHvtZuSv22+2oRjKgEc9+1dX1uLR57oNPZHoBXVJjie9TylGdqjaSfqSax7m80u0Yu11uc/7W7n0xxXFTwWkWTd3TTKnUhiSPwrOub2NxsSITQcfP/EPr3/Aiuari/IuNE0bu4srqfM8Ko3Y7SufzK/oTWfPZ2G7OGjPX723j/gagf+PUkM9vGuxHMat64Cn+aH8QKtq6oAjDaGPBX5Mn6cofwxXA5c250JWIY7SWPDRSfIe7pkf99LuFaEcN1gFQHHrG2f5H+lLDZlPmhOc/3fkcfh0P4GtJGdMNJhwOpxhh9e4qoxFKRVW08wgSnBPcirrWpiTdksvcZ/l2NTfasx/u2ORyVb5hVJrlFb5Djn8PyrZKKM7tiHaBmL8j2/GhAr8kEZPQ9KWUoy+bGwYfTkexqh9p2/Iw3c8EcY/DpTvYRr+fEo8tMbvftWlZOXx5nJrnhH5uGKhsfhVqK4aNsFCuPxropy1uZTid5BIQAQoWr88giEd5tG5SDnk59q5OzvpZBtAb8Oa7CyMVxAYXZskcgivTpy5lY45qz1PbPCurrq2nrIiFNvB9Pw/wrqK8I8H6kdH1c2LysI5zgf3dw6ce9e6qwdQw716NCfNHXc4asLS0HUUUVsZBRRRQAUUUUAFFFFAH/9b9UqKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAoopCM8HpQBGwaQ7VO1e57n6U9VVFCqMAU6msSTtH40hkcjFsgHao6n+gqP5VUPJ8qj7q/wD1vWkZgSoQbsfdUd/c+1DqsSm4mOWHf69lHvUsoo34M0ewqSW6IPvH6nsK8d1YixuDEcb888cD6V7OEMcL3F1+7BGW57emf85rw7xfqUElypX5MHoPT0rixklGPMzqwsXKXKilLOoTbGMs361kTXT2wKxncT99vX2HsP1NSefuGd2MelUZW5yOQK8+dRvY7lCxmaneX7jELhdvLHGT/uj+tecarquqQOZHUMmCPlHJrvrmRzgY9cdhz3rFmjjdSjLkH2rgrTk3udVKy6HgOsz6zdwzSRQiJpF2lu+D2UV51c+CdQeaK5vXDXEozjrjjpn1GOfSvqqXSo5c7F5Jqj/Z0MDtLMu9lGBx0H90VnGdjrjVPnHSdG1y3uJEs5CXBCqh+6wxlRz6811VrJrke6aIjkD5WH+fpXoj6couwuBkLuz/ALdTNZJNllGOc8+//wBfrUTqXLc0zjYZdaxHM6gB+GHQ5z1/KurspblY1jP3R69j6/SrkcDQoUl+7nH0I6VnXOqRwqQq4J4weP1FYuXYzbNln8va4b5W6Z52nuvuPT2qi17bl2MJ8uSPqB/Me1csNVe7kZW/1UgGO+G7H8DTLUXPnb3Pzp07/h/ntUO5J18U8c5ZWG3d1I6fXH+Fa1t5tuQmeG5x1U4rmoy6tvg+63+SK1ILiYy7GGOMAduKcUJs66O4idv+eZHUdVIPXg1YaaSEgSZZBx6jHseufxrmnRjgqSCR19D7+1XIZpyOchuOR7eorVSZm0XC+9iYGzjt/hUWS3z55HFJEke7cp2P1xVpBC52PhvQjgj/ABpqIFZXYNhBtJ64rShthLgnj3Ax+YpVsWjA3nK9iCCKvoiqo2sHz+YreEO5nKXYlhthF941rxW3ngDBOe9QWtszYOc/pW3BDsIHeu+lA5akhINOSP7oPHatKImDDBSv41LGCRgnP1qfZhcHIFd0YpbHNJ33I5EimdLnJDKeq9a928P6gb2wjMhJbHUjGR9eQf8AOQK8LQRjMY6n1PFeg+C9QVZWsmXY3UAHr749fXH4iuik7S9TCqrxPUaKB0orsOUKKKKACiiigAooooA//9f9UqKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooADVWZ2yIIxl25PoB6mppH2DIGSeAPU01EESlnPzNyx/z2pPsNDo4xGOuSep9apGWMyNczsFjiyFz0z3P9B+Nc34k8WW+jQMF3PIeBhTx6nPtXzP4s+J+r3ebG1l8kvgKAv3B+fWvPxeYUaCtI7cNgqlbVHuHirxnaws0TS7Y4xnaCBk9s/4V8+3WsNrGoNdE/JuwB1yfXA615Fd6xf3twz+Y+1iAoJC+wzk9+pP1rq9Gv4XQOju0ScEqMZ+nTOf0/Svl8Vmcq8lfRH0GHwCox8z1uwjijjEW1pJX+bLdAB3OeM/n9KvtFztUdB26fmawdF+23mREggSQ7nLdWC8AZPQevb0Ga63y1jwu7e3oBgf48V6mH96CaOKtpIwJLPqcVnS2y8jH5V1jx7ztH5VDJaYHA5q5UbmSmcXLAVXEYAz3NZ0tqSp3HiuzltEU5K7jWNdwuynoormnRsaxqHE3SxRnccZFQx27SKZApAPTtWs9mJZxHGC5PftXQLphWIBuPSsI0WzV1LHBzwHaQe45/wAa4XWIhGA5OdxIA+nWvXr2w2qSDxXnGuWbSTC1jYKz4LH+6o6/nWcoa2LjK5SstKJhQ4wT0+lbdvpJIG0YwME/TpXY6fou+OOVRhWAAz6f/XrSmsBbHbjKk801h3a7JdRbHGafppTNu68A5H49a05rEK6PGvCnk100lhmQTJxjg4rQSyQxgN1atY0OhDqnLpY+egHQ9anEAtyEmQD0btW7Havb53DoP61PJapdQYPKtVqiS6hmNYQ3CAgDNKNFglXLEq3qKnhtLmzwo+ZfQ1rxM7KGIrWNNPdESk+jMAaVJBxFcZHcEVehsn3AsM+4wK3YlDkbhiriWidfu1tGguhlKozPigRCOTWjGoKgKc49RUiRFM7SM1MmBw4x9OK6YRsZNlmGMkDB5q0Y5FXpuHoDg1HDk/dbcP1q6FLDDHB/KuiK0MJGU4YHK7gB2atTTpFjuY2mJUZGGB5U9iKhljfHzZYeoNQxMw4PGOmapB0PfrKfzoVJIY46qeKuVz3h68S5skyVLAAAqc8fjyPpXQ13Rd1c4mrMKKKKYgooooAKKKKAP//Q/VKiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooqrcXlvaoXmcKB6nFJtLcaTexapGZUUs5wB1JrzHXviZpOkq3luGI9AWyfoBXh+vfGnVb9HisIQir13KT+YJrzMTm+Go6OV35HfQy2vV2VkfS2o+KNI01Gmup1UqOBkcD/GvD/FXxngBez04cgffJ4H0x1rwG/8Q6lq05uLoZ2jOMlAT24H51nwuCw/djnPIyR6/ebNfOYvP6tT3aWiPcw+TU4e9U1Oj1DXr3UZPt158yIM4PUntgH8K465u2kEr3DyFnYqNuOvU+/tV6a5txH5kqlt5zjPYdPzzmsLUb22gIjSInYdoA4AY9T6V406jk9WepCCWyM2Qrcb1hBfGVXryz9Tx7Aiuo0aWZFS3x0+6vRj7nsBXMKzGzVSAqszMwHUjA6nsK3vD7xQzM0Z3SMcZHRe2fr6VKKlsexaLqUwQ2cIIUY3lfvu3pu7D6c/SuutBc3WSHCx5424xj0XH6nmuDsLJUaMNlA3ylR1Oe3HPPf19a9HilihiSCABCgxjoEHfJHf/wDUK+iwMpNWm9jxsUkneK3NeC1VFAxz70+SFB93k+tEO5UAf5e54/n7+1TMwAGOvpXuRSseTK9zGnthjJ61iz2bSfKB/hXViAk7n6mq9wiouAMfzrOVJPcqM7HNW1gsR+Ufj61cls3K8DNbNtbNjzHH4VNLEzdeB6UKirA6mpwl7p/loWOXduFUdz7VwmtaUtnCsAG+5u3VCfQE817Q1sFDXMvHYZ7CuYi00XetRTSDhDkVzVMOrqxtTq23NK3sFht1UjGwAfSm3tsiWzDGWxxn1rqGhVR83T0rPmtvOJZunat5UrKyMVUu7mJBZgICef8AOaYkQW4eMjiM5B9mreWDbGo6Fev0FUJUAnbjnA/WodOyRalcrMmWw2BtP5ioo7dYgVHCnt71flRmUnutKIgycjg8fSly6hfQorHuO09quJDGORz9KcIjuJPBFTLEcF1OR3FUoktiLEpGTz9P60pG0ZGRj8aa4OcZwaZuZDg8eo7VexJZQ7scA4q6qJ3z+FU4wfoTV2OQ4wwx+taxfciRMiRjBP5irOExnPHrUajseM+lSHK+/wChrVGbK7oDnaA30ODVOMAOQDz6E1bkIIJBGfcVm/vDKMqM9ip/pSbGj0Dwrqht5vs8oAQnB3Njk9COMc9DzzXqgOa8a8PJMmoBWyC/r0b2I6H/ADg17BAgSIKo2jsPT2rqot21OaqknoS0UUVsZBRRRQAUUUUAf//R/VKiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACmPJHEpeRgoHUmsfWdfsNEt2nu3AwOB618x+Nfirf6p5lpYr5MI7jPzV5mOzWjhlZ6y7HdhMBUrvTbuexeKfijoOjBreKYSSDrt5r5o8SfEnXtcmkj092WIepwfy61xrW97cM11cJ52exPr7mswxmGfYgRPZTz+OetfHYzNcRX0k7LyPqMLltGjtqy9CLy5YyXMjeZ6nJ+mBmnXFpjJkDFR0y/PT0q7DGJkxg78YzuwPxwavwrbRZV9uS3Uct/XFeeo6bna3ZnN+RJHEAm2IH5mySWPp+lEFsDG0mNxGR3Ix3+g9TW3L++y4jOGJ5f07Yz/hVKSM7WEzgonqcjI9ccfhSUNbj5uhRnd2VQo2+WvRMcn/erjbrbkzSssarkFiSx9Sff866m7lcIdiMcqAM/KB6/h+Nci0aSF2275cfKF5Vcep6D6CmwSHGI3Wz+GCMk+WcjHTBf1b0BNXtIuo0kIU7Qp4749/rnp7/SqcqPgvIwVd3zFccsewx1J/SqVr5oljjjb5Q2QvU8dzn/AD6VSYmj6S0ScR2P2ll/ePwOcnGOpP8An8a2NMvLqWQRpgKDkDHft16n0rgNAvWuLVUkwiqOSTknPcn0z1I+grprS6hhm+0EkAdz94A/3ey7vXsPevWw9bWOuh5tWlvoekfaGgAhyWYdhzg/zJ7f54uRkx7WuPvt91B/WuMgvpDulXCJEMsx+4mexPVm9h+PpXRWAPmYQEuBlyx59h7Dua96jX5tjyqtKy1N7qOeT3pIrXzH8yUfQU63AlYqhyBWmgUERiu+KvqcUtCMQoo6VHJEuNxFaOwd6gkTNaNGdzBuIjJy3QdB70y1sSkwbHOOa2vIy3ParEaAAn1qFBN3K57KyMtoNxIIzWfcMY8ZHFdDInU/lWPcW/mgpnjOamcdNBxfcz92XaMDoM59qaLPcd/8Qq8lu6Mrt34qcJj5ug71nyX3L5uxkSRkZGODUcce1ijDjofoRxWsyK/H51D5Q6N1X+lLlHzFFojkEdB0P9DUghIO5O/UVaVMNg9DzUwXAx6UKInIymjRhtXgjsf6VEsJU5XtWmYBISfTv7VKIVPXgjvRyXDmKMZJGCuQaspCD8yHj0zzT/JZCQg+o/qKN5X5iMr3x1qkrbibvsOVWA55H608uQNrDingq+Np696hfcowfzFWQU5WUg7QRj/PSswXLb9u5XA9cZFWrqcpk7d304P+FZTyQu4mkj2jPBweP6j+VZSlqaxR6P4c+d0aRsoCDkE8duR6V69GNqgV4v4entjMgDbS3Gc8H+ley2+fKXJzxXfRehx1tyeiiitjEKKKKACiiigD/9L9UqKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAriPFPjjSvDsRRpFknP8ACG+79az/AB34zg0KxeC3IaZhj2H/ANevknVNTv8AV7kM6Ha3TjP4/WvnM2zr2T9jQ1fV9j2cuyz2v7ypsdF4m8TXviC4eZWwh6Dt/wDXrjxbys3zISAeDj/CplgW3JaWRiwxxn1/lTvtZjbaIjyPvHpXyE5OUuab1Pp6cVFcsEV57G4kQebnBzgMP554FUzYK75j4Zf8/Sqt/qEpl8rpgdeh/PkVY09S0XmeTuychiMk596zTTexrZpFlNMypZnMzr/tAD/CpY7GdJDIWWLGcbckir6Rkk7js9hjGPqxwKoXUyBfLRTI2T1fj9MA1o4pamd29ChlhON++bHO4kk59AOBUTshnUC3Z5emAMbR7iqa3Lgnc6x5PVMbhj/PrWvbIJyWZHbsC7bQfXnv+FELNFS0Od1Zd8YEo8sHnbgMf0P86wigUrDAC2QNzOcAH0CL1/H9a7DUbezjzJIFGOOrHHscn9BiuNuJZ7htkA8iJBgt32k9AB0z6Dk9zS2Y1sZjxSzXAi8whE3fe5b3OewH/wBarVrttr8IyiQt8scXJLMO745Cjr7/AErRKLbJvVQjYBO/kADpn1PsKyrGSRNQ807laRtpZx8/94sfQkc+1VEJHeWU7wzNHcAeYTlx1yMcL6D3rqbebzWMrD92gLqnZn/vH2WuMkASdZB845Az0Yn+fuTXRwXWYE89tmOScE7iOnHoOo7cVUJO5lJaHc6VK9xNClx8zDLIpwPm7yOOw/u/4129k32pdlkc2zH5nHWQ57e1eTvfbx/Z8LFJrsjz5By6oeFQH+8Rx+fpmvXdJdbYx21suNgCDP3UVcfmfU9M8V9Bl81LQ8jGQtqdZGv2ZBbxD5z+g9T71owxqg45J6msuGRTIVjJxnk9yR1P/wBatJJB/qxwF619DCx4syxj0pCnc0BxjI79KQuMhRWtzKxGUydtEhCY9qsDHWqsnzHPpSY0MkJxjvVPBaQ46VeAHSmiPBz61L1GUGySB+NI3y+4JyKshVZmI7GoJOAcfwmoKKO3DFenWnkg49e5/lTXDMSV+opURs4btUFDcgkL0owQ3XkfqKkeIHGOpJpwUN97jAp2AIwN3y8VIwXqOD6UnTgilLbxtxzVIlkTqGHHIH5g1ErqSQetSHI5H5VVkAchkJ3e3X8qljQ+RAvK8A1SkeSPljkepqXzWYfKfriovm+8mSO+P6j/AAqG+xaRVkYMpypx1yvI/KsiSIEF4JDGTnjPH5VsMARlBjPpx+orOvFmEZZDz9A2azki4mjpc6uO6uvccV79o00stojO24FQQSMH/wCvXzNpl4qzBWbBY7fYEfWvonwnPHPpMZQ/MvDD0NdeEncwxMbK509FFFdpxBRRRQAUUUUAf//T/VKiiigAooooAKKKKACiiigAooooAKKKKACiiigArl/FHiW18O2LXEzLuI+UE9/pV3XddsNBspLu+lEYVSR3r4k8W+L7zxRqcmCWTPAJ7dif8K8POM0WHj7On8T/AAPUy3L3XlzS+FEniPxJLr109xLymTySAB9KwYb4sdiJkdyeuPxrO4+UzsPb0AHXA7D3rTgMKxh1xjj5cZz747/yr4fmbd5M+uUFFWS0NJo5powbceXGvBYABR+I4qg2mRiPzbiVkx1zgZ/rV5LiV8NyFQ98YH4dKytSvk25D7snqSBgfjVSUbXFHm2Mq5Fqv3jnHTnH8+taMDGYBidyjgbnIT8h/wDXrDQwSXSKxDhjwqLliP8APvXbWUFrDs3Y2ngLgM3P0JwKmmm2XN2QsEdsIt7RcjPJG0cHsCSf0rEvruMErtVQT/EefyP+FdzKhEe23iRFxwAOBn/PfmuZvbJlTaQuWPoP51rVg0rIypzu9TlLVfOuBIFRu67hwM988fov411SDYE2yKzN1252jHv3/OobXSbYSEySqMH5gMH9MY/PFbUMUJPmwBpPSV+B+A6fgOKdOOmo6kjltTtB5LlW+Zz8zHrjrxx0964wKHukhtznbk7VGW56sx7fj+Ar03VdPlnQuANgPLkkk8flXmV/JLF/olsdqEktjgE+5HJ/PjvWdRWlsXTd0VrphGCARLMCQBnhTkAe3A6CseeNpr6HyiW28swJwf8AZHY5PU/StG1s4mVsjOB8zDgENyQB2z3PJIqxIuJFRywOcKFBOF9SP0AJGT146i8hs6mzXcE+0Zc4HA6D/ZHrnuatCOe5lM0mdxbYBngEckfQDg/lU0I2ugUbEQBVycnheWPvzz6Us8saiOyQ/O2AAOe+efbPp1qktLGXU1fDiifUUnIyuSy+vPGT6E9B7fWvX4LlZHaK0O1Y8B5QMgHqFT1xnJ9WxXjenLHA0kMb7dp3O2eSPqOnp711theTW0W24OyMMSFHGWOcZ+g5Nelg63s9GcmJpc+p65pu5VMmMKq8kc4Hb6k960IrhFQb/lJ5I/z6VzVpfxiyRJX3STp5jgcEID8qgds55+uK2bdSz/apgFQkdeOnT8B1/D2r6alUTirHhVIWbubMUkjAyuNuecHsKsJwCx7Vh296t3KJfuxgblH+yP4z/T86sS38aYij+Z2BYD8M7j7V0KorXMJQd7GtvBJx3/pTJHVBj0qhbykRbmOWbGCe59celSRFXdpCc5OBVc1yHEshiy7iKfIdv4YpjSBRgdqqtKWUu3fHHtTuKw1jsVmH8RqIyDbgDkkflRPKqw575/Wos8YIxgVDZSRYhVc8c44qVo8AnHSooSFBPbBP5VYLDaSeOtUiWVyAOvSmOB1B5pk0yquCeCKqeeSwH8VS5IpJlsPkevtTCw/h6e/amFgenDVF5nzfz/8A1UmwsK7lTkDj+VQlwwJHOKSSZVqjLh8unbrt6j6iolItInlUsNykn19RVYybcEHPcEcYoVpWG+JskenWk3LMN5O1uhPSouUkKbnzFOQC3TPr9ayZpwARICh/T65qe5SWP5miyR3H/wBbism4u4dm2XKqffB/A1nOdty4x7Ebv5c4fuec9z9cdf1r3fwCUezMgIycjKk/kR0+hr5ze4jZtsE2/oQOjcfoa95+Gt2TC8Hl43c54z/n+VaYOd52JxMfcPWaKKK9c8sKKKKACiiigD//1P1SooooAKKKKACiiigAooooAKKKKACiiigAqC4nit4WlncRooyWJwAPXNPlljgjaWVgqKMknoBXyX8V/iDJeTS2FlKWt0yFUHhj0zgYz+NedmWYwwlPmereyOzBYOWInyrY5v4neOBq+rHTLKTz4g3XkL9fevN4woXbjbzz/tH3/wAKq6ZZTXI+07QsshPzOedvfA6DFakgit8kMGIwAwHA9ev86/PalSVSbqT6n2tKlGnFQj0GCJ9yMo+6OAe/19vatCB7VWMkwZwOw6k/lwPQVyj3xhk2QqZJJOdzDP8AOtC2v7mOEtNJub+6nQfjjr9BUQavoaSTOlaXz0xEghTHHGW/M9K5S83zTPFb89ieBgd8kZ4/WrzzXSxia+bYp+7CAN7E+pPT271mRaq8JVZrf5c8RoAQfXPBzj15rSWpES9p+m20lwH2+aSPuoSB6dckmvQ9Mso7cgGOG0D/AHfl8yQ/QAnmud0m5M/zSM1onoACcegxgA/rXbabKxO63ilKk/eLIg9TgjJP511Yanqn/X+ZhXm7W/r/ACNKa1WO3BkWWRSON+I1/pXFX6Pknaqgduo/Wu1lBmLeWgZmxjLF2AHuK5fUrEoSZsDnnHHP61ri46aGeHlrqYUUzg+QoQ4+YAYAHueBWxHIk8mLu4ds9ewx6ADt+nvVKKFUJWEcn/ZJx79q1rOxiE4e6GXYbiM5OB3bsK5qSkb1LFTVI0uItqgsgIwvb0/z3NcLqGmQq77tzseQTwAOnAHH9PavWpRCwAhiDLjA4647nP8A9asOezklLfZxg4JkfPIA9D/hVVqV3cmlVsrHlaW00CyNCBu/vMcbT7Z/njNVrKB5J5BayMzhvmKjJ49z+lbuqWy6a73DuDkdevHooH6mjTFk+zmRotkLnJA6yE92Pb2Fc67M3fcmNvLDbRPM/QYYZJGeu0HuP7x78Uy5doCGU4lGW47DtVq7nSNVjOJjGOeyj0VR9fzrN02P7Y7ySKNjOxc9SQvbd6MevsMCmmr2Ia0udHo1ukUQ8+TdJI4bHUe31J/zxmuxihju5P3TBtuQMnhc8lm9yBxXBpI0c+wOB/IZxk59cdvoPWvSNJsiLMQvkCQ5+bjK/wB5v89K7aGr5bHNW0XMRWrTxyrISSJGDFh1KqcKPZQefeukuNd88nI/chgqqT97bxk+2f0FU7wWywkwg4cdT7cAH+eKyEjDPukBYrztx0GOPpnrnsK7FUqU/cizm5Yz95o7aO/URGGNd807ZIPAA64+n9Kj04ZRmllLKxO+TvIerH2UdB7VgfvUiAwFONueg3HqPwHWte2XESoH4GAM8c+p/wAK7oVnJq/Q5J00k7HSi6Z9qRL878Aeg/8ArD+tX/OSJc8bYzx7k9K5kSrAi4yA2eT1IHU/jWfJc3F5KinKjeWx+GBmuv6zy77nN7C5082oqkIkByd2wf7TDqfoDViGYuxAPAA59BjNc0sMl1KigYjiLfQAdTWm7MEkVeDJgfRT1P5YrWFVvVmcqaWiJSzzSqDwuQfwq68iqxz0HP4AVSZuQg4IHP4/4Uk0qDnOdxx+FUpWJcbloXBS2dm6rj8z1p3nko4Y/d4/Mf41itdJG5VjlTgAevc1QnvjuMinK4Ofp7VPt0h+yOiYLNGI88/4io4gUYbhlen0rDg1RAEd2wcfnjity1uoZVAyCcU41Iy2FKDiWZBwADn0P0qpJMSwDdexp8kioD3U/wA/8ayppt5AUg81cpWJjEkeQsenTt/WjGRkcEfp+NSqoIy3U9D6fjUUjPEQ2M1HmV5AAW5Iww7jjNRyMXyG4cd+n59jUu+J1DJ39DVaZ2C7vvr0NDGZ1xNMg2n5R6jlf8/jWJPdW8wZZGBK9z8w/wCBd1+tatw3lbp4MOvcY6ewwePxrjtSuEu13wKFk5zkd/8AeGCP+BCuOtOx0U43K93LHDMqsuwnldp3D8j/AEr6H+FZZ4WZSNuM7SORnuPY18nbszCPJGD91s5H4HtX2D8MbM2+mJIVChlA4ORnr07GtstvKdyMdpCx6vRRRXvnihRRRQAUUUUAf//V/VKiiigAooooAKKKKACiiigAooooAKKCcVRv7pLW0knkfywo69efYetTKSim2NK7scD8RvEMdhpU2nwMDcOuSD0Vff3PavjO5DXV4XX5znJY9hXpnj3X4ry+Nra52gksWOWJ75rgIdyoEjPLHJ4r88zPFvE13J7I+yy7DKjS82CWowfMJ257jlj7+w9KwtVmVmJVuE4OBhR/ia3p51C+WDhh6HoP5Cse8SPYrL0XgADA/Adz715stdD0Y9zjUju7qbYVCKD0IOT9ea3XkMMaRI5WV+gXBx74AGTUsoeCNSx3buoX5QPfn+v5VRub2O32x26s8jDO8dh+X60r9EUacFooQSzRH/enfGD6n39qvqoK/Iy7W7r8xI/3R/U1kxzmFUZ4cZxt3AFvqAentnFbNlZXZ+afJMnO3IwB+FapENm7psd4oJVdg7Fhlj7egruNPg1FmWS8xEpIOXbIwPpXKW0H2cDdIqEH7qDcfxxwK63TnuQd22QDozuy5Gen3uB+Wa9DC6OzOKu7rQ3J1iMZd23Bhnp1+ijgfrXJ3lsWJkVfl9TwOa6tp7KAMrXDyyMOcHIz/vAD9K5TUWmkIMabQejHr+Ga6cXa1zHD3uY2145MGTYf9kY/KtbelhAsYGHc5yRzk9yT39MA+1Yq5SQsGBxwWHcn0PJrV0uCdpfNQEksSZGOMY64znGPWuCje9kdlTa7NJIQGSa/YgMCQhPp/sjOPqcmor6RXi8uBfkBzycfi39B19avratNJznYeWfucehPb/P0r3youLaBguTgnr+tdU4NRZzRkrnj2vPtm2YUk8sTnHX27ewp1le3KhI3YiTaZArcEJ03t/dX09eg9m+JGto5z5fDZPJG5sZ4IXpz6msjTFAmkeSJnJKuEI3NIw4VpOzbf4F+6OpzXlq3Mz0Ps3NW6lR0UIScg/OeOvUgfyqjbyyXDP8AZgyQWwwenXPAPr6n8q27izmlZ57puGB4zjGTyc+tUYrR5JVghP7odQBgAdTj0HqepqVfmHpY7mzs0ljinRR8pAAPGT1J98dq6eyE16v2gMy2+7AOfmfb1I9F4x9K422lmkKE/LEq4H8PA6k/Xkn0AxXplu8cemx7U4kwEUdSn07Z4HP6V7OFpqVzy8RNozmlVsmZPkJHlr+mT9T+lTyXHkKhiXdJI2WLenqfYDp+FTSwxoXnuyAU6KP4fQD3Oaw55JHnyUwRwF9O5/8Ar1rNunuZx940EuY1uxO/711+SCMdAc9QP5k966S3eFBsOHlbsPfjk/h2/wAa4GO5a3L4UtI/DOf5fl2HA+tb2m3MjI0mcg/eYHBJ/ur6ADjPQCtMNXV7E1qTtc27hlEyoxDsxIZh047D2FRwARysWPKqWYn+HJ4z7k9vSssalHHKwTBAA3N/Cg7KvqavytnyrVx88rbio6/ifYdK6YyjJ3MHFrQ14JGkjwnCuRj/AHBx+bNmr7N5Nu079Wxj6DpWatyqcnBxgceo4AHsP6GsbVtWkkim+baiZC+2B1/PmuqVWMI3Zzqm5SsX5dR8uN5V+Y8n3JPA/rVFr9pGdiTj7qjucdT+NcpFeyXN3HDED5MYUMP9rbwPwGPzNT3Bk8wJFyvVj6+34nrXnvEuWx1exUdzVEsu8SEH584z+VSkqIGiA5zjHuetLaIfsyzzZ3McAn070WrRsdo5JYE/yNaRX4mTZQj0q5kYKx4Q5H9a2o7KW0G8E5U5rWhlh8tmHPWlkYzAKvrj866YUYxV0YyqN7kIlaVDnPv/AIirMcMUgEg6t1x3I7/WlRFi6jkdakJjHzIcA10xXcyfkTBABjOaqt8pKHlfTqRSGZlPTg96Z56FsP1HT2/Gq5kJJkDwgBnXp3wcj6/55qi0kUQ8wnA9c8VduZBGdwYr7j/PNc3qE8gRpvLKnu0XRh7jpWFSSjsaQTZNctA585Hw7dcAEke471zl/wCWqsWIViOu3bn1yOQa5DUtVgik2TqYXB4LLtBP1U4NZa6+88ZimQFccFCTx9Tz+BrzamJTdmdsKLRbeOOfUoCrDaXAODxn19q+5vBlsIdHhVWJwBkNg/qOtfn7p8xfXEJBeLIwUIJ//WK+/PAhm/sSIyAqCOAR6ccEZGP5V7GWdjhzDZHcUUUV7J5AUUUUAFFFFAH/1v1SooooAKKKKACiiigAooooAKKKKAEIryz4oayumaQEyFZ84OefoP6mvSr29ttPtmurt9kadT/Qe9fGfxM8Y3HiHUHkiXbBb7ggPQKO59zXh55jI0qDpp+8z1Mrwzq1VK2iOEleB55L68O4t1Hp6AemamF0ot3mlATrgf41gWazT7JSpkZ+V3HAX3NWblI2iEpzjooH8XqfpXwvMz6/l6Fa6kiWMSEjHXc3Cgn27j61k/aJLwiGDdIzY+bGT9R6/wBKrTNczXBd9rZ6L6A9zWpZ7IlYZIQk5K/JuPueTj2BqEaNFpbVyhjchWPqQ2B6kf8A6vpWPd3kiSeQn3vpzn1P/wBc1XuNXlMn2S0G1RyNi8D8f69agWOdtimRd7dEHX6k8CnzXBRtuT29zEj+Y4adv4mYjb+A/qa7S0lkMcRR/J3dY40zKfrkYH61woL2jGV5QwHIAAAX6Dp9c10ejSSXLtKx3I5GQqE59B1yfzxVQepM0el6SbVY96AKqcfvCSc9Dwvf8a6gSQvtdIBxwCRtUfRRnr71y1k1yUUpG0Y6buARj6A4/Ct5NsW4PMAw5bqzj8zgV61GVlY86qrs12u4o9yQqpYew4H5cfjXK6jeSysWk+c9ieg/PGalnvjErRxRsN38TEAnNc7cISxMzYZf4eWPPr1x/Os8TWclZFUaSTuyncPICPLJJJwPXn07/lXU6XblYjNdSZhTkgnClh2OOuPQcZ61z1rE7S7ijFh2/iOffnA+nNdTBHPdyCW+A8qPHA+WNR6Z749Kyw0Nb9TWvLSxoxzSTIbm5XZABiJO7HscfyzWReSs2ZAOTngdB7D29TWvc3Cld7H5UAHA7dgB6n8653UbyRUYY2MB84HVF7AnsT6Ct8RNJWuY0otu9jzfUfMF1IW+aQ/OPRewwO5+vStLS/IhjYsd7knJY9SOMKOpA9fWs+9jR75Zp8hA28qM/NjoD3wPzNbCRF5Nyx5eUABQOQnUZ9AOpFefHV3R2y2sJeM9zjaCyLj2FZxE4jMUL73c8gdFGffHbk9810N5gcOwHXAXHygdTjpnHGT0rOEEsifuBsGMnAJ4/vHPUnoPepcHcFJWMyC68p40dyURtpPUuxOAB69PoBXoOi6yby2WcMFklO2MDkRoOB9WOD/M8V5lPaSrMcMd2SBn+AHgnP0/M1v6M8to6YUoFGEUjpnAGffHQds5NdGGruDsZVqSkj1W22uQ7NiOIE5OT9Tk9WJ7/lVO+Rty4B3uQqoBzj3NMaeSRktkf5wdzMOg44/+tU7vDEmVPlxL95+5HoD7160mpKx5yTTuY9xaTRSBCCGJ2oi8n8/5n61ZVLiGBbQKQwGWx79gP5Vu6dbsxM8KFGYZLt1Vew9QT/L3NbkdnHbqZIh88hA3t0/AfyFOlhL+8hVMRbQ5e1toNPi+03n3oxvWP0PYt6n+Z59K0LaZrWEXt4S1zMP3aD7w3cD8z/nilu7cNOI1b5Ac7j3Pc8/kPz7VjTyBkeZTjnJY8t0wAPw6f/XrRv2e3Qz+PU0Li6LSpYQsvmRgs5HQf/qrImYXqmyGGQKXZvUe59/6Vz8EspaSaQ7FlGCc87SeBn3/AJVuWCNb2MjyHL3ICqPQHP64rBVXUfkaOCgi3a2iCItH1Dlif1q0pTgMByCT9OgH4mq5vEjXy1/jyOOgUcH8TWZJKxjAHLE7ifbt/jWt4wWhi7yeptXMzSCO3U8DA49e9VS7WUSugxuIY59zgf402FsKhbqcGo5/NuI4oz3G7/vnpVt316k26G5bTxeWWQ8Nz+nFMi1ZmuHUfd4IPr7VnCMjYIjjKge2arrhJtvG0g5/r+NU6slYnkTOplvHnUPCfnHH1FVLXVI5TtfhumPUj+tc5NeGzlUZzFMOG9GHY1KphmQy7uTjd7HsT/jV/WG2T7JJHYLfREbE+Y91Pf6VRvL2ONcEfKc8ntXHSyTWsg3HhvuntUUmqzPmFzz0wx4P0Yf1pvFNqzF7HXQ1pNVSJC6uJkHOAfmUVkN4gtWBe3bYw4Kt3z79KxpkFxJvwUcenIP8qiuNPSZcg4fHOPlb/wCvWDqyextGEepXvtdtGR1eNm6/d5x9UbI/EcfSuammVQLjdhW6EKB+Py8fhVu90h4o/NUhWXkNjDY/rXGSmcHyXcbs/eAAz7Gud3clc6Eklod34Js9OvdVzcEFWb/dI9wfUdcd+3Nff/h6yistPjSBmKMq8Mc4OOoPoa+Hfhr4fvbzW7Z7GULKG5BGCQPzBFfeenQCC1ji2hdo6DpX0uWx0bPFzCWqRfooor1DzQooooAKKKKAP//X/VKiiigAooooAKKKKACiiigAooqnqFytpZyXD4CoCSW6Af1+lTKSim2NK7seIfFjxHJBALcv5UXIAB5c+v0r5Sc/2gzahetts4/uL/FKw6Z9h+prvviH4hOu6rJKXymdo4xwOOBXJJFBNJE33ghwq9Bnv/8Arr84x2I9vXlUvc+3wND2VFREsgWjae4Tg8HsMf3cfz9TTdR3SKCi+WG+7nk/gB+grWuLwISjISF6Dp+H9Sa52aRppnmuHLMw2fL0GeSAOw7Dua5GlsdavuYc9rcy4itxlW++R0z9e59uAO9W0svLjLSOHb0HOMerdB/KtuO1eWBYbdFVDxycE/gO3vV1dGhto1NwfOkbOFx8qj6dM/Wj2T3Y/aLY4iXfMuIIjj+HB2qR67jyR78ZqqiiPdKxHHBK8L/30eT9c/hXY3kMYG3diMnBA5LH0z6fp71nx2kEf72T92qHCrgZ/XpUcrvZFqWhg2+nJP8A6Tdx+ZHxjzDtXHYBOTj64z6V1emRkoGct6YxtH0XPIH5U2KREYlysRUfKoIZiT3Jx19eKlhe6k+feuwngtuJwfQYz+NCdnoJ6nW217PHEqIqh2OFznj0Ax1P6VuWM8iQGSZ1iVsbj0JJ6ALyxrJ00KCqru7bn+7ge3Uk1uSG3WIICsaAZ4wpP1J6fqT616lFWXNc4Ku9rFSa73MSkZcjOSflH9SP51z9zdzyuI7YKg9cd++B/wDXrXuBDJGCh2qBgdPzP/1zWU62yHAOR3PQH+p/zzWFZyelzWmkugtpckSGL77gYJ4O3146D3JrsrHddp5znaqjK57gd8Ht79TXI2sSykuzeXAnLM2ACB2A4AFdRa3TX6iOzj/dH+I8Z2+nr7D8cCt8K+jMq/kW5EH2Yyg5ZjkYAzn1HtXJXjSRt5eOhyecjPqSepr0SW0McH7wheP++cD+f9TXn+sRtFG0pXoMKDnH19zRjabjZsnDTT0PO9YkleTNsTuUgk4xtUn+Zx1POBXX2jrNZx4cRjIUEDJJHU4HU/oK4yd5I5HtY873+aR2P3PTkdz2A+tbmjXBugLSBfKigQZPUgHnkn+LA6du9cVJnZUWhtP5EgcRfMyjaSBwD2X3Pc44HfNPiiEcJ8sjL/ebOT+Z6n+VRxSRfZmMI226sQ2PvSE/wg9ceuOTSW264uPtCKQFO0eik9FUe3860nZSsZK9iyLHIWGJMySEHHUnHbnsO5NSz6e0GHfLHPBHAwep/PpWnZRP5Zhjb945O4ryce59P0ropLNfljPMjDGCc7V9T05rqp4bmjcwnWcWc9Z/NwQRCvUDguew/H9K6eyt7e9cXkwUxRDCZ+5kd1HcD17npVSa2hiVlwAgyDz1x1A+veo4tVTzI7KBDLcSsAqjhFC8kn0Ue1dlGKg+WZy1G5K8TqYZYolM92NqHlIx1Oe7e57DtUcl1b5+037bdnQDAVQOgHqfp9KzZ7iEgAyKVRhlUHVvc/zrCmuNy/a3IJ3fIT0A7ED+v5etdk6/IjnjS5jYvrtHcrIfLXALf7K44GOpPtXL3lzBP80g8u1Q5255YdyT79B7fhWXcaqqK8u7zHc9evzN3J9T+grKfzLmNY3YkDLMPTP82P6Z9q8ytieZ6HZToWL9u51B8sAAz/IMdcf4dAK2JrrdOI41+UHCjvxx/M1Db3DW+Z1RVaNSsX91MYG4/wAgPasCXU3RAIQS4DOCep2tx+ZNKDtEUldm7aypKZZclgAwX8Op/E1ppBmP5upxk/lXN2EzwWsKT/fbCnt0yxx+JArqY2b/AFOPuxoPxHJ/wropWa1MJ6MnX/VxepHP51a8j59g6Lx+eRVFMCdd3GP5VpW8wZC5PpXTGzMZFecbLdcdQP5f5zWPJMjSSleAc+5BH+FX9Rn2qyp2IIxWHHMkjEg4Y+vtWNWWti4LS5LFgqYLrHlSdGHQH1Hp71WYTWjlVOdvB+lXJkj2lwSufvr/AF/z25rEu2dR945XofasZaFrUmkvxgxvjb3HpjuKrSXKAYlXPo4/kaotcK/zPjf0PofQ57GqsjsrbofuE4I9D9KnnZXKaguo4wHVxn+7nB/D/Cl/tOwvgbecmOQcBtuMfWssfZn+8SCfamyx2RGXAZh68H8D1q1JhYz9UaeKN0Ys8f8AeXJBA9Qf6GuLjuPOuzBwQDwTnP455r0m3SMoRbz4LfwtyP1rPs9Nd9SPmRo27PQrkn1HQ1rCKbuDlpY9l+DXmm6J8rzI8gA7gQG7Yz0b05GelfXMX3ehH1rw34T6F/Z8LzMWV5eDkDa4H3WHHUchh9DXui9K+owUWqSueBi5XqOw6iiius5QooooAKKKKAP/0P1SooooAKKKKACiiigAooooAK8k+K+tPY6T9jjbb5vLegA9fqeAK9Zd1jQuxwAMmvkH4neJF1rWXsoW/dQt8xz1I9P6V4ueYn2eHcE9ZaHpZXQdSsn0R5XDEbt3cnapPLH19BWrFZmBo8DbxlR32+v4+9Z1vI10+6VTHDGcBVHXnp9T1J/CtG7uXAMiLyRwcEhQO1fDxStdn1zvsYWoyRRJvI+d8hQT2Hc1iDb5qlyxLfKicZbPU+2fXH0qO4e5uZ23sdqdAD2HTJ/pVrT0EbGTcWkfPz9So9j/AC9fpWSd2bWsjrUc2y759qM2FKry2R0XPapLy7f/AI94+XUZPPOT249uvpT7dQjRhPvL6/eJ7/T3NUrlmjbEimMscYUZPXoP6mt5uy0MUrszRsO4AGaX8cJ7f4k/QVTuYZJcJE5e4cY6AhM/XCgn1NK90tu+PLO92+SJR949ief896s6bDwZJQJp+vllsopP95gACfYCsormNG7ai6bpTR4e6myP4v8Alo5P+9wPy4rSmZEHlwRMkeOSx6j3ParUM8jB2hdEA43ffbJ7D/AVHeWSyKZplYAfxMR19eSBn8Kpx090nm11KcGpW8JI+YyAcKmWCn19/wAeBW5ZSPqDeZJHlV/ikPyqfXA44rkY4o4v3qDKnkdBu+vrUqfbp33Tsqxrnhhx9MHj9CadKo1o0E4Jq6OwvpklAijke5ZeNkY2x/8AAm6fhyT6VmOsm7E0aqB0AGMn37/nU9lJbxoFLSXEzdgdq5/22+6o/U9BVqeSExlY8FVyrMDgZ/uqR/QH0rqlHmXMYRfLoUGV5SomfCKeExgH/gPUk+9dxozKAqwgvKMgjsi9/YZ+uTXGpbp5gUyrbLwTg5kIPUeorpH1FYLZLLTozEmcAEHdIfUDOSPyHv1NbYW0XzMzr+8uVHUieJ0MEJzI5ySTk+5PoPT9K5rVkVlzKSFPy56Zx2A61q6OIYbYyy/vHfkhMHJPAyRx/QdqoayplLqxwdo4XIGD0APX/Hk9K6sSnKnc5qTSnY8g1C6SKdo7VNu05z6E+/8AePcnoBxRo8YLtHcDCcE88sTjH06Z+gFbWsWcdjE+EDP3Xoo471zGmvNaTSS6gTGZQCU4LkHgDHbeeMdcDHrXjRVnZnqN3jdHXLOkirEOY0yeOmfr6f55phuwkDTTZC/diVeM5POPc+tZdxcQRMJr5jtQZWBepz3b+np7k0Wn2q6uxf3UYATKwxE4UNjqT64/IcdabkSonf6PeGBlghVVdFyVPPPq2e3p6mus3RWkDuWLSkjOP7zdvXPtXD6LbSBhsPznlnxgFj7df89q60RRx+XHEd7ZIjz6/wAUje/YDrXp4Wo+Q4MRFcxkXMsrFsv83IJ7L6//AK6fZFlJNqvloeGb+JvRfUknk/lW1Par5YVPmZflwOgJPOSO/sOlRC12DOduzjeRwueoQd29TVqnJSIc48tjMuLuC3jCIgRFyBuOScnnPuTXHanqSySGAPlgOSeka/QdT6Dqe+BWxqbbInmhXZuB8sHsOm8/0/SuIttKnvW8yZsqTu92+vp7dvrXHWqu/KdNKCtcvxXGVV7cbyPlXA3ZJ7Dtn+83rwK27m4W0hEVuge4KgADnLNxyfrz9BUNtBcF8wrtWNcIF4VR9e5xyT0rd0zSohKzu2doLSueigD/AAq6NJt2RFSaRkyWs12qWaEiCMAtjqzcd/oP1rUGlJh7+cdcH2AHOB7dq6W3todu8jb5zBR+PP8ALH51Xvb6OOLy0G4K+MZ7AZOfxr0FQUY3kcUqrbsjiol83UZEI4gx+f8AFXXsQrMCRuBA/HG4/lXNmAWrGYHOSQ3uWGf61K85aWd0PzIMgepY/N/h+FZwfKOSubsVvKEE754BH+fwqhbzlbeRSc7QCPwrW0u+ju4pLLO1mHyg9MkZH4Hp7E1y0rGFSGU+XKvB6445FaTaSUkZq7bTK1xdurMRnswx6/8A6qmjKyjzVAAPzY+tczBcyB/JkG7b8p9cA8flW9aiRBuHTv8A41xqd2bNWRff5kwGPHr1x6fhXN3cjKXjyQcA/QjjI9Qe4rWvJhBtkPAJwSOx7f8A1q5bULxGbcSA69e34g1U2KJW+1fN5brg4P4j2P8ASrcRcjZgkHgg+n171ioGmYlgAc8EdD71tWpmhA3KSPUc1ES2WI7OZe/X8aklsnkXBchvQ9KvwSiRQc81caQOgQgMB271vGKaM22cva2VxCzsdgI5BIOD+NaNmZJrgeYysD0PQD0PTNaBkjEbFAeuCcjj8f8AECqdoqxys4dtp6g9Pr/9euqnFInmufSPwr1m+mZtLnl3Bfuq3PA9D3+uele9r0rxr4aaN5WlebMDvLBlLegPY+o7g+1eyJ92vpsKmqaueDiWnN2HUUUV0nOFFFFABRRRQB//0f1SooooAKKKKACiiigAooqnqFwbWzkmVSzAHAHc0AeS/EjxnJp9rNZ2UoRBhHZfvZPVQfp6fU4r5QvLszMSSQZicHPT3z/n861fiB4guNR1yVnbKQkqqj7o9cVzFraNfFVlysaqCzZ+6o7Accnv6V8DnOI9piWl0PscsoKnQTfU17ONZjHawgtGo+Zzxx359zVzUS8lv5EZ+VxxxwFHcfXoo/GmWim8k+XEcCAgD68Zpbx2MJRAS8hC7h/CMfzx/OvMXwnf1OQleK1IJ+Ux8c9Ccd/XHU+pxWhp9u8WLiXJkfBjD4JIz94j1OeB2piWttJMnngnB4Uc59AT6nvXXWcKp/pNwBtOAo4JZufzH8zSpwvqVOehJbxGGI9cnl5DyeOuP5DH1NZGpSHmNdyMBzkcge4/xrqriOWMhM7ZpOeeNq+v+ArBuUUny4gTHzliPmkOeT7Ln861qQstTKnK7OLMDeW8odkB43McO2OoUDt61Kl1KAsFp8zYAZuh2/0H61auYJJZGI3YHykjjOP4R7fSorayRcxldqnjAJH16Vx3aOrQ37C5kjZlC5EYyXPA49McADtn8q1kuXu8yRR7woyDtGFz7k9fSsa3tLoFVEgSPg7QCT7cf1rdmsrifYkjbUX+Hdg59cD5R+OTW9NyaMJ2TOWvjFGN9yfKBPGWyze/Hb2rOkuJCAI0IU8cnLY7jpgfzrsJ7Lyj5sUkaMOM9W/MnrXMXluxnKSzE9ehCjJ9TjP5VMo2LjK4y0W6d/tFwCIlyFG4genHYe7cn0rprfUBEWzCTsGBJwiRr1wnp9eprBtvkkCF0IU44GQo92Of0q1PeQ4AEgk9MqQPw4J/KtKc+XqTOPN0NZrmaf5bZPLDnICcf99OwyT9M49q1LdoYQIY1a4uHHT7qkdySSW2n1PWsaOO6QiTCu+35VJyRnr8vv3P8qYZ5UkAjUSscBneT5cjrkLgH8TXVCp1ZhKPRHo8d1b2iCCL95IxBZ/4dwHYDAIQdO2T3qhctOEaRHBlYjLnnDnsM9wMf/WArnY7+eaQ28D7fMYGSXAUH0RM+vUnHA9K0SiX2YIpP9HgwksycBjnLqhPOexP4Cu7n542RycnK7syriG3SP5j8395uTnufc15pqlwI73Fqryt0B6HPTJPbjJ/ya9DvJDc3jwx/JDCPmxxgdhnt05PX8a5DULS3CtN8wIA3AdW54AHbJ4HevNqQtsd1OXcq21tGzxyzttDsPl7se+Py49K7CDEtxF+62KgyFyMAD6/nz+tYDJHBHFeSN1Unt8qjjI9uw9akhklv5TAquUfkID8xGeGkb37AY4rnirOzNZbXR6BaahExzF8+T26nsM+3oByTW5aFnm6jzOjNyQo/ugDjjv71iaXZRwqIY8s7cOy9AO6r6e7eldrDZBVESYVQOdvyjHYfT/Jr2cPSbSPLrTSbK8bLM0iL8sacHGAfx+vXAqLULiHyVVOpOPTKjsPTJ696aZ433eRjykOOOAxPue2OSaw55FlleVjlD93Ax8vTIz0BP51rUqcsbIyhG8tTA1V4pAZZ3AReWycgnoAo7+gH6VRikYgI0OE5Y7unH971Y+g4Axn0qldNLeXf7vJSM5G0cE9sZ7Due/auz03So8RrMTJO5Bcg52gdF9M55wOAeSeBXnUKbqTbR3VZKEbMLW2ubtWlkULsOQG6bj0z9OuO/5VYvENtZR6bacCZsyMerH39s8kfSt+YJH/AKPapiKBTubr+A9WJ/KsS1ia7nZn6IpQe2PmP54xXr+y5Vyrc81zvqyxDO8stv5fMaTMFPqF7/iRx7ViyxhHkicniMZP+0Tmunkt0hiiEY5JOPrjoKzpYftE8uOCXP5ADAoqQdrExkrmbJEZVLHlZFP1DoeP5Cs64UQ7ZVBO8duue9dTFbeQ2YxlT0/4FVDUIRDumCDyznPbafX25rKdPS5SnrY465nlhc3Fq5DKMj+nWoLq/e+QzqRGZjv44CydTx2B/wAamv2EM4mxjaMMOzDrnHt3/OsC4ge3dxDzDN8yZ6euP6VySbWhsknqWYzK0hM2cg9T1H410lq0oQA88ZU96xrCNpRvIw46+49a2yuyIkcKDhh02k8g/j2pQXUJvoZepzqkbMccjGCOMehHpn8q4WWQyP5bghTypB5HqPcV0mtXDSRnGHzwynr9RXFWbyhzESWGejc/lSm9QitDqrJNoC5BrooQF5/yawrRcDArWWbbjdwaIBJF54kxmPg+3Iqk8jxtsmOM9CKcJwDt6Z/Wonl3EJIA4PY8H8K6ImbJfMHlt5jK47FRhufb/A0unNEuY3zsY9eR+RHf+dZsUUTSbYZGX+8hOf068fpXqWheCZZ0inkJMbkE4O5WX1VgTyO4NdtGnKWxEpxivePojwEWj0aC3EizIqjDA54x9AcexGR06V6IvSuS8N6bFptqlvEchBXWr0r6ekrRSPn6rvJtC0UUVoZhRRRQAUUUUAf/0v1SooooAKKKKACiikBoAWuP8c+ILTw94fubq4+ZmQqijuTXXswRS7cAV8zfG7xC5gjsIuDJwq9Tjux9PYVz4qsqVKVR9DfD0nUqKCPle/vWvtTZiCzSMWx2/GuktYRFGlk5Yyzctk87f/risG2gS2mluJV8xoyAFyAAx5JJ9fQV0lnLI0MmoS8yHgDHc8KP6n2Ffms5OcnKW7PuklGKii5cOVjIh+XadqgYxuPH546VKZY7ezMikbgu1e+49yPWsFboGTaxJI4X8eCePUd6vrMbiRZHj3RxHaoz1yeB9KUZDcS3CFESyzrjI5HTgds+nrioBqDxXAeQAOwBjU9Qo7+gyeB6CqOp3bwAIG3zO3Poo9MdPpWVYXAuLmSRCCzMQCWyBt6k+yjj6/jTUtbAo6XZ6Bb7ERrq7b963zOSOQSOuAfThRV1bNpJFkkG6QgBIVzu/wCBY6Y9P61zum3KvO00LkBeQzHnPeQj/wBB9Pwrozq8EMWzb5UZ+bv5sg6DOPmC4+n866lKLV5GElJP3SebThsHmhccYUHj6DHpUUejIR5r7trc7Y1GCPqThRU9jN9pIk28cYQjk46cdABVu4LqdxOTnOCcn8ugqnGDXNYjmkna5QQyW6F44gFPH49ueM/yrPngvbxyZHKqP7uefx/wrWkkiEiiSUPJgZI6Y9qlN1byhlWViEGG24GP8B+prJxUnZsvma1SObXSZPMLLgN0LHnr6ZqnLoJdjufdjjPH6Zrt0lswoQLkAHvlmPqT6UoOX3eXz6noPYCl9Xi9Lh7aSOEHhtAn15J/+sOtL/Y5iUbIzx0yelejfZlnUeccBfbA/Cs+fS7EhvOTCg59M+9a/Uo9CfrT6nGRpd2oJUtGx67QAv4kDJ+lVgss5XzXL5YZLqVAHbHYnPrXWtZW8oZIfMZvQE4A9SenFZz20w3IqkMMfNjOMdBik6DiwVVMzn0mVGUQRkhSxZskHnvuPQU/+04tPgisYm8pYyQi4LHdwSxx3Azj3xWluug3kD7pIXc/AAbsqKD1NU7y2iuVEafM+ck4HY8DA6fnWi93WJN76SKxlW30/ZCAXdsbm5+Y9OnGQOe+OvWuei3tFGJFLIGZ+vzOegwPX+VQ3FzexzC0SMCLIVnXcwAz8xHHXAwoHU1Umu3+3FrYbCSI1B/hxyQR2AHU+2KG+axSVjoBp8uppL5qjzdyqFAG2MDgBccEgcDsK6W3sY7WKO0tkBMjckHJb3dvT2Hauc8OXUqyRqFIVizru5Zs8bz9e34V391IplisrfYspA3yDog7gDueM9h+FKnTi9QnN7GzZJDFEUc7iv3seg6DjufTsKszySyRqk7bIyMt2P0z7CoIhBBGJERiCCY0zlmJPBOOme1QX5nRlg489wBs7IPc/TrmvSu1A4GryIrm481fKhjLRqAAq9y3RfbPc9cVjaiHjUxjEspOCR9xpO4x/dQfyrbdks7ZDGSTkhSerE/eb8uB6CuV1KVijEfLGqdfb09+etcuKlaGptQjeWhgx7llJuCBEhySvGSPQ9z9Onau2sbmQRBmxHkfKO/1C+gH4k9eK8ut7xZ7wBBuMXJzn5fy6cf4D1rq4dR8sfZ45A8xzJIzcBF6gsew7geg+grDB1LG2Jhc7WWZrXTyZhzKScHkhVBbn+Z9Sa1PD9u72yPMMNMCdvoME/yxXF6fcvqzbiT5bncS393+EfiFyf1rv0nENr9oj/hXao75cZz+Ve1QkpPm6I8yrFpcvUbcGNntnyAu4n8zVGQpbt5yjgtnPoR/+qlmZWmhgUfL8uP6fzP5VUtJftAls5fvoSV98cEVUppuxCjoTSqYbh4cjZIPkJ6YbkfUetYlxdXMRkDp8ycbG7/Q/wD6waS8uWMaR5wQTtPoR1H9RWZqd2H09jKN8ZXa/OGVv4WH8vbFc8pp3sWo9zzjVtagmmjaNGhkUlW2nKnHRh+tTWd0JgIpeVPb0PqK52QR3N0ZRw/cdN3v9a6Cxg2YGMgH6GvNcm2dVtDrbFDAB3XPB9P/AK1WNQmWPaR/GNp9CBUFozRxAxncoHQ1k3crNI0XQKNyfzre9omdrsxNScncrdV4NY9rEWYMeoOD/wDXrQuHdi0q8n+YH/1qkgVPMZk4Dc+3NYPUtFiNzEwJFX1kBXL4OOtV3RWDoOoPT69qzpJ2jYSA4I59iKa0HY2WMU0nkhgkn8Of4vb6+lZs9zNA54wO6E4H1B9ayL+SObKplNp+Rwc7f9k+3pTvOmv4fs10w81R8r98/X+frWkX0FY39Cv2utURJEEjA4+YDdt/r+dfc3hPQkt9KhkUKCR8wA/mPUfrX50+EtYu9M8SkyRf8e55XOAR6iv0T8DeJpta02KR4Aq4HK8fp0/I19Fljjszysw5t0dlBbiJiq1oAYpAMEn1p1e2eS2FFFFAgooooAKKKKAP/9P9UqKKKACiiigBGPFNHWhjTlFAFTULiK1s5bmf7kalj+FfAfxB1+41vXLnUOQM7UGOg/zivrz4j67JpujTlFAjVSCzdCxHAA7/AMq+E7ktcSl5mxvJdmPfvj2zXzPEeJtTjRXU9/JKF5Oo+gqw5hiidtrOMtn8yfqT1rppIoY7WO3jOcfNj+p/nXKWTrfzmXd/rMqMDoAeceg4/Oupjie8uSsYIjVQFzycHrmvkI7n0kjHthIDNNcdByxPp0wAK1EnIt/MIB8z5gFHGOgAH8hVqdEZhBHnyVJLseASOcn8eB+PepX/ANTlRsVcDn/a749T2x0qoxtcTdzgNZuvIjKkkySHaxHUdiR79lH41zkd7JEzR2KABUAOeTn+FeOvqavakBJfCAne3IO7ooHr7mrOh6VJcRrcyKdhO5COM9uAP0NZxXY2ubelx3MURMjkAY3HoS56jP49AOK63TolmmI2GSXILHsMdMnoFHpVI6ftQRyEb8HGScIO4Udz6n14rrLC2ihtltoSBHgOzYyW/DuPTPGfWrhGTlqZTkkjSW8jgVg77io+b+n0HoPSueu9QklJEC4X6457kkce2K0roQrEIU+VTz6fUn1J7+9ZBga6JAUlegA7+vtgdzWlWcn7qM6cVuyk05RfKU5ycsRxk+gI5xWna28skQByijBA7fUg4yfrVq1sUjOM8+wz+H/6q3I4fL+ZkIxjGRz/APrpQot6sc6iWiKkVsY4yxOB6nqfb/69W4ZCq4AOTwOwH+NE2xzhOq+nAAqJCm7BO4jr/hW0dHoYvUuu0zjKybE/U/5/KrUWnGch5GBA654Az+pqmG2MGAJY++MVKsxLbd+eeABwP6mumFRX94wlHTQ6SEoV8lACv5Dj0FRXEAkYE4RU6E+vr/hVOKT5sSuNvQ9s4qF7/wA9ysAwpH1IA7D0r0FVi42Zy+zaehlT6Wwn3RYJY/w/5/Wsm6W1gdllQggHaT+Wf144rsZJP3TDbhjkj1wB/WqqRxzkRLErepPOAeT+XGKwlRjfQ0VR9ThNQ0aC6wtlGAoYAsCfu9jx3rzoadJa3l0LxgLaTo7nAVSfmUD3zxyc17y6RJI8cciqqK25R2BzzXEa54dnvLYSIm5guQSMncjBh+n86xqUuXVG0Kt9Gc7JNNFOXTFvEoT5h2UjhB/ujH8q6rSLy3c7xhIA+WZj1J4C89fXr/SvOru+MlmIMM8rMXynOEGOv+8x7elWNLmQyImoEiPAEcWcu5YjJI5CjsT36etYuVp3Rvy3jZntWj3ltfTztYOZQjFTKB1YcMFP91emRgZ4FaEsSyyGJCFXq5B4AHPJ9+pH59qxoLmNbFbS22oCOFU4yM47D1zjP19hkzahJPdCyiICAEyHJ2qB1ye+P58V2SrxjFI5FScm2X729E06wQN8ozlzxtUdcfU8Vw+t3yysIYc+VnCjoBjoW9enAqa81iK2t5mgDFjlVz3AHAHuep9OlcRrcsz2kUMrkbAGnKnB+b7qg/3mP5CvMr1uY7qVHlFOrKqvFZjIQk7hwN36Akfz+lXtNV54I7dyFRiZJjnmSRui+4UVRtod0Rto1WNYiEwvKhiMke+3ufU11EFvEsas/KxDcwAxx6E+/fHPaoptp2RU0rHciBEMemxkDKBS3f7o3tj36fQe9aGoawbi8MNqpCBnIx3JwB+HT8qxLWC5WNr+5Y+Y4YDt15Yn+Q9qktrabyludvQH8yf/AK9eoqsrWR57gt2WbzUJDMm07fLVSSPVEIX8Oc1lSX06FpEciUNuz7nn+f8AOp7q2ky8bckjae3+eBVDymhILkHcoGPXFQ5yb1DlikE96lyzGI/J98Z6gE8j/gJzVC8d7m1fZwcEHHPI7gdx6j8RWNcOLaUO3yIw+9z8pJ4P09aItQeIEgg4POP7w/kf0PWlGbb1FKPY5xICH3jBxzxzkV1dlh0GOGXg9/pTI7RZibi3GAxJxVu3tzu2jgiklqSy8zeUmE439u2fT/CsyYZlGeCBx/WtOeMyIEPDH+fY/j0qjcEsgkxyMcfzB/nWrRCOWlO07Dwe2aWBieP9lf1p2oKDMjg/5P8A+qktmDsX/hxj8qytqWWLmXaflP8ArBwffNY4Du6xn3GPUf8A1jVlFdkIl5A+b/HFakMAlf5QAcbh9euKSTZT0MaO0mD84PTPoR2NWLu0RLcSxg/StlDGAOMFePwP/wBeq13cCNmtZ0PI3Kw/iH+I/wA8dOmEEZOTNfwNo2hajchtYZombC+aB93/AHh1wPUV9oeA9Ht9EtPs1vOky5O0owKsp7qf5j/J+LPB9z511EI1WYCTbhiVDc9MjkV91eG7e1+zo8ds9szANgsHDAjqGHDfU8172Weh5ePVjsKKKDXtHlBRRRQAUUUUAFFFFAH/1P1SooooAKKKRulADepokcRxliQMetA61nazcw2mmz3NwcJGhY/hQB8sfHDxKtxcxaLasWSP53x3avAEUqgEgzJcj7oH3U/yK6fX74axqt3qdwCY1PA/kBXOQCbzzdHHI9Op6Ko9Bnp9K/Os1xHtcTKXQ+3y+j7OhFGxYW+LtbdYx8g6DvxhF/mTXaW1ulvAYBnzJMea/wDtHog9gOTWPoURt4nuTlpHyqnHfH7x/wAOg/GtdrlhIVgA4BBPYZ4yPUnkVz0kkrm1Rtuxk3SJ5qW0RO0nnA6he/PesjV74RRLFFIcDcR6n1Iz+Wfritq53NKQv7xlBXP19+1cFrd1H9tdVIY7tvHQ4HIA9BWNR20RrBXMOy05pJXku1BkuvmEZzlI+o3ehbqc9uBXfWhg3xiEnCDluhY+w7e3tWPpdheToYSfKeY5dtvJ/wBlc8n/AD0FdjpdrbQ5SD5mBxnGcEck++PyFEU3sOTSJ4bQs3mzjDSkKiDrtHQAdh+vU1p3VzHZps+8R1UdiOmfp6dqfFGE3SgfP15OD+J7fQVi6g7b2CAEk4+X29Pxq5e6tDJe8yhPeTSuEBAZhyT0UZ+vX2qytyY1WFgfQK3U+5/wxxWN9naBjO5HHOSen0q1YnNwrrxu6seW5+vbFc0Zu50OKsd7YlEtw9y+1nPReuB0A9zVq4AkO6X5c/dj74qtYlDGsyDcxOF3D5j6YXGPp+dX5kkiUqyDPQ5OT9M/zr1knyHnS0kY7s54j6evqfQChZRGmFGB6+tPkBdhHH7ZJ44FU36bj6kKP6/4Vxu61OhW6lgSbQ2OWb0q35626iPP7wjn2J/rWQd0SbjjceFBqZESNA8rbmyBwO5pRmwlFG9CiSMN5zxwKv2sI3FkAAXv24rm7OcCTk4znH4delasWoSSAbCQo6HpwOB+td1KtHqctSm+hvPOgBG0MxHf17VnX5aLMYf92U+nzGoAftBIAJQfxHpjBoS3eV1edwwGTj1Ax1z7V2+0ckc3Kosx/KDXe5pAEAAAJxuwOGY/WnfbJHstlxgSqVQ4J5LHOR7mr93oyyFQ2c4BJHBwRg5rNNoUJWQgYI9s5xjH5Vg+aLtY1XLI831iyXQr5r12xaPFkgdAOAo498VkgC0DTR7TNIeTzwW447ZwDj0HNem61YJqGjvaFR8q7CMY3Acj8f8ACvIJ/NtYorS5x52ZMMBk+ikL7D1rCcUtEdEHfU37XUVjk8iKUStn5VUZJxxyegH9B787j3EvkmHG9nwrbBkfTPfHc965XQ7SVkE+RHaxAhd5DM79C8jdCeuFHH8q9MtraMoiMegySRt7dh1+prn9jNo1dWKZwcsM0b+dMRhMhUUZycdPw71lPYzsRdTryzEoDyTI3G4+/Yeg4HevVjZRS/NCMjGN2OMe3rVOfQpkjLFCG24HGCq+3pn86x+r1FsjT28Op5g0Lxtb2Q5yT8o4475I7nufc4r0HR9PkvJYlnQCPjCjhQByT+Ap9hoMKzSTy/My8N7Z6IPc8V3EViYYCqDEknyAfTk/0FdmEw0pO7OfEV0lZBfwRuqwxfdC/Me2Rycew7/lWi1uFjgt0HBwcd8Djn8qgMsVvvR13BEA/wCAxjcx/ElR+NWbe6Zrwsw+baFHPfODj6Yr2FCN/U8tydjLu7dvNZh/ECx+vIrm7vDiMY68DHr0/mK6x5fJn8sgkjDHPp1rKvrdCpEYz8hcD15OcfhWU6Sd2hxn3OPmtlcESL8wyCvqO+Pf2rn7vS2RC9t8w9OzL2x3BHpXYSkHJYb84IYDJ9v/ANVZ5mSTfAevUj09x6g/5wa5/Zo1U2Z+hygSfZz/AMtPuj1Pp9cVrPb7H82M4Knn6HkH8e9c0++OQoGz/FGeuGHI59D0rSOpO8YnJ2sPvZ9Cev4HqPxoTVrMT3L8rkuCOGH6H/A1i3M/Lc8Nkj6jtVma7jJBGAcfl7fT0/Cudvb2JQN3RiPwPQUSYJFG6nLE5OM4wfekhBWBgeC3A+orKabzVz/zzYD+ecVsRI0jR56qcmsupZrQRhoVRhknPP1x/wDXqeGF45M+nf2/+tToomyFPp+grQWMnlumc/gf/r1rFENlN4OTkc8gjuQf88Vy+qNJcRfZN33TlW7o319DXYyFS2yU4I6H+VcpfSMJf3i/d7dCD9R69a1WhKO4+HPh5pZ4vnyCw8zBw3JxuGfQ1956ZaNb2kQfAkAAfHCsRxux2J618b/DaNZdVt5I8ZXGVzgsp64zkfnX2nbkeUAoxjseD/h+XFfQ5ZFcjZ5OPleSRaooor1DzgooooAKKKKACiiigD//1f1SooooAKa1OqNjigBVry34s6kbTw3JCr7d/wB76elemeaFPNfOHxv1pY5YNNTJ3fMw9SOg/OubF1VToym+iOjDUnOrGJ82XrLFahpu5yV/ln6020tJruQZ+RV5yOxx7d8cDjvUl6pMyyuu5YiT9ZO/1A6fWtqytjp8KichpZmDKvXaO5/E1+cW5pOTPuE7KyNgZtLYIgZndR8oGDtzwo9Af5CpIEJCgDcQTuYcAtj+QGcVn3jzF5JlkDPgxR9ueN7/AJYUfjUjyLZ2oRGX92m12PAyBliPT0rRPqRYjuZwd0UJ2b3JAUfdwMDP071yY01BcmVySVwoJ4GO/PqOp9zir0+rKEzD94puOATjJ+Ue+euPSs8yz3UYe4U4TGyMDGT6n6Ht6msZtM1imbqeSQtvb7meZcE8Agdznn0/zjFdbCIkUW1qqrFEAmTnk9cDucck1wltJ9mDK0oaSTLOeNoHufQen5da6Oyu/wB0FjAXJyvHJB6sQO59KuE1YicOp0O8sCWIQDkcdDjjP8+KpvblV84DzC4+UYP4E/XtWdbyqeJd0yL8xAOFwegz/k10clzGYV8wqCc/KDwMDoT7Dr+VaxSktTJ3i9DnWsHZA0sak9gemc8fh/OpLawCEzNtIHB9Sc9O9WVmW4YzEYEhwD329MDHTPPPoPerUssCxRlHWIbQFwOBk4GB7cmpVOD94tzlsdVpMcfmLIx+aMdM8KB157k9z/kak8KyR4bCE84/ur/PmuUg1aKGF1tvljiIUEkZwo5Y+/p2Ga0ba6N9cKASFzudz1OACT7DsK9anOHLyI8+cJc3MTtpayfu0b5EGW9ye1Z02nNGPMc4RM8e/oK7GEbVEaqOcE47ZOcn8BWTdSjUJQkQCwofvew6nHrj+dOrhoJX6ip1pX8jh5IHch8cnp1wBnNNEZyFPPPH19a66SwR8sB+7Xt3JHr7VDLYpaRtLJ/rCceyj0zXnSwclqzrWIWxgGEqSQuSwx6YHens+0NHG27nBwMDP+f5VEzPOSqk89f6VdtrdjKI4gPlwST9ayjBt2RcpJK7LcIkDCMErgct3+gHpW5a+Woy3GB37/Wqa2uZjKzcbSP8/ia0444wxV+COT+HavUowaZw1JJof5jStuT5STg44I7/AKVz+owyRglQMFsgZ5B649xxW3KpWPeM5OOB6nGDVe+t4zYEXIwyYJ+jZH4VvUp8yaMoS5WjmHuEW1klJwyuFH+6QCD+GMVwnibT1EX2pc/JkZHdTg9uvX8q7bUGLjyOC23A/wBrB7/571QWGC8hudNuh86qpQ9zjKn8wRXnSTb5WdkXZcyPI4NQNrMlywVFt8qC3KoT3wOrt2A5xxnrXoVhqH2pVS6J2cM+eNxPQE9z6gdK8e8W2N/b3EUVuwTYSTv+6h9WxyQPTvx2rf8ADlwCiKztPLINxd/vBPUjsXP3VHb2rTVpWG0e7JqsEAzGQWwNuP0/D6VYmuWkQBHBduASc5J5Y4HpXnXnLBliDJK5x16exP8AMCqr69NajzUBeRfugD5VHrz3z3/KpliraMlUL6o9TR7ezjijRsuCWJb+AEZLn/aPX8aR7pVn8tDtSJfmz2UcufYlj/TtXlMHieKLcZ8mbGVj64zjBOe5PPPQVvpeieAWfnAIcNdSk5yV5Kj2HT65raGKi1ZETw7W51QYyBrqbHztmT02g7yo+p2j8K0o5cGKUjJjUZ+pJP8AhXKSa9DqV0NK0seYsP3s/wAUr4x+XP8AnNb9oXkgSNAT5khYt1yFyB/jW8JXehhOLS1NS5KedyMZKgk+hGf0zWZcoY5sDgxybfbBHFbc6rNeKWHyuCg+vT+YrOuCJIVlcfNIgV/95Bj+VdM1uc6exzE8Ij+UDKc8dx3x/hXF38JQfaICQU+brge/Pv8Al616DOhyDnjgEj1I+U/Q/oa4rVphA8jJ93B3pjt3IH8//wBVclSJtFmHJcIZME4zjGe27kfqKsK6Sgg9R1Hsev5Hke1cqshMnkEjHVGHIx1GPyyPTpWhBLnK9COQfT/61c9zSw27n8oBQ3Tgf0/UVz1/c+bIsYOQB29+RVvVZxl4xwd2Qaw4w9xOO2cD6ev61LRSNbTY2kjUn+I5z68YrsdPt/lMjDAJOKyra18lAFH3eAPb1rp7R9sXIx0wPr/k1cYgy2kONzegx+ozVmJM/Kf4ePwIp7KI4zu7/KfqcVWeYrIhBxuG38eK12M3qMlEcqmMj5gOPUjv+INcTqZlVVdPni6Er1A9/ofXpWzq98bZlnIJVz1U8g9Dj36GuR1CacS742EiMM8HHXr/APXHak5Iaiz6S+DEcFxcrJIvmIpAPGduf85/CvsSNdqhc5wK+RPgNbTQ3IkliYRzr8rg8Z/usOhB7e/evr4da+oy5fujwsa/3gtFFFd5xhRRRQAUUUUAFFFFAH//1v1SooooADVdzU5qEjNA0ZdxvZ1Re5r43+KmsR3/AI0mgR/N8giIEdAV6/lX1b4t1OTSdFvLyDiRUwh9Ce/4V8BJO02ozTM3JDZPU5bj86+fz+u4UVTXU9nJ6PNUc30NCRfNkXeu1FywB646rn8OTSw3qvMX+8ygE55HOcA/Tn8K5q7v55JHtI2O5s7sY4wevtj+ddlpmneXZeeY/nf5Uzxub/ADGfavjI3ex9Q7Jal6EhVE7jDRgYyPXkAe5PJ9AK5bVbzzSLbAKjkg5IznofXHX3OK6Zl224jwJJDwuf4mPBOPaqC2dnbbkuvmnds4HJwOQCR+Zq2m0kiU9blS0s5TEZ2By7ZCnP3iOB9AP1rOv1WAFbYF5B8u/OAMdST6gH+g55rsod08YSMgZByx45I5x7Ac5qlcWaNtULsRULAE9FB6kepNTKnpoNTszl4wtvbZmG/glFHRmHTPrjqauxFzmEFpJXKhznk5PCj69+nFMnidlyvWTAGP4RjilguoYsKAV3kksOCxOAcH0CjH8qiNky3qdAk5EaRbRtJ654wByc9TjoAKtyzqFZZBufLAY6Kq+g9s/ia5ZtR37REdqYU7sYAH8KqPp1PvV6G5nM0ixLltoLueQik7jge3Sqc29CeWw+e9uUCbAfmHyp/EAeMnPt+VQG9fzA8XzM0h2kDCJhQAefQVQvP9FtVSQtvmQ9OXIJzj9Rknr79KzL17vIiGUjVAu0DgEnJ57+5qLtbl2TO1iuUYQ2tucKr7mbrnnr79sev0rrdN1KPZ9jgP7mMEysx+ZznJGe1eQWlvfMrtDnfIoHmMcKoPUKOOg/8Ar12emxTpbpbI4CKRtVRwTnr79uTXRQrO+hlVpq2p6jLqEnklZHw8oG9h2LZyB7jgVLYy29vZM8SgeXuCb/4mPVm9h2HfFc1FbyyxI7El1J69Pr/npU5LyARLl9pCgLzluw/+vXpRryvexwunG1rm2dVAgKQ5+UHDHqWbqT7/AKCsvUZ3bZCSBtUFs+p/r/KoZoHZ3VTtWP7xzxgcY/E1QuJvPkB24Qc89x7moq1ZWsx04RvdGnaxtcr56oEjOFjz0LtnH1wAWPtVm2nt8stq26OIDLf32PTn3PSsf+0Zm0/ywuC25Y/UAjBP5VNprpZwurbdiOrMW/vLyB74ohUimrDlB2dzq/kWF5CQMFQCe+DgHHu2aQSqzrDHglmBPPY8/wCNcuL1JLcRS5QyEM5b72wZIH/1qjtb9o5zJ/E4KAemR8v6Vr9ZjdGToM6uO8jmSKVcETHBHTA6kn+VPupeMSYPmfkMDIH5VhCSFYWOPlAYD8v61RlvLweTIpIiUsT65/yK2WISWpk6V3oRXsc0Eu9VLRIRx3Hf/P5Vm38VygF3Zt+8jZWP0U8HHt0b6V11xJa3Ba5YABMCVewyN2f89Olc7KS6O8BIO3I56qzdPwrKtTT2ZpTk0efeMLCPUtLuNVRAxAzg9QU7H8q818Ms0ql4GZWlOXkPDY9F/uqB+J4r2i4T7MZCU/dy5WQEZAz3x6fyzXiWtTyaTqbadFE1tEAMyDI75AReS7H1GAPrWVJ3TizaXdHoj3iRx+VbsoYfLz1H19D/AJNYt7LcSkQQt5ca4y2cM5Hv1IH5VQhmuZFVreHyxx97AWMe5/vevXmtm20ia5kLvk5PJOcfmf1NcVZNvQ3ptJamRFZzSzYtkDtGdxbGVBHTjufTPGeea3bK1ugGWbfkkHJP4hf6101paxwxbLYHpnIGBx6Z6D3x9K0RC0KeacZwQFXls92J7eg/WiNBilW7EGhWc1vJNLF8rSNhSDymeD9WOTn0H416PYSwwZYHcvRD7kH8MccewrzaCVYoWmd8x4UDacAAnoPx49WJrQ+3CEQ2ksoZl3ySspyAwjPyj/dzj8a9HDT5EclaLmeiSt/oyyOCHiVX/Niao3bK0M0AyGQhk+vOf0qpLdC7W9dCVjG2NQfSMDJ/GoNUulY+dGflkAHP/XPNd0pqxx8juUZGDgM5IV8qw9GXn/64rjNfEnAb/WgbkYcZ9wR69CP/ANVdW10LmzMK/wCtfBH++oyP++hkfUVyl5cQzRspJ2scjjO1uh4/mP61zzaaNIo8zllEExEiFCT0x0J9D0wfb8qu292q+YSe24Z/M1X1RiZTC2OvI7Y9s9K566uGtpvLz/D09ewP58VzpGhqXLreSbx0OVP+NadhbgOZX4xyfqOlZFgFReeg5APqa0hOq/KxwOp9lHrVD1OqjnTaMfxevpV23uVkdN3CsR+AriTqG8sxyEXgDuSeP0FbunSljHn+EZP480ORXKb818JVC55wx/HNUJ5/PdihwGyR7EHP9aqqGwhHpn6nNV7wrBJlchTnGOxHFZOoWoGbfXsnzWspyj/MCPUe/oR+VZxleaIwyY3fwtjg46Ejsfcfypt8sm5ZegHHHTP9Pas64dIxGYiY2yMAZ9eeO2etFN800E1aJ+g/wg0K40zw/byMNpZRuRuV5AIIPb9a9wHIzjFcF8OruSfwrYNKyyYjUFl6jA/iHY+9d9X3GHilTSifKV23N3CkHNKaQdK3MRaKKKACiiigAooooA//1/1SooooAYTTadjNBGFOaBnhHxx1VrDw21rG21ZDlznknso/rXxg1xJp2mGcAfabkkrnny1PAbHrjkfnXvHx/wBbe51+00KIfu4hu25+8fc9h6+1fO9zcJNflpXLwqAWPYkfyB9PQV8Zn9bmrqC6H1GUU+WlzPqWNDtna5UPgSSHccjJUfwj6mvVpZ4ra38lGBIHl8/Tnp6dSfavPNH8yBmvSAGZSUyfuqemffHPtx3rRaWXy9jjaihVYA9R94qPrkZrwVLl2PXkr7kt3rEdtjyk3u5CRgnHA64+p/rUOnXLXJd3beSRuI/vMcsfZcDA9etcld3b3Fw/lgliAcAYAXoAD2J6k9667RrVoIIpWyUDBpDxyegA9R247A0J9ymtDtlnWO3DOAmW4HUknGc5/ICqJLCRpJyP3nLE/wAIA2qPzJqja3cd1qNsX/gaSVh14Rfkz+eT7mpZbuGByZEMiQhQqn+OTgj68nJHtitr30MbWZBNZgyuu0tsBA4+9k9Pof5CsR7VWlCA7jHkEnnp1zjgD2rqW/132CQ77mdcORxhn5cY68dKzlsJX85IGChyVD44GTxgdyOBzyfpWM6ZpGZk29vPdTLDChRCxZAerEdyccDp+H1ropWEDG3hbzJGHmSuOgA4AA9BwAO55NV5JIrSMx2/II2MSeXIH3c9vc9u3NWtNUCaTzHBlnG6RgOEQcAD0AHIpwp23FKdzPkjedTPtJ2ZIz94tnAH5+tNi0+UDCvkKQgZuckY3EDpgdB6mtuCW1mtmMI2RLn8FDf+hEn8z7VSM8mTg4YEAADhck4H5kAD2q3TiSpsox2ksl03XyIMrnA+YgZIHocnoK6uGOKxRFkLNK6gOOm0HovsO57msxw1nIHK7zDtBzzl3bKqPXnk464x0rMfVLmVnhQkNkK5OSSxyTz64H5mhOMA1kdJcalIVkMjeWnAIGOFHQe5YkYH+TswXxskG3AZQ2B3GAc8+p9f6V51FIzOrzsqlWDhScgFeh+gPJ/AVblvELRyGTbGpwQeyjk5xzlj1+oHrVQxLV2EqCeh3MlysNpGHbLSHlAPvORz+Cj+dULmQed5e8DqpOeODg49TngVzBv7ieRro5VdpEaD7/P8R9CevsBk9qt2qp59vNP8xR0AXsMf0HNN17oSpWN+2iCiTJ4QYG7JPvj/ABNVpZCw8vGQ5+cduPesyG8ODGxOWlHfAJAbOfpnP1+lS3NwnmLGMhQ29l/3xgZ/LJoc48ocruXXOZhcynIGFC98dPwFIkZNyEJ+UEAEeqgg/lUNu+yzM7H5o5CeehUghj+HatGOMH95H0jYkZ913Y+uRinFJ7EyYs1yY4ljGcyHn2CdMfzp5laSM4AwOWHvinSRq8zvngEY/wCBDPH65qjbPslltmPB4U/7eOv41V3ciysXfOW5SYKdjgE89c/56VTW6UwmL7syKQB2f/8AXUrnDjnG9Dg/SsogFjGemQfp7/nWilJbk2RqMEv7YkZLgc+vHr+HGa8/8a6Y13owvLSXyZrc7fMx8wU8YOf1/PNdUolsbhLtXG09cnGB6Z9+2atFEvFeaDarn5Xzna6443DkZ9+4rSMk/Uhq3oeGafC/2mKNZ2YRc4GQDjqc98nqRgdsmvRbMRnCTO0jAdAfkHru5/MZNec3Wl2WjavNEYZFnkbLLywUH/aJwAe2P1rsLWURJm7YliMKi4VVH48/pTlGzC522AY9u9lV+cZAaQ9hx0H41Vu7hncW1viOCPgkDqB/CvesOPUoU2oWwyjc5BJ2jsP/AK351iXmswxusikbE5AX7qqPfuSetZzmkrFRgy5q0txHGJYcq8cilVb7oK/dYj/YHI98Vk+HtQNvPDDeP5gVmMpP3cswO0Hjg9D7VQuNUdylpKTuciRyTngckD3HT/8AVWZO7xQqIVyzfIEPUmQ55H0JJ/AVjzO90bJaWZ6dp/iR5ILq5ly8jAr04LsQ5OPTofYGrn9ryTW8QZtwU5/EI2P0rzlp0hEwictlGjQ/3pSBuY/ifyGK07KVktWZmwscarj/AGgME/ka0+sPYh0kdQ99LHiROsWPxKc/qKx9UmKyvJCf9YN4HZhjkj39aoRXLbd8bff+cDryDz+YqK7ZY4yuSsed6Ec7fb6A1SqGbgjnLu9hkJD5BHKnqMHg/ke1cfLdmW/aOXrtx7HHU1b8QSzKJHYbCw+cDoT2YYrjtOvZbs+Y3EjfLz2rshHS5zdT022n/dCRuW71HLcMW2k4Xgn+n/1vzqDTYfNGF5X7o98dfzNbz6aZVJA/z3/wFZTvc2jZGXAXYgyYC9cfXn+QA/Gu0sl+cjuFBP1PJrn4rQ+YiuMAf0rq7GLAkbB52/zrJ6lstCFsKf7vFQ3dr5pKjvmumt7UMWDAY2qM0klgwY8dKHB2I5zj/sBeIxuu4Dhx3x2P/wBeud1GxEclq0fAjlQliOi56n1HrXpM8ZjKug57H19jXMaxcwWl9YBEyss444wMnpzXRh6dpJmNSbaZ+jXhSwt7fQrLywmREuGjPHIz19PY11NcF4DETaLH5TEhAFwccDqOhI6V3or7SlblVj5ip8TGsT0FOoorQgKKKKACiiigAooooA//0P1SooooAKrXtzFZ2kt3OwVIlLMT6CrNeHfHbxK2g+EpESXy2lBAA6k/0AqZyUYuTKhFykoo+LPiB4nl8QeNLzUpn3BiwXB6L0/lXNCdfKjXIDO2QB2Lcc/hz9K5eH7XfTlpPkErYLY6YOSf5VqySg3CxKPLCtgAd8cD8z/KvzvG1Pa1XM+3w1P2cFHsdwt0rWcbSchpMYJ5PPAHtxk+1Xp2e33SSkZAO845LuATgewrm7RlkuolBIitw7HsBjj+Z/Gtn7WWkJRdxDbUB5O9gAM/Qcn0xXGnobPczIjbLcorxlMk8E/xejY9O/oSRW3ealIGt7aBsmRjyOFCgYBGPTOBWNFFEsvmyYZmZsY5J56H3ycmqLXGdTjjhXccmJD/ALowSO2F6D3peha8zqLS9RpN8Z2Ql3iQDkkABdxPoOg/+tViS8D3CPjAjcHHXBI4HPoozn1Nc8R9kkgQnc5IbA+6mMYB9uhNTQoZZhCuWHzscHnAGM4HVjk/SmmJnT6VcPMW1O5yxI+X9e/5kmtG71Ke1AS3GBgFmA6Z7L6nqSeg/DFc/fzG3tYLW3AUxKFYHkBiw446gAfia43Vtakk1BbSGT91GhLdySTySfp/PFXzdBKNzsre4tzcwrt3gMQuD0Q7ecnux78cD0FR3msPAt3HFzPP930G5tqL+eSfYCsDS5fs8YuDlpZZG4zyS+cZ9gqfrgVo6UJb7ULmbZgA/KW9Bk5/X9fak5N7D5UtzYsbk2emJYRnLtJEu7/ZQ8n2zg/hV571BOZICdgkaU9hgfKuffAP0HNcvtMiXH2diQHUE923nPHoNo/WtWOyjEH2QNmRzCq54DBgpkz9cEfQ1UW3uS0jsCyJvknJULjaO+SpJJ96IIIluIncAKwMxXA7jcP8D7D3FZ+s3cD3EqdVKsxxnB3BT27cgCsObUZJdSzHtaONAqbgeqoO2cdc03JISi2jqZLONIVuJvlllIYDoQvXHsCSST7H2rLOmGaRLdkCR56kYxkjHHrjn6mpkkuLx4xdZ2KPm2nGWxnH0HTHrV9HkjKPMm+UsehyqySE5/CNM+2feqVOMkS5NMY0UUe1DGM+WSMHnaMd+5b196iljmaIuiYwVDNnG3OTxUEerCa5DodxJK7iflCsQCQBjJABA56kmqM3iBHvI7RT5ah8nsAu04+uOT9TQ1DYacjTmVUEZdNhyQqevXk/XPI+lRsXkjjuJeXI+YdFGD3Hes3V7qe4vrKNCyjylL9j8z5IA/vNxk9hW/qEcEaLA65YF3wvOAoAUH1JPQfnWcqd22ilKyQi3kf2RpOsS4Ue5PX8wM1b0++Ml3LA3y7hyR/f9/xFYUpit7gaI0gNyQp25yQ56E+mM1GrmG4cSEBZcLn0JySfwI4+tJOSYNJo6+S8hFu0+eARj229f51VmlaLy7xR8n3sjoCvOP6VzsF2JdNadASjFgV65wo4/L+talpdAQC0mOBK7AkjI3EEqfbpg1spX0Zk422Ogu4ooo4LiEAoPMI9GEg4H4HisQykFH/iUAk9yp/qD1o0m5a7sFguQQS6oy9vnXg47YYD86vJFHNHhRtdc5/E4P5E10O72MdtGT200N7C0SqGJ4x0B9QQensfWube1ezm8y3YgfdZTkEew7Z9qttDd2su6Pg+nrVuJhcEiT/WgZ2n+MegPqO3ftzSTUtHoxNcu2xxXjDThdWa6js882ylgwGSnHJx3x6H8DXkdvrl5cqFt3Aj45U8uR3PevosSQpMQBtLDH1/x/KvDtc0K2sdZla2McHmsTtQYJJ7+34HHpWsk3GwoPXUgWSaaIi4Jbn5ix2k8dAvp7mpGQPAqHnd6Hnj+XoK0rLRmMYZRgL05wo/Crn9jR7PNlDbUOFXHX3PTiuKUWdKkjh2kEkzNy7kY+U8AAjgH6/nV5b+OCK6l3bp3JHm54jyAoI/vEjp+dbsujSCM8bd3IHfHb86ypdA3XG5iWwMhQO/r7AetKLtuW2mW7gRyXgt41YRp5b/APAiMgfkMn8K2XikjBDcJLMxOfRlP6c020sJ2EcZfJyWdsevYfQCusTTvtMoLDCLg8dBk4xWnI3sZOSRhw6a5IyeoxnoAcZ/nV6ezZ8LjDMMqT0yR0/GurjsBGqk87k/Vev6VNPYEjC4II4Hr36+vpW1OizCdU+bvExnsdyFcBgec8A/3SO49K4TSPJWbzZMglvlQd/9709h/KvUfilaLFZkyOsco5Qn7sgHbgEgjv8AmO9eF6bdtCFJ+8xyr5GMexGc16VOPuq5yN6n0Lo8qR4jkByOo7fSu5ilikjCxpg968l8LTs4RZTkHoB29zXtNhGsiK2PkH60vZofMyibMNMvAHety0s8ruA69PpViaBBKsan5sZY/wBK1LeE4GemKj2SuPnZPDbhlZh69ParMsX7vdjoeauQqqBeOox+dQyOTGSpAZB+YFU4pEXbOXvVEQJGWA53YyMe4/nXmHiKC7l1nTJ7Z2VFnTzI15Df3SOD/SvSr+5CHkFS2dp9SPf1rz6/uEn1WxQxNkyg+ZEMY543YOefp1opu0kOS0P0l8ESWtxodvcQxiJ2RQ6jPUDrzzg12Ncv4OkSbQLWZTksgz65xzn+ddRX11L4EfOVfjYUUUVoZhRRRQAUUUUAFFFFAH//0f1SooooAa7BELHsM1+d/wAfPGr6z4m/seBibe1yzH+83b8B2r7116RlsZDvEaKCXJ9B2/Gvyu8b39vqnjS68pi0e85b+8c/1P6V5Wb1HHDtLqellcFKsm+hWtXjVo++O/b1P+fetaK2SYhsDcRvGemeQCfYZJrAtmaadGYck8enP/1s/nWtMzQQvsJ3uNue+M8Zr4SW59d0LNsRHuQPu83aQehCjnt69fyqaCaSCeRo1JkT7gbqWfjPHYcmsn7SBckwHBRgp9SVHGcfhWhHIliJr24P7x2bap/uouM/QDJ+posIa17HbRzzZLCF1iHr0y38jUtlZi41Se43FYrUCNRkAAjGSxPH3j19qwriaV7a1wuQoEhUf3+ijj1Y5+lGrahJaeHfsdjjzJpI42fOSxzhj+DE496IrUpnS395A8qxW/zLwMjku7Y2j6AAsT34JqxozCzt7m/mOVJ8vP8As9Sq+pPy5+prjZ5yT5YYgRy43DgdAMZ/2VAHuTXQzSmVdO0kfKqymR1z1IYEZ+pIqoq7Jehtagq2oMpfbID82MnDP2HuAPwrlU0+NJmLg5UF29yx+X8B2H0rU1rUIr5pmVv3dqyN3wWlUkn1PyhQM+tPZXuEMeQqSgHI45Bwx/ADNEo9UCl0C3t33xkEFXyFIGAONoA9TnOPWuuFsILW98k9I/KTHdyCCfw5NILO2fYsK4SKSNFPbaqEc/nioZL623/Z5B5ccAaVx03YB3A9gNxHHfmjSIXbMuS9huJ7extcAuY2l54xtx19l/x71uW95A08N0wAYqJE5HAC4UfiMn8RXExNu1CW4ZQpO9FXP3UwFXOPXGSfatu5cLClwxEawZVyenGTgD1wAB+FLm0G4khujLumY7Q0m0n1C9APQZ4Fbnh2w22wuXjUyIcKDzlupGT2Ud65yG3SdYrdv3cUCeY2WIVe+W7kKo/Emuim8QiOx3WgECuQIh0YRbQWfnoTnj25NOnFfFIVR/ZRrX11FAtyI3KeWQpkB+bAGW2D1B4Huc1WkmuJfMikPlgKECJ/AhA+QHuxY/Mx5OKbZ28V3YwJL+8MjGZ3P8UmcKB7Dv7CtRrdY9vIbYBhuox90cepJY/lW129EZaI5q+W00q03R8MoDKo7/X/AD04rmWjVpGuA2S3B7EEjJ698D9a27yYz3QjgG5pSfmbnbjHUfSoW06C3gnZ2LrGwV27l3+Yge+MZ/KsOVLVGil3MqfU7hLqC8kQmQtGx9Bx0H4DjtyPaug0nUJFV4bt9kkUSPnOfnwDJnPHB4HvXOTW7yyPcXHQ48lV/vKDk+nB4H5+lIlsB5Ue3ejxvuYchUQbv1YEZPX8aIKSKk0zes76OTXQkC+ZcJvJc4JC8Pk565zxn0rVvpbWOeG25MxBkIY8krDkqT6gHn3xXE2X/Et1Ge7blzEIxz/G4VyB9F/QVorEi3UNyGIdrUIGPVWkYHP1OefYV0X01Rk0bWniWGB1lGYbeUpxxk4BYkezAj6U+Kdpnls3JDofNUDnKruHX8SPwqCG5iksntmf5ZSW2+uOSfrt5NZVrPLJJNqVk20uZRtA+7tdXAAPHBP61FlZMfkdlYmOVkkjfi4UMvruTr/MGti3nlR2kbknGfr/AHh9R1rjtKuoHjubVV+zyAm5jx90K+DuUdge4/hOexFbegXjanpUjZ8m4gBVx1B2nKn8q1jHXQznsdmMXcQnYDcMh1759R/Os69tCuJYmIOePcjn8DVmzmItwxG18fgf/wBVX5AwifzADk546Eeo967VTUlqcTk4vQ42ZmmUtt+Ykkr79x7eo965fVRb3CeckoRo+WDg7l57j+Jc/lXeS2yyszQkFiM4PGR7/wCIrzzxOEClpFMbjgNggq2O+Oo9uhqYxa3K5kyrbyoApeUbc8eWDz9STn9BWst5Z8yZ34Hc9fw615O+sXlmTuEaK5x5o5T8Sc/TAwfUVs6detOi7JlfPBb7mPrx+nWsZuxvGNz0QOJ5PkVt2MklflXtx6YqUWaRjZMfLZz90AE8ep/X2+tcsk7EeRJPtRiOWJDN745yPTNdRaW+3JHTuTyx/wD11EbMJJofaiOOdSq52ZyegAHXPv8A44roLBbeQyMeDOQMf3QoJ/PGM1h3UQRPIiOS3b174H45P4VBayXVrdbTwNxkYnqA/B/SrjNReqJcbrQ6qzaO4td+SBuyufQ/Kf51N5iMfIz85LbNw4Yjqvsc5xWBBcSLEtpzhXGD7nIH4etaDL5kDNKfldt4P91ud35EHPtW0ZpmU4nnnxJ0T+0NGuHKkoqksDnKH+9x1HqD9eor4itb6W31l7aXywFbaHGBwPoQCT6Yr9GLx5ZrZ4F+aZVwAeQw7g+4/UV8HeLdO0+TxBcRW8c1nJG53ROqmPIPODwQD2P54ruoy0szmmrHqnhi4UbdnzF+2c/mew9a9v0icuidwOfqa+bvC96LVAiHDHgk/McfU5zntXvmg3ieSpkbJb05x7Z7n1o6jO4Qt5gYnLHOT710toEMXv0rmYdsuGU4ArXtZdqbl9P51LdgsbZJdNydCeRVK6Yqu6Pl0HHoR6Gn7iqK/Qd/w71BLOjofLP7wZxnv7fUfrWMmXFHF6jPtaSMqXhk54PIPYj0I6VxOs6Y1xf2ctqFLxzKfnO0k8dD0z9RXpNyY5laSNVD56fwn1+lcRqXmXM0FjHIqM8ihCWCkMDkDnrmoop86KqS90/RvwI27w5a7i28IMhsZB+o7V2dcn4LSVfD9qLmIwzKihx68da6w9K+zpfAj5ip8TEXpS0i9KWtCAooooAKKKKACiiigD//0v1SpG4BpaZKcRsfagDzP4i6pBpugXV5csFijQliemPp61+XcV2dR1e+1PaUjlclR/dXsPqRX2F+0b4rH2KLw3A/+vbMgB6qvavk6CCOO1YLgs5BwPzr5fPcWm1RXQ+iyjDtRdRmxY+UybeFI79hnr/hRqEq/ZnmVwFR8KSeuByfoOT9MetYAmNqArPwxHf+f61jeLdX+x6dDZr/AKwhnbHbd/kV81GDbse42Wbe5bzY51J27Wlz0ycBUH5ksaveW9/eSRSPnYPKBz8oLkFm/wCAiuUtbnyYo5JWztGFGdwbYM8/ienrTrTVGEQUcuwO8f7Z469uo/KtHBvYEzck1YXV1JKj5t7dnkC57RINn0GR3681k2089wIbE8rGYArdw7LuJ/EmsaAtboUOcSsFf18tF3H8xXQI/l7Jx8rTGS4Y46fKRHx6LxT5bBc1kZoYnSX5lbcR6LtJbk9Mkgk4rW0S5lWGS8lCs6xnb2wQck+5JIA+lZkNutzbHByzxsVPqCQrZ9sc+2TV95Y7VBIr7k8oSMByuA44H5k1CXQG7m1Zxq9w8IPzvKGYdz5MeAM+gOfzrQkuI/N+xKd0ShwD2+UrnPqSSfzrk7bUthW5jYiV45WlPTglRx6HkfhWva34tra4uJSG8uRNgbsqgZJ/4ESSKUkxI27fxFbyaemXILR4QHuyEhuPdhj8a4uzu7u+vZI3XfHIC8/spwVUe5PGff2qt4gAg1RZIZBHErM0Yz0V+qnnqD1p+h3kSRXLMdspjLDjum5v6jFJx3aNIvS522k2UjsLu7df3yY+Xpk7l49Bnv8AX2rpY7M305R8GKCTpj5flznPv0Jz9KxdPu3SK0W4yrlFJ4GMKDhc9M8kn6VbbWo/3KwKWW4YLtA6+azck98459Mj1p8q6GbbbK8qxXLywqS8SmSMr1BaNVzkDk8uBj1qhNb3VxqP2Bn3HBaR8AgNIfkUZ6gAZIxjp2rZ+1W6Myq3loTJukxhvmIJxj3LN74FW9Gt5dUu/tAh8mDeuQ3+tcLwqk9FGAMgcnuaI67Dlpqy60674ULbQAwxn+HoB9TzUV/qM3mBYxhArEDudufm/M4H0rQFtCLvzcZRd2COnHDHPoBwPc+1VUEZtZdWnOEYlIwByw6AD25zTcXFXIvzM5SW7fSza7TlicMO5PBOT+B/OqMl9MYIrN5CEhLtJIOhd1JY/wDAASP96m3cjteR3Z5RSRtBzg5I/kKuNYpbWqxXvMjNkj1Zzu2j6/yqL2jc0S1MdtQeWZ7qJNi4XA7IsascDtxwPc1clvbZL6CwkO3fbFpMtwFt4m2qPqQXPviqN0qLOlrbHLSMrk9eFy+OPr0+lZ1xayPercnKmENCPcurE/zxTjUG4GnaXCXG++lUCV2dVJ7B0WIMeewz+Aqnf6s4v47ja3lWjxSbXPJA8xGBA7NkN+NIllOyRKhO0Bdy56lDtP4nqfY1A2npNfxyTSfJMvmFjx8q/L09F71cat9EQ4pFB/FCQa2t0TmKQSMmOhZB5YPPQFR9KnPihdH1mO0i3fZ7gtMJABx5o8wfz21Cnhh7drm0nTe9uzKjYyAh52t6DOSD6HNSajpyafLFJqMXnCHy4HXjIDLlGHr1I/Cr5r6JC0Oo03W7K/WKJphDdqdqnoCrLkf8B5/MYrttE3Iwa3O12jZSPcHkfUEdK8ts/DsDnYrgLKSIZN3GCM4PoD+hq7aX2seHrqO51COQAsY5uhxJ/C/HYjr7j3q42vczl2R9AWdzujVJUAdS2cdyP5VorNjCKdw5wO+Pb3x0riNJ1cX1tHcnBZgQ5H98DqPY10FvMkn7gkAj7p9h/hXdCRwzjqRTuPvRkr7dseo9K57VTbXlttvMusnAdOcHHRh/hWvcSbZN0q7gThsdj/eH+etcvqQ8qQ7SEL9Tj5W9MgfdPuKYkeXXOjQx3heznG5uCwfIdRx8w9e3OeamsNKAUmSYMp6BGJGT9BTdVmMV2Xb5MnP3TgH13Lg89/et7QrwzgsJlx36Fh+LDdz7VlVp31NqdRo3bLT0tYwxYFvX73+fwzXQ20VwVCdEXgdO/c4qK3MWP3Sl5D69B7k1pweY2CwVQOeTx/ke1ZRhYtyuI8cdpt/5aMwIZj1wOw9KR1jYvI+GG0Ej3IyFH0x+lWHg3ZZk+VcDP64xVJnYHzScOBwOuFPBz6kmnJISGK0RjmLEAKQAewIBA/Dv+FRpfNZzRxFt+5nkK9iQPnx+h965W7vCjpbnJVhk4PocAfTqPxqlJdSDVLZyfLDuQP8AeBOD7Dt9KwVW2xr7M7o6pDcRp5a4IAAPXcPT345XP0r5q+M3hZpL1dbsGVknBZl3gNuHXhsZHrglh6GvfNOCyItxJGTFgElf4SOuPp/KqXirRNK8S6TcaNqkRaN18yKZGwSQOGBHRh3HX0rtoVWndnLVh0R8leFjj5rpGAQ8Andk/wDAe1e9eHpnnkTA4GMA9AP8a8gtNAbRbtre6kynARgCwkH++OvuDzXqmhzC3ZUyO24qOnooz3rsbT2MD2O0mXZheff+ZrZicIMgZ6AVxtpcB1VTxz0Hb/69dJZ/vThTjAz+NZSZSRfutQjjDFThh0B6GuevNUB+eDIzw6N1Vh2+nv2rVurVbiPyzgcYFY8lgdrEjLgYPcNx0PofQ1xVJSudEErGVcamzqZmyG/hZeuff1/EVzEl8mrajaKYhcAyqSUHTB6Fex9D0zWpqsU9pCbiHIUY+Vxxj/636VzmhW0kvjnTryx4LOpZVOOfY/41eGm3NJk1orlbR+pngsxt4etXhkMibAATnI46EHuK6usbQhB/ZsMkKeWHUEj3x7cVsZ+bFfcQVoo+Un8TFHSiijIqyQooHSigAooooAKKKKAP/9P9Uqq3zbLOZ+mFJq1WRr84ttFvJyMhImOPXApMaPzF+L2pzXnjGbzmJCdMemf85rzqXUBaxGXglS2APU0eLtZOoeJry7LbwHIHcYFcZdXfnk28WcKPmPqc9q+Exq568mfZYVctGKNSKSS5ZJbhslwAR+n51y2uXLXV6sjnKOyoM9wvzH8OK1ZLtYrctn1P0APHNYcJM88UzkMbdtuMZAAUkk/57VlFa3NmTzXM6L9kVczMNgyfu8bifrmt4RrBG1wSSHDvyeCfvk/gM1gI8cc0d8hD7D8xJ4JJJJPtkfjXQRt5+nxrL/HCwHQ4dwsfH6k+1OWiBFZMXNlLPOf3jbYlHQfOWzx9F/Wutgt1ubSdVPyrAGX/AGlWMqB9c1h26w3tx5MOQqYkBAzngKMZ69atS6iIXuGhUqQse1en316H6ECsHd6Is14d9gUgZg32fCY/2W2hzVKZZJbK4tl272jZG6922L+ozS27mRBdDLSThiS2OpAP9PyrI1C6ngjZEHzMoyR3yc5/E0kndDNm0lVyyyybVYR4VV3Eq2CSf0/KrzHz0l05fnWVVQufXIzx3OTXPRSMs1kD8rZAYHsvPBx6L+tdfdQrarAbNcyXCvlge4zhuenzHFTJPoVoUbqKTUYmIO8tIERvVSwUNjr1zmrUq+Vq0kKDYrqYuw+WX5RjHfpVmWwEbwshKqFyP++vu/gKzrmVxdy3yHcETeOORtbYmD2wcEDvQrX0Jvobeoa2I18lcERyLAORypIDMPrnAo0q9AS1vJG3C3QsPQswOP1ArktWjW3iBcAtGsarjHDjPOP724856YrZhmWGCJZ0VvLJkKk8DjgH0Axlj7YqHqrRNIpLVnY+H7aQx/2pfA/u0PlqTkAE4Vvd3JwB9T0r03Qre5CMj/uzHw5H3slTgD0Pf2+tcTokiu4ku222tgw27+styR97HpECMDH3iB2re1PxImn2UiQt++Q4X18xgfT06nNbRtFXZjO8nZF24kingNhZDCHcHbH8Kj7oz3J4H6964/XtRW5vILC1bMVquQqj70hyOvoP1NQ201/cWoijBXzfm3E8hen5H9fxrXt7BrYi3sU3Tycl/wCLHfk9Pr2rmnVlN2RrGEY6so6fpbxpEswwSRnPXHtRrZS92vEoI4RO3J+Tdn/dB/ya6me1t9Ph33MmXfnPbHqvfHbJ615vqWpCaby7RcwQKWyRhQTwB6kk/pR7OV7C9otxiX9vZa7bk5lW2LZf+HkYAP1I/Ks7U9RUXe2D5i0weNR6JkE5PryazRY31y7PduNvLAdBzxwPoTyaz7y7ZtSYWeWKjYG5xznOB7ZPJ+taKlshc/U6nT79WjIuAInilYnB6oMY/EdD7Yp11fW1i0V26iVMvbyoBygckhh+HB9a5fymhXy1YlFDE45LFzk/pjFWAsl1HOrghlKFieQu3g/XGce5FappENNlvRL+50a8NtfMZ7WeJV8wgnMeONw647eoqbxM7XCxwWh3hCV2Z3AqvIwe4Ixj0rOaAz2W1ly1uTHnPJjbp+IYfrWdPFeQ232mMMstowfj+JM7c/kcH6CplK70LUbbl2z1O5s5jcoBNaSkGSPuFI5A+ma9BsZYtQguLJ33q4XY5OcgD5c+9eaMFZ3YL8kygtjsT6egNdTprvpm4qTJE4GfXB/zilGre1yZwPRdGtWhRVU8SAblJx0GP5VsTyusBuBkSQnD49OxqlpTOpVh86EAnPUZ5z9Ks6g01tE00ZyrfLz1UnsfUGvQprS6OCo9R0t+l1CtxD82eGx2I9q5q71KEo0FywVSPlb6eo68Hr6VjSXr2lwQCE83GAeFPtnsfesLV9RlQNHeREg9HC8n8OQfcjrXRFXMmzC8QajNYy+TcRiWPkjnkr/eRx1x6HNO8P39tLN5Ts21cEDcBjPcZ5I/GuD1K9jlDRQuCgOdnPykdwDyM96zdK8RT2E4EbM7xnA3HsfqD3rSUNBKWp9X6O6XalwRj1Jzg/TpXRAlgP3iqMYyvJ+mfU+1ebeFdaW/VfPjXnAyWzz7gY/WvSLXy5NvlKqkd8/4f41ycupvcvGRXIt4eNh5P8/x7Vk3cayeZkggjPp1IwK1JNqxFd3AGT259K5LVr5lHlQ/Kq8uxz0P+eBWdSSS1Lpxu9DndRt4iBOTjYRjrxz1/M1mTKpkZsZxIMBuxAwcfnVs3quzgEhY0z9STgcdO9YwvHlFwYSciMHJ7kEgt9cYzXC0tzrSZ2djetaSOY2O1SSQvI2nk8ewOSPT6VfubiVAU2b4/vLkfKc84BHQ9fx9q4e0uZgI5weUO5+fXgkf7px+FdXGeEaCUb4+sZ6YPf02n34raE+hjONtTyrxfoKxXyavYwxrDNgOMnG/OckDgH0PFVbKRbd0EkgkbtsOV/Pua9d1ezj1DTZtjbJcFk29yozgZ4OOoHWvJLWyjvALpwN7DG8DYT67l6AjvivToybjqcU42bPQNJlWUoEPXv8AzruLZ9jAD2ya8x0w7ZlZcqq8fh613kcjSIET75APtn/9VEnYSR1cbJcJvUDzEGPY5rFvXuIV8+LGGGCvuO1WrKfCsHGM/wAhRqMZKOMbt/PpyOo/EVyzd9jaKs9TjdR1F2QzRIZIDw4HJj9eO4rn/DsVvD4u0+YOcbwQyjkjPtjJFXbxJbF9okOwk5OOqnsfpVHw8qw+PrJLfAB2bk59eCO3NZU5e8mFVWi0fqvo/OmwNuD5QHcBjPHWrFw5jKsKi0sqdPgKdNg6jHb0qxcR+ZGQOor79fCfJvclVty5pOajtzlMHtUhHNUIfRRRQIKKKKACiiigD//U/VKvO/i1qLaT8NfEGoocGCzlYH3xXoleQfH0kfBvxSR/z4yfzFZ1XaEn5F0leaR+R8ty0ss8khyXk+ZvyOPxNZ0LtFtixmQsTn0+XI/n+lYFzqDmWKPI65Hrk5/+tU8N0JSJmbhF5zxyBtxmvh2ru7Ps00lYdqFy5U26N8p3MT6gcD8OtQLcGO0ZtxDdzjkkkf061KHjuNsmMgDaR/suCP0JGaxLV5CZBIS2FXH1LA5PtTS0BsnivH+zgclpUBX2IPp/Ku9tJwbGKLIX94kYbHOeWJOexNcZDbwsZPKziJQqgf3Vdef1re06YXLjzgcRO0qgHGQi7Rj8aU0rBF6mvbmZLZFVS7yDA46bpFPGParIWRRNdxgne5PP90rkA/StVzFZ3drclduXGVz0UyADPpUN+/2dYooxtVy5b1ARgOfqAa5tb6G2ljbifc1pHE2SJZGOMD5UXCj8cnP0p4sVlHmTgMI4trY6/ujxx7g4+tZlrtj1GBN4KqyHg8DcDn8iK32uI4r3epULLA+4Y67cbv5VNgZnadaC4ypCmQKyrkd9uMfmQas2t4WuhHKwCBBuBOQGXaCPqzsSfQVEr/Z2eInO+aNx6jzcEj2wRgfWktWt4o5rqQHYzSsp4wcyfp93FOwrs3SZnaS0243F85PTORnH1X9azikiziGIbpZtuEHZf7x/E4H4ntV6W5iivZZQu0qDu56855/GqtgWea9vWbZPIzIhPAQ7eST6KvQfT1rOK6lXKD2yXBlu8HMPRe7Ox2oSPx4Hqa6+3tLe2htA43PJ8xjOAo2EAFvXBAPPfntVzTrC3ggWYR5OVZN3HEecOx7ckn16VB50t0d0eI4cZaZ+mOo49CTwo68E1LfYerNNNWkZEZUDbFEkeeDliTuOe5J3ewq7pmiTXzebdKJNmHZj0LEc59uAMdags9PLSB5ydijzCpA3tg43v6Adh6/jXZTW0klukfmG2Rh8qj7yg9WPq3ueB2pcrm/ITlylISRRE2tmPOuScE4woA7n0x2Fb8cX2OHfJ88koy24cuT0B9FHZe/8oYINO0wJ5KgEYCRk5I7739SeuPSuZ1jxMqhltjl2Jw7dz65/w6VqlGCuZ6ydiHWZIFR57iQFySSzHOcDsOgHpXDtOkQSGMGR3O47z0GPvEenpnr6VVvNSkvrsbnE5UYB6Rrj09frjH1po84/LYANITueRucehOf0H4msnUN40ieV4bdXSV2kmuCNzOcEL14XsMc+p+lV1shcrJJHiCDd87fdLL/BEo7DHLHqamgs5EbAJeVieW5Yk8l2z69v/wBVb+leFrm8YtKT5YJz689QPc9zS9q2U4RS1M+LS47kAQ58tQeV/iJ6tn25rp7Hww8kRgWPAZQSo6gZyAT2Pc13Wm+HpHWOKNAAhHAH5ZNej2vh62tIBA+C2Mt7nqSfr0FdNHB1KvoclXFRhseNjwrEcELiNRnH949vw9KyG8PvvZ3X5CCpHXKnr+Ve5XFjDzF0XHb+Z9sVnXNmokJiTauMc9Rgbj/hXQ8DYxWLufOraC1rKN4ym1s+69R+taFppA2CIfeJ43dMk525+vSvT302Oa4BXiN41U8dCfX8aYdJ+yfuZo96sQwI5zg9vcYrCODd9TSWJ6GPbWstqiSEbdgwp9j2I9P8in394Gt2C9CMPH3Hr+XUGt2aRY7c+W26PJOeuPUMO2K4y4kt5/nuNqgnC84yQDgj2/8A1V304cuhyTlzamLqFqHi8qZ1KHlXIypU/wB7vj17g153rE95pcR8+Iz2hyrRk7sEdCp6jjv3r0W4kNrF5hbzM8qGGM54wwz0/WvIfEt19qQ3Vk6hj8rJ82Vx1XjnHcccV0wRi2zzPWbuFroXNpIxQ8gH7yexB649azYpg8gkjTc/8SnGGB9ORTLx3LbJGUr2PQ59uKoLaNKRuGCOjY4/L/OK3toZ31PdfB94IWRbWOONu4RIy5/FmJH4CvoPSr6ZolaRdo9B6+7cZ/Cvj7w3BdBU5XOdy7wcbRnnAwSc9MntX0BoWoTK0UbsWJxztbn2BPb6VxVYWdzqhK6PWCkzrufKlugI5OO+OwFcdq9o5/1mfLJyRnBb/Ae/5V0MGqhiyq+EX7zDkgf09gOamkmt57YySjABPLdz2H0FcVanzLQ6aU3E8seGfZckHfI5AHoB/T0qGK3dEMEbY3KVPHYL/ia7iXT4PNZCwAwHY9Mgep9z/KiysIZEa5YD5lZ8ew46/SuT2bvY6PaKxy8VsSIpJQVJAGPXs36VryG3jlt5IQU4IUnkbckFGHdSQfp2rZ+zn5VH/PYEZ9MHANU5rPLrGV2iMuyg9wSSR+WDVqNkQ5XZLFDItpMjRllIBKA5OR02H1H8Pr0rg2v7G5M0kbKxJIPGCSPXOMN6g9/evQVjENtucHf/AB55yh6Njvjow/EV4p4mSex1czQcSORz13D2PRh2zwR0I7120HbQ5KivqdlpjwsS78DAAFdvaQvEolPJwT9Djj9K80055JTGZeApBI9T2r0/T5fPwgO1exPfHGfzraaIRupCJF2qORyfpVhXUw7cElevsRVOW5eI+YvPHT168fSmm7QvHLG2C/4cjqDXJKSTNlFtGFrVisTtNgPE/wAxzxj1B/oa8+Se1svENvOMgxMGUcH6hT1+v516zMgu9275WyTtbnOeM5rktY8NPdQO1uA04Gfu9x2ot1iRJdz9GPA+u22ueH7W5t3MnyAHI5Bx0NdlX55/Cb4vap4bePRNSLJbg7cDBxjjo3TH0r7w0HXLTXbFLu1fepHXj+nFfX5fjoV4K2587isNKlLyNZEKOfQ1NRRXoHIFFFFABRRRQAUUUUAf/9X9Uq8b/aDbb8F/FTeli/8AMV7JXjP7QpA+C3iokZH2J/8A0Jayr/w5ejNaH8SPqj8UsCQrLIfmUA/iSRip8bLfyhhi6g8fUnP51Ru38t/KTAKqf++c8fnVtd7SxIBwF3Z/2ck18a1pc+uuR29xNAquvXZlh7delWIpRFYGZBwqsCMdR24HYd/bmrgtle1N+gyNw+UdcY5A/TApjaf5ZXY/7iTmNhxtJ5APqCOtTdFalewjuokkmQn5TxnoykrnB/CtXTrpV1eFlb92QUfP4ZFXk024miTzLcxNCpIAzjk8nA459OhrGn05/PLw5R0JGMHn2/D1pOSlcdmjsYpYr3TILwvvH2grIQOQFYn9c9PaujlW3nvD90O4QbegG8vjI9Mr+deSaffLpbm2mJ8mQZYA8Z6E/hXbvq9rcR+bnfIVEcmP4thZsj3OeD+FRODQ4yudXeMtteRiMAEL3wORkjH4YP1qJpViuVWTc4wVB7ASD5vxxVbSryPVmtpdwd1lABYfeGNvPueDUrzi/tUGQsmGbcO7RZX+WK57NaM0Kj30iDznJLufvAcZRWK/pinm5OyLTnyy8NJtHqN5/NjwKz2YXVq4UZY4Ck56E7SR9eKcJZo7tDAuZpnGAOTgKvT36VdgTOwKyraz3cimS4uCcAYyOQQMdskfgDzXSaTFIfLmm24KlwM/u+T8zt3OSOPUDsKyra1iESQuwuFi4ZMnaWPPzFevPYdQOa0w5urd443Hlg5lcDnC8HGOFHYKDwcDrnHNJ32LSNqa4W9Tz3DGCU+XEucM+eCwA6A+vYZNWYLCOe8hMhyqkCBF5GQOSFGee+egA9ScULC0mvZkvZGG2TKxogx+7HBwD64wSeMZ9a72wFzGGby0abZkAfcRe248Z9amKuwbsixFbQQEGbCow3FM5Z2HTd7DtnjNc9f6xcTXAgsUM0jn5mXn6AHv7f5Nbi6feXqO17JuQ85xgN6HHH4cAeg71UurMW8vl24AB4GBzz1J/wD1VUlLoRFrqc/9pngWUykPI24MYwX6dVT15+8x69uK5S8Fxejy3QgHjYME8diegHsK9JGiLy052IqjgsRwPXp+VPRNPjP7mJVCg/M3T9OT61m6cpPU1VRLZHnWn+GbuWZfMGQecdh9a7W18PMh+zRDJH3sjoTxkj19P8K6KxmtlbI+Zx0JHAJ/uj+prrrCS3tcNKBtJzxxz7+taU8OpPVkVK8ktjE0zwT5Cq8ikE8lm659fqa66LSIodsMfCL1an3GvWyg7+/YnAH9a5e+8SJIfLj+c9hnAPue4/nXdy0KW2px3rVNz0OK40+zj+VgdvQDnJ96z5NZWY+UgJd+MdT789gPWvN5dZ3rs44HRDxUqakI0I4HHCjAX8T1NU8x6RWgvqS3bPQjNbpEsSuGcnc2Oe/c1TnvYmDRnJ3H/Dqa4GXVJDkLJ7nbwPpmqEsl9McE4UfhWU8fJ9Co4SK3Z293qNtGdqEFj06ADH865+fVVjk2EmUEk4XsT1xWG6+UdrSFn7nqfpjtVSdoo8Ce4K54CqOSPr/hXNLGT6HRHDw6k9xLNcv8+1dx+5nOf+Ajr+NS2+l2hYXT7rmZOuQNoIHGB0znv0HvUUKQKwZSF9MAkk+5NbkYZ49rvtxyBwf06fnRTru+rCdJWsjnbrToLuLM581yNohT7gP0GWJPqTXnOtfDzxBqkjl4QIW4WMLl8EdCAR17biSO1e8W6nAKOeOpOOf6Vu2ckFqdxXczdSzdz6mu+niPOxxTpeR8qaL8ANXu5Wn1XFpGMbUzuP0OOp/r7c16Db/A1rpUtVZY4cAMAPuqOdoI5JY9eg/SveZNajQ7FCn1OcD8+v5Cta01NXA+6sajnA2j9eTXdCtGT1kc0qUktjxuL4HWX2mCeZlEEPzGJBjewGF3NnoO/b0ro7L4QEv5z3HkhioO3OAi8iNM4wO2a9Q/t21UgMwZh0Hf8AOaB4hkQb3URgev+Fb3ofakZWrdEc7F8NrK2hWKGRlAOdoAyx7ZJ6AVg6v4I1wCQ2Xluq/6pCcRr7k8s5z24Fd6/iyBT1OT0zirdtrsVyQQoH15P61MoYWekWUpYiOrR4jN4RvLWHydTnEhUjcFGNxPOSc9KqRhVkcOAEhUcZGBgbjnHHGRxX0JKthcp/pWHJ4wB2rAl0DSbkNHDGFUggjGcA9a554JL4WbQxT+0jxW1xHCgky7Fywz/ECev5VZlSM3L2znEgODgjqqgEjPrmu81bw2Qmy2cABSucc47fl6Vx82iXE088sCkb2LbyeVyAD16dK5J0nDc6I1FLUwNRDhGlU+Y6jeFHXvnb69OR7e9eRfEPTL24toNX0xC7j/AFkQIOR1yAepx6c17e2mXLq8dz8qE5Djs3UEZ9/0rjvF+nvd6JLFZQEycNHngsQckexz0rKLd7oqSVjz7wdcNf2+Y2yMZAJzj25r0uySeGXDLhX4/P8A+vXE+ErN7Z1ab/XPyVIGRnn5sd8/j617Db2nmxZcZ9a73HmV0cvNysekalFY/dyAR1wG/wAKnTTlGQw8xcg5A9uD9Oa07ezCgbujZB9eO9b9npUkaCUEg4xx3Fc0sO5PYtV7I56DTAi72jyg564P+cfnWtb6bFcA8Bh78EfiK7uzsIgRsXAPr1rYaOKGMhVHFdlHAdWznqYq58teLfCLWeoC9tElR+zA7xntjGK9H8BeNPE3hZzbsxlRSN4PIAIJxknrivQJdOguJlaQYwc8U5tA014fJCYBbJPcnjP54xURwNSFV1KMrEyrxlHlmrnpWm/FuzneGK6h2hiA7dME4HA5J616np2q2WrQmezYsgOMkEV8oTeG5Ul8+ybYR6/pj0re8PazeeH7qNbmVtmegJwc9eK9Wjj60HautO5wVMNBq9M+o6KzNL1ODU7VZ4WzkVp17UZJq6PPatowooopiCiiigD/1v1Srxz9oLj4L+KiBkixc4+hBr2OvL/jVaLf/CnxLaMM+ZYyjHA5xx1rOsr05LyZpRdqkX5n4fW/lvdLLL8yr27ggZP1Hep7e4WJ0k++A3U9MZzj6dawZfPW9ltYx843A/8AAsD+ldHaxJCEjkAIjHz55ycZx+Zr42pHl0Z9dF3JFfz5HhhGFmBxj+FwN6n9BWojefaNCNskUpyQQdyE8gkDPGc8de4qhZ2Qe7FypLLHkAD0xW6lmj+VDEm1Ao3Y7+hz7Vi2tC0SadPqMUgtJJQyx/wBjkbufXkd+n4V22mCAkiSM3Ibgo4BAI9GIzx61hraQXMsHk5WZMqWH4A59a2BEkcmyMk7BknHUDgcjnJzWFWz1RcXbRlhtD0K/JTULUiM4OxJBkn1BOCPyrnZ/AGjeZ52l6jLCWHCSx7h6jDLj+VdfAkMgLs3mIFB5IJDdx0zUrFJCHiBOe5JA7Vkqko7S/r5lcqfQ5G18Gazpd3Hdx3UE8W8OVVip+XB6MPr3qWWwu7HVRI8OLfngNuJDFt3T/ezXfW9iZl3zTHccfcJ/XrWkdP06Ebxh5TxluT6cU3Xn9odo9DyeHSdb8tVaB23EEEj+FenA6V02m6Bd3eoJJEjMseN7DIy2OEBIHy55bua7FruNX8yQbm6Y+n9KedanCBY3Ck8A9Mf59qTrtiUexBcaLKqrbuCqkY2qwRQc9vTPcmpbexjRIracgQxsT5a/KjEdPcgc/jVEyfaD5c8o685yRXRWNva5Qt8zZySf8Kwu3sabbnQaZYqkKQxEIzAZYLwB6DP5V2ELW0Uaw26F1U9CcjPUliepNcNJqHkE7NpwMcNk/jzTv7VncEOfoq9fzrZTsjJxvqdlc3sa53EMzdk6+wrLkv1gH3PLLAcdT7+/wCPFcjLrEyt5UeUPY5/wrImkklBxOArehLN9KmVRvYqMF1N6810SziBGL7cZwPu+1RRJd3Tgs2xRxwMEfnWZCFUbly+3uy8/hWlHtX19ueTWDTb1NVJLY6i3MNquAwPbAI/VqdNqT/xFWx0VTx+J71ySXcAO1gMD8cmrS38Eg8uOFmZhj0H51abWxD13NiWa6u1DMQF7Y4x9arpbzZx8oHODjqfw5rMIULhgR7E8D6D1q4sicKXKj39u2KVrjv2NWGMKB8vTj5jgA/QVYWO3iHmTMu7/PassOF+dn2r2z1/AU19RtoiRbx4duNz9fwFV6CNQzRb1bGMdA3r9BVSaUuhZnwPZSf04zVSO6IBctuYdh/XGKryXdxdsVMhKjsSF/QVEikicTRoCqxsxzyWJBP4DpVhYc/vpESIn2x+p5rKlaS3BcThAPTB4/U1mnUo92ZZDMc8E9PyrNRbKudMRHIco+BnqoPb61rRzY+RTwefTmuNi1GSYgIMk/7PC+/Na9vjcGZ/mPGTWqViZHWwSDfkjfj8qu795ywXHYDnH9K5+3uY412liR7nithbyJQGJA710RMJJ3NSINkFVUe5GavMHcgTMX9c/KKwlv4ACzMTjnIzmpP7UjXBjJx/nrzWimkZ8rZuc4IjTBxg46VQl39T87f7RrMn1CaZQMkp9MVX+2vs2qAAevGTUTqI0jBmv8ztiUggdhWlb7jINik+3QVz8M2Puvitm3llTBGR74p0pK9yaiZ1kMT4ElywQdQBTZbyJAY43z2wOlYnnM6hnfOR3OP5VA0oiyHIVR/dGc/jXofWbKyOP2OupqNcN/rJnUAeo/pWJfatbEeWqbyP73SoJZIuGKs/sScfpWXLP2ZAg6cdf0rmqYh2NoUVco3N09xIHmJ29Aq/KPz4rMuWuLpCkZ27uMjqR+PNX55ogMGJn7DIP8qqmSeRsEGNe+BjP9a43Wex0qmjMs/D8UMomkcK3HCjHPue5rt7GGKE5B4HU9Kz7aNEUGNcY7t1/CrouEAwRlfbmuqlWkupz1Ka6HTWos5G3cKRzjsK3I5FTJWQdOB6ivPGuyn3F2g+poGsSRLsTDdhurtp4xLRnLPD3PVYb2MDAOW9+gq7HJHIclgTXi8OszRuFkfZnoByTXoGj6q8kfKkZ9RjNdtDFqbsYVcM46nSSxjdlTzU1uw6MaqyXJY7UGTjJqSF8oG6HvxxXVfXQ5nF2L7EYOKxdSiW4iKggHGcj1rSdg6lT0PXPFcbqOqraylFk46gdf1oqTVtRRi+h6F8L9anjv30qc5x0PPSvoavl34ZW8t34me8j+4ByexzX1EK9DAfwjhxNufQKKKK7TnCiiigD//X/VKuQ8f6Z/bPgvV9LyV+0W0i5HXpmuvqG4hW4geFxlXBBH1pSV00OLs7n4A6lpwsvEN9A4xIJSvPXjPJHXvVqCZZP3MSYXGGx3yDyfYYr3H9pzwNJ4G8fPeWybLbUMshAwAW4NeGW9s9pAtx91iMqOvOeBz6ivj8dTcarTPrMLUUqakjZ0nTpAxkAyvJZugBAzgew5rYHlvEwUjywkZcng4PJHsSOB9aoreqBNAgPlsTgE43LjYpP86rtfItvIjrhskBTwSwwV/HA6VyWbZvfQ6OG4TTwbfjdJtdTjhWBzz9ScGpbeeZ53VGG4/KRkY69B9a5qFQtv5spLMV+XPTgir5litysiAhjt69ee34VnKOpSZ0LMkUxtZQzdNuOmDgg4610cVuBB8xLKoJ+XjBrmXn8yHzIhhhjPfp/wDrq8t8YAGlIjfoR1BFQ43RV3c6aO+8tNsYHAHf171VuLxZQ23IbHXoM/5Fc5JJaeZINgBZQcgHngHp6Y9OlVVuITMPsxbAPOG4/X0qOW4zVa7lUjfISBg9OxqaO4jeTzcbT27k1T81Ng4UZ7E5+nT1pSkojWQgBccruHT881PIUpG9HfAv8iKoXuSTmpVuJT8rSAhv7prmG+0OoSKNRk565yP8KmiWQRn7VlVXqVPH0470vZhzHQx6nDbEnkdvUVeTVJZFV49wQewwf/11yi3ljHhVXhh/FjJFLJqSqmI3VcdB2/Ok4Bc64XoZ8yKQSTnIH6VIt3bQjIP3fTA/+vXCJqTNyVJ9cEf061YGoMjfMDk8cYx+gzS9kx3R38N9CQCZME9Tgk/nUjanbRAbQGHr3rikuJHwJHAJ5/8A15NS/b1hTfwcd6OUR2a6nG4yi49SQBikF3CxwhJyeey5/ma4OTUVY7mcAEZGCP8AGlS83/efjHqM80nF7jPRPMjAzv59sf8A16WG7tlP7xgceuTXCR3RZf3Qb645/WpV2kBpGOO45FS7oDuzqNqcsWC546c1Vl1G2hDFFyT0PGa4xrqONti9O1PSZCPl+VuvJGam9xnSvqyldhGB3xms6410QKVjXGfQdKz3fgbiGqi6zyHjhT69P0qlFMLkr65JM7eWjOT9MUxL2/3F1Qfj2/OodgQ4Y/MPSpY3QY3NjPbg0+XsPmLi6hdqCFySeflx/wDrq3DqV3ECX+UD1z3qAW/AZhuB6YIBpj28VwAIXBkB6NhsD6ip0KOis/EAjX55GP8Ad2LnP410MOvKyhihbjqxAFcFbRzwyhJpYzjtgqRWoJNp4dsY5xhhSvqJo7FNTVmJfYR6A/8A1qurrCkeWsDMO3GP5DNcdFJKy/6MRx0IVc8/72BU0l1eRcs0iknuVQZHrxg/nRcVjrptTm+UGPA6/e5/KrC30zfcjCAj+IiuPN1qBUtFIip33gH9RTre8Df8fLRPj+6x+lQyrHcR3B7kJtrVgutxUPnB6NXnkWr2aS7bhmxnjcrEfz/lWxDqFpMP9HZsdwoxn86uLa1IkrneC5k2hI2xg/Tj61SknmQ5YEA8jvWAl/BkRsyoexk5/rUxvHWM4kVsHtjj861U2zHlsWzPMQww34n+lZk7XjtySqnsWx+maZJeyAAiSPB/vc/qKozaiYifLn2n0HNS9S0TSRXGcMWcjryB/I1ArPFjcEUn/aOayJ9RkUbmkaQZ5UDms06lAmCylD7kfyqNizv4bg52uQTWgLlCu0/L+GR/OvObfXF3EBRjOck10Ca5A0eFaJT7kg1pGbInFHQyMjNtLY49OazpWAPy7vr2rObU1wFUJIT3XJ5/Gq5vJpSRJiMLk5Jxn0Ap8zZnYeshSUskjBie3+Sa2rLWZ7Z8yM2c8ev65rAaQsudzAew4/oKbvA+XnH+fQYq4VGtglFPc9as/ElvjPlszE9WfJ/GurtdajLkS/LkDvxzXz7BqHkTB0/x4rqNOuNS1Fxb2sbZkPPHbHAr1sNipy0POxFKMdT03UfEtrCTAP3jP8uBjj61zuleH9R8S3iiKErEWwWxzzXp/gv4UO6C81TguQ3PX8q9/wBN0XT9KiEVpEFx7V7FHBTqe9U0R5VTEqOkDE8I+F7fw9YLEq/P3Peuxoor2IxUVZHA227sKKKKoQUUUUAf/9D9UqKKKAPnn9oj4Uw/EjwkWt0zfWJ8yL1bHVfxr8mfFVje6JEbG8iZJ4iTgg5O1gv5cgV+9bAMpU96+Zviv8APD/j2GSaNBbXjDAdR755/GvLzDA+1anHdHpYHGezThLY/JqCYSWKo3zTI5Vj6gjgfgazDetJqCWrnd5hYpn+HCZBz+le1fEL4E+LPhvGXmjNzaqeJUGc5Bzmvne5tb4TRXKIwMZGTj2wa8T2Eoyakj2Y1YyinFndQ322KJD/BuBHXryPzxTGuLiRhbt8zOQ6joc+lc3pmpKE/0kFXQ4II64/zmrEVys80ckb5aMlg305IP9K53Bpu6Nk9NDsrfU4oZWKEKdpYAZPAPTHr7VrRahbupud2Y2IBZSCFz1yOo59RXmsupGNwMbQQuJPQj1/HvVoX4RpJG3bm5zjG4H1I61Dplcx3+54iAx3AsCCxIyPY017v7OC5KsoPJ749+K89fXWiKkDaoPKqSB+PX9K0INfNxlh5ZUfwhiGx9TU+yktWiudHVf2las48qUKT74B/Gte2myMRzEP1ySCvv24riIr63Zy0bKsigBlkOAR9eKb/AGtFHmGF1icHgbgQefUVEoN6IpSXU7WUynEhl74ABP8A7LU0eo2JRUuA+TwME446mvOl8RxxTNs+fP3ieuavf8JBp6hsAoXGQEUde+eafs5W1QcyPQBc2wLISVA5yeAR6ZbOaadRsY8fZ3B56bg3Hvj+Veet4p3bfP8AMeLjIJ5H0yTSW/iGzdtq26uOTlmIP5YIpOi+qFzpHof9rSA5MZfkngHgfhUT6tJCChhYAdCwOOffiuIutTec4hLc+j8D8gKgiuyx2zXGW6D5iRSVKw+c9FjmaUBt+09gDgAe+etWfnXAllOScfKw/lXn0WoR28nliVdx7gn+taX9oJtCySFj2OeB+FTKmxqR1StaqxKFsdATg9OtPaexOTuxtPb/AOt0rlXuFlVkTDE9BnrUAE2Vbyh0weDx9fSjl8xnapqD5Bi+YDv0/LPWpDdSTLtCdfVsGubs5TjMjjI471ea/YfK649Mc1MoAmaQWbhpdpK9+OB+FT+dOPmRxtPXoOn05rHa+i7ptI7H+dVzqUSy7GXgdSTxmlyeQXOnS6hT5SM5Hfn+dRTakqrtBA9Oa546jbsTFCp3MBznIGaoy3F1uIVPlwME9PzoVNBc6Zb63cYDfMcA5IAFSC+to3G9iM9cc4/IVgxXgMYLOmO6HOT9DiqrXwk3LHblj65/wFPkC51s98GTdG7H0HP+FRxX1xHh88n14P4HFc4t/LCuHmEeR0bqAPoahfWE3ZFyoJ4KhSf1o9n5Dudn/beonBLgYOPm6/1FSnWJh805Td6jGfxxXHQ6orPsYsxzjdWit3agExn5/TOefyqPZLsPmOmXXkBPJJ6DDZp8ut3ZAUlhkj+70/Ec1zpnkmwOwHHHf1qXzJIRtRGwR1PPP5CmqC7Cc/M1I9buoVMkJA2/3cH/AL6UH9RUkOuXskmFSN1b0wck+hIFcxJvkOZTGGx/HHz14w4pht3ucg7o89dihl/kDVfVm+hPt0up6BHfqoG9fLbGSudp49MjFaVlrthGAfOYHvgq2Pwry7yxb/u3LMB35x+I5qazaWR1jtI5N3YKN/P8/wA6f1GT2RLxUV1PZI/EUAO77a21uoKD8+DUp8T2A5EzSf7WzI/ka5XSvBvjzVF82y0maRGGQ5h2gn6k13OnfCD4l3i4a3MB9WwP5dvrVxy6p2/r7jCWNprdmTJ4hsZEMIbzR1yu0Y/A4rOkvPtGBZY3H1cKRj0r06P9njx+xBkmjIPrj8jxWsf2d/GyDe0ylem09PqK2WVVukTJ5hS7ngMguA2XkZGPZmyP0rJkYQvv8yMA8nLMT/8ArzXr+pfs8/EVZG8kRyIxzggdPpjBxXAX3wJ+Kcc5SLTiRnIZTjp70/7NrdUWsfSf2jn49StY5Pn4IPVGx/OurtdZteiStk9ScHFSx/Az4o3EqLDp+MgYD+/XkV7l4U/Zu8QPGr66UiY8sE5qo5VVl9kynj6a+0ePLd+fzHK2T6qDVi3t9Qdyke2XIz93JODX2FpnwH0q2UC5O84xXY6X8JtCsJA5UEDtiuuGR1Hucsszj0Pie20LXLqQi3tnOeflXH867HS/hZ4k1Jl86GRPqCK+6LPQdLsVCwQKMe1aqxxr91QK7qeQ0lrNnPPNqj+FHy1oHwRkQA3vy+tez6D8P9J0bayxgsOenevQaK9OjgqVL4UcNXE1KnxMRVCgKowBS0UV1nOFFFFABRRRQAUUUUAf/9H9UqKKKADg0wpn3oIpvzDpQBkar4e0vWrZ7PUYVmifqrDIry68+Avw5uIXh/s2Nd/cCvaPMI608MGqZQi90XGclsz8/wDxj+xZYanetNot2YImOdp7fSvFNe/Y58aaD582iSC7UrwOhz61+tuKjKVzywdJq1jojjaq1ufgb4g+Hvjrw3dtDeafLHtznKEj+VcM5urYtDfRtGDx3ytf0N3ejaXff8flrHLn+8oNeaeIPgV8MvEhZtQ0iIM3UoNtcs8rh9lnVHM39pH4QvdeTIyqdwHI3D+tU3vJGIKKAtfs9c/sffCOdy32Z1B7A9K5PUf2H/hrcvusppoAeozms1lzRr/aUD8ivtby/I+5x79q0YU81PLjBTnsOtfrRb/sSfDmJQr3EzevNb1v+x78OLZQkbSDHc8mn/Z7sH9oQPx9XTXceWqn8Rmta20O9YgQKxPptzX68W/7JHw6iffNJPIPTgV19j+z14E00AWlvnH98Amj+z5Pdk/2jHofj5D4e1GXbuteFAGSOv8AKmy+G5Oph2HPPHFftAfgv4ObBaziDAYyFAzVab4HeC5wQ9qv5VKyrXcf9prsfjbF4cuQCqoTnBPbn8qVvDl87hljIx6f/qr9hW+AHgY9LfGfSq8n7Png1j8qbah5U+jGszXY/ISfw7fMf9W3H1/woi0e+i5EbFvfp+NfruP2evBw6x5pp/Z68HdoRS/sp2sx/wBpo/Ia5tdQziCEqW5bv+RqktzqNtmNkbjGNwyB785r9err9nrww6lYYgPwrj7/APZg0e5B8squfaollTtoi45nHqfmPDeXdwC0uAx5yoH+TWtbi6kbLy8dgy5BP5ivvGb9kaFixjuAAfQVAP2Tnh4ScEVxzyytsonRHMaXVnxBKbhV3SDn24rlby/u0lO2F9g6hiMfgetfoev7LNx0eYEVK37Ktuy4lkBJ9qUMrrdYjlmVLoz8zpNQuXbZAjBE/PP14qzBrd+oXaMMp4LqOM/Wv0kX9k3TmPzMAK04P2SdDXhm49MV0/2bO1uUy/tGHc/M8yavMm8DoeNi4xSm71/zQ8wkHTlSVyB9MV+pUH7KnhmPAJOB2rbh/Zn8IwjBjz9aSyur2QnmVPuflVHqGoPhWWZzg8k9j9akt7C8mG8QM3OSST+FfrFD+zt4Lh5EA9627T4K+DrPG22Q49s01lE+rsS80h0R+V+n+Etfv2UWVg5z15NezeHfgx4wv1UpamLOM5NfpDpngXw9aYMUCjHTiuzt9Ms7dQIowMe1dVPKIfaZz1M0n9lHwhpX7OeuSov2namRzzXd2H7M1vsC3cufWvsIIo6CnV2wy+jHock8dVl1PlyD9mHwtjFw5I9hW/Zfs2fDy25lhaT8cV9CUV0KhTW0TB1pvdniEf7Pnw3jO4Wjk+7mu40r4deENGAFhp8SEd9oz+ddvRVeyh2Jc5dyrHZW0ShI4woHYCpfKjHapaTAqrIm435R0FOAFGBRimIQqh6gUbE9BUbA9qAGzSGSbFHQU6kHSlpiCiiigAooooAKKKKACiiigAooooAKKKKACiiigD//0v1SooooAKTFLRQAwqDTdvcVLRigCMFhT91LgUYFABkUdaTFGKAA7u1MLMO1P5paAGKxPan0UUAFFFFABRRRQAUUUUAFFFFABRRRQA3mkwafRQMZg1GYsnmp6KLBcgEIFOCYqWilYLkew00x5qaiiwXK/kg0eQvpViiiyC5EsSjoKlAxRRTEFGaKMUAJmlzSYNGDQMWikwaUUAFFFFAgooooATFGKWigdwooooEFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFAH//T/VKiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooA//Z"}]} ================================================ FILE: test/apis/realtime/multi-container/app/main.py ================================================ import base64 import requests from fastapi import FastAPI from pydantic import BaseModel class Request(BaseModel): image_url: str app = FastAPI() app.server_url = "http://localhost:8501/v1/models/resnet50:predict" app.labels = requests.get( "https://storage.googleapis.com/download.tensorflow.org/data/ImageNetLabels.txt" ).text.split("\n")[1:] @app.get("/healthz") def healthz(): return "ok" @app.post("/") def text_generator(request: Request): # download the image dl_request = requests.get(request.image_url, stream=True) dl_request.raise_for_status() # compose a JSON Predict request (send JPEG image in base64). jpeg_bytes = base64.b64encode(dl_request.content).decode("utf-8") predict_request = '{"instances" : [{"b64": "%s"}]}' % jpeg_bytes # make prediction response = requests.post(app.server_url, data=predict_request) response.raise_for_status() label_id = response.json()["predictions"][0]["classes"] return {"image_prediction": app.labels[label_id]} ================================================ FILE: test/apis/realtime/multi-container/app/requirements.txt ================================================ uvicorn[standard] fastapi requests ================================================ FILE: test/apis/realtime/multi-container/build-tfs-cpu.sh ================================================ #!/usr/bin/env bash # usage: build-cpu.sh [REGISTRY] [--skip-push] # REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test image_name="realtime-multi-container-tfs-cpu" "$(dirname "${BASH_SOURCE[0]}")"/../../../utils/build.sh $(realpath "${BASH_SOURCE[0]}") "$image_name" "$@" ================================================ FILE: test/apis/realtime/multi-container/build-web-cpu.sh ================================================ #!/usr/bin/env bash # usage: build-cpu.sh [REGISTRY] [--skip-push] # REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test image_name="realtime-multi-container-web-cpu" "$(dirname "${BASH_SOURCE[0]}")"/../../../utils/build.sh $(realpath "${BASH_SOURCE[0]}") "$image_name" "$@" ================================================ FILE: test/apis/realtime/multi-container/cortex_cpu.yaml ================================================ - name: multi-container kind: RealtimeAPI pod: port: 8080 containers: - name: web-server image: quay.io/cortexlabs-test/realtime-multi-container-web-cpu:latest readiness_probe: http_get: path: "/healthz" port: 8080 - name: tfs-server image: quay.io/cortexlabs-test/realtime-multi-container-tfs-cpu:latest readiness_probe: exec: command: ["tfs_model_status_probe", "-addr", "localhost:8500", "-model-name", "resnet50"] compute: cpu: 1 mem: 2G ================================================ FILE: test/apis/realtime/multi-container/sample.json ================================================ { "image_url": "https://tensorflow.org/images/blogs/serving/cat.jpg" } ================================================ FILE: test/apis/realtime/multi-container/tfs-cpu.Dockerfile ================================================ FROM tensorflow/serving:2.3.0 RUN apt-get update -qq && apt-get install -y --no-install-recommends -q \ wget \ && apt-get clean -qq && rm -rf /var/lib/apt/lists/* RUN TFS_PROBE_VERSION=1.0.1 \ && wget -qO /bin/tfs_model_status_probe https://github.com/codycollier/tfs-model-status-probe/releases/download/v${TFS_PROBE_VERSION}/tfs_model_status_probe_${TFS_PROBE_VERSION}_linux_amd64 \ && chmod +x /bin/tfs_model_status_probe RUN mkdir -p /model/resnet50/ \ && wget -qO- http://download.tensorflow.org/models/official/20181001_resnet/savedmodels/resnet_v2_fp32_savedmodel_NHWC_jpg.tar.gz | \ tar --strip-components=2 -C /model/resnet50 -xvz ENTRYPOINT tensorflow_model_server --rest_api_port=8501 --rest_api_num_threads=8 --model_name="resnet50" --model_base_path="/model/resnet50" ================================================ FILE: test/apis/realtime/multi-container/web-cpu.Dockerfile ================================================ FROM python:3.8-slim ENV PYTHONUNBUFFERED TRUE COPY app/requirements.txt /app/requirements.txt RUN pip install --no-cache-dir -r /app/requirements.txt COPY app /app WORKDIR /app/ ENV PYTHONPATH=/app ENV CORTEX_PORT=8080 CMD uvicorn --workers 1 --host 0.0.0.0 --port $CORTEX_PORT main:app ================================================ FILE: test/apis/realtime/prime-generator/app/main.py ================================================ from typing import DefaultDict from fastapi import FastAPI from pydantic import BaseModel app = FastAPI() def generate_primes(limit=None): """Sieve of Eratosthenes""" not_prime = DefaultDict(list) num = 2 while limit is None or num <= limit: if num in not_prime: for prime in not_prime[num]: not_prime[prime + num].append(prime) del not_prime[num] else: yield num not_prime[num * num] = [num] num += 1 @app.get("/healthz") def healthz(): return "ok" class Body(BaseModel): primes_to_generate: float @app.post("/") def prime_numbers(body: Body): return {"prime_numbers": list(generate_primes(body.primes_to_generate))} ================================================ FILE: test/apis/realtime/prime-generator/app/requirements.txt ================================================ uvicorn[standard] fastapi ================================================ FILE: test/apis/realtime/prime-generator/build-cpu.sh ================================================ #!/usr/bin/env bash # usage: build-cpu.sh [REGISTRY] [--skip-push] # REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test image_name="realtime-prime-generator-cpu" "$(dirname "${BASH_SOURCE[0]}")"/../../../utils/build.sh $(realpath "${BASH_SOURCE[0]}") "$image_name" "$@" ================================================ FILE: test/apis/realtime/prime-generator/cortex_cpu.yaml ================================================ - name: prime-generator kind: RealtimeAPI pod: port: 8080 max_concurrency: 1 containers: - name: api image: quay.io/cortexlabs-test/realtime-prime-generator-cpu:latest readiness_probe: http_get: path: "/healthz" port: 8080 compute: cpu: 200m mem: 128Mi ================================================ FILE: test/apis/realtime/prime-generator/cpu.Dockerfile ================================================ FROM python:3.8-slim ENV PYTHONUNBUFFERED TRUE COPY app/requirements.txt /app/requirements.txt RUN pip install --no-cache-dir -r /app/requirements.txt COPY app /app WORKDIR /app/ ENV PYTHONPATH=/app ENV CORTEX_PORT=8080 CMD uvicorn --workers 1 --host 0.0.0.0 --port $CORTEX_PORT main:app ================================================ FILE: test/apis/realtime/prime-generator/sample.json ================================================ { "primes_to_generate": 100 } ================================================ FILE: test/apis/realtime/sleep/app/main.py ================================================ import time from fastapi import FastAPI from fastapi.responses import PlainTextResponse app = FastAPI() @app.get("/healthz") def healthz(): return PlainTextResponse("ok") @app.post("/") def sleep(sleep: float = 0): time.sleep(sleep) return PlainTextResponse("ok") ================================================ FILE: test/apis/realtime/sleep/app/requirements.txt ================================================ uvicorn[standard] fastapi ================================================ FILE: test/apis/realtime/sleep/build-cpu.sh ================================================ #!/usr/bin/env bash # usage: build-cpu.sh [REGISTRY] [--skip-push] # REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test image_name="realtime-sleep-cpu" "$(dirname "${BASH_SOURCE[0]}")"/../../../utils/build.sh $(realpath "${BASH_SOURCE[0]}") "$image_name" "$@" ================================================ FILE: test/apis/realtime/sleep/cortex_cpu.yaml ================================================ - name: sleep kind: RealtimeAPI pod: port: 8080 max_concurrency: 1 max_queue_length: 128 containers: - name: api image: quay.io/cortexlabs-test/realtime-sleep-cpu:latest readiness_probe: http_get: path: "/healthz" port: 8080 compute: cpu: 200m mem: 128Mi autoscaling: target_in_flight: 1 ================================================ FILE: test/apis/realtime/sleep/cpu.Dockerfile ================================================ FROM python:3.8-slim ENV PYTHONUNBUFFERED TRUE COPY app/requirements.txt /app/requirements.txt RUN pip install --no-cache-dir -r /app/requirements.txt COPY app /app WORKDIR /app/ ENV PYTHONPATH=/app ENV CORTEX_PORT=8080 CMD uvicorn --workers 1 --host 0.0.0.0 --port $CORTEX_PORT main:app ================================================ FILE: test/apis/realtime/text-generator/app/main.py ================================================ import os from fastapi import FastAPI, status from fastapi.responses import PlainTextResponse from pydantic import BaseModel from transformers import GPT2Tokenizer, GPT2LMHeadModel app = FastAPI() app.device = os.getenv("TARGET_DEVICE", "cpu") app.ready = False @app.on_event("startup") def startup(): app.tokenizer = GPT2Tokenizer.from_pretrained("gpt2") app.model = GPT2LMHeadModel.from_pretrained("gpt2").to(app.device) app.ready = True @app.get("/healthz") def healthz(): if app.ready: return PlainTextResponse("ok") return PlainTextResponse("service unavailable", status_code=status.HTTP_503_SERVICE_UNAVAILABLE) class Body(BaseModel): text: str @app.post("/") def text_generator(body: Body): input_length = len(body.text.split()) tokens = app.tokenizer.encode(body.text, return_tensors="pt").to(app.device) prediction = app.model.generate(tokens, max_length=input_length + 20, do_sample=True) return {"text": app.tokenizer.decode(prediction[0])} ================================================ FILE: test/apis/realtime/text-generator/app/requirements-cpu.txt ================================================ uvicorn[standard] fastapi transformers==3.0.* -f https://download.pytorch.org/whl/torch_stable.html torch==1.7.1+cpu ================================================ FILE: test/apis/realtime/text-generator/app/requirements-gpu.txt ================================================ uvicorn[standard]==0.16.0 sentencepiece==0.1.94 fastapi transformers==3.0.* torch==1.10.2 ================================================ FILE: test/apis/realtime/text-generator/build-cpu.sh ================================================ #!/usr/bin/env bash # usage: build-cpu.sh [REGISTRY] [--skip-push] # REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test image_name="realtime-text-generator-cpu" "$(dirname "${BASH_SOURCE[0]}")"/../../../utils/build.sh $(realpath "${BASH_SOURCE[0]}") "$image_name" "$@" ================================================ FILE: test/apis/realtime/text-generator/build-gpu.sh ================================================ #!/usr/bin/env bash # usage: build-gpu.sh [REGISTRY] [--skip-push] # REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test image_name="realtime-text-generator-gpu" "$(dirname "${BASH_SOURCE[0]}")"/../../../utils/build.sh $(realpath "${BASH_SOURCE[0]}") "$image_name" "$@" ================================================ FILE: test/apis/realtime/text-generator/cortex_cpu.yaml ================================================ - name: text-generator kind: RealtimeAPI pod: port: 8080 max_concurrency: 1 containers: - name: api image: quay.io/cortexlabs-test/realtime-text-generator-cpu:latest readiness_probe: http_get: path: "/healthz" port: 8080 compute: cpu: 1 mem: 2.5Gi ================================================ FILE: test/apis/realtime/text-generator/cortex_gpu.yaml ================================================ - name: text-generator kind: RealtimeAPI pod: port: 8080 max_concurrency: 1 containers: - name: api image: quay.io/cortexlabs-test/realtime-text-generator-gpu:latest env: TARGET_DEVICE: "cuda" readiness_probe: http_get: path: "/healthz" port: 8080 compute: cpu: 1 gpu: 1 mem: 512Mi ================================================ FILE: test/apis/realtime/text-generator/cpu.Dockerfile ================================================ FROM python:3.8-slim ENV PYTHONUNBUFFERED TRUE COPY app/requirements-cpu.txt /app/requirements.txt RUN pip install --no-cache-dir -r /app/requirements.txt COPY app /app WORKDIR /app/ ENV PYTHONPATH=/app ENV CORTEX_PORT=8080 CMD uvicorn --workers 1 --host 0.0.0.0 --port $CORTEX_PORT main:app ================================================ FILE: test/apis/realtime/text-generator/gpu.Dockerfile ================================================ FROM nvidia/cuda:11.3.1-cudnn8-runtime-ubuntu18.04 RUN apt-get update \ && apt-get install -y \ python3 \ python3-pip \ pkg-config \ build-essential \ git \ cmake \ && apt-get clean -qq && rm -rf /var/lib/apt/lists/* ENV LC_ALL=C.UTF-8 LANG=C.UTF-8 ENV PYTHONUNBUFFERED TRUE COPY app/requirements-gpu.txt /app/requirements.txt RUN pip3 install \ --no-cache-dir \ --extra-index-url https://download.pytorch.org/whl/cu113 \ -r /app/requirements.txt COPY app /app WORKDIR /app/ ENV PYTHONPATH=/app ENV CORTEX_PORT=8080 CMD uvicorn --workers 1 --host 0.0.0.0 --port $CORTEX_PORT main:app ================================================ FILE: test/apis/realtime/text-generator/sample.json ================================================ { "text": "machine learning is" } ================================================ FILE: test/apis/task/iris-classifier-trainer/app/main.py ================================================ import json import pickle import re import os import boto3 from sklearn.datasets import load_iris from sklearn.model_selection import train_test_split from sklearn.linear_model import LogisticRegression def main(): with open("/cortex/spec/job.json", "r") as f: job_spec = json.load(f) print(json.dumps(job_spec, indent=2)) # get metadata config = job_spec["config"] job_id = job_spec["job_id"] s3_path = None if config is not None and "dest_s3_dir" in config: s3_path = config["dest_s3_dir"] # Train the model iris = load_iris() data, labels = iris.data, iris.target training_data, test_data, training_labels, test_labels = train_test_split(data, labels) model = LogisticRegression(solver="lbfgs", multi_class="multinomial", max_iter=1000) model.fit(training_data, training_labels) accuracy = model.score(test_data, test_labels) print("accuracy: {:.2f}".format(accuracy)) # Upload the model if s3_path: pickle.dump(model, open("model.pkl", "wb")) bucket, key = re.match("s3://(.+?)/(.+)", s3_path).groups() s3 = boto3.client("s3") s3.upload_file("model.pkl", bucket, os.path.join(key, job_id, "model.pkl")) else: print("not uploading the model to the s3 bucket") if __name__ == "__main__": main() ================================================ FILE: test/apis/task/iris-classifier-trainer/app/requirements.txt ================================================ numpy==1.18.* scikit-learn==0.21.* boto3==1.17.* ================================================ FILE: test/apis/task/iris-classifier-trainer/build-cpu.sh ================================================ #!/usr/bin/env bash # usage: build-cpu.sh [REGISTRY] [--skip-push] # REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test image_name="task-iris-classifier-trainer-cpu" "$(dirname "${BASH_SOURCE[0]}")"/../../../utils/build.sh $(realpath "${BASH_SOURCE[0]}") "$image_name" "$@" ================================================ FILE: test/apis/task/iris-classifier-trainer/cortex_cpu.yaml ================================================ - name: iris-classifier-trainer kind: TaskAPI pod: containers: - name: trainer image: quay.io/cortexlabs-test/task-iris-classifier-trainer-cpu:latest command: - python - main.py compute: cpu: 200m mem: 256Mi ================================================ FILE: test/apis/task/iris-classifier-trainer/cpu.Dockerfile ================================================ FROM python:3.7-slim ENV PYTHONUNBUFFERED TRUE COPY app/requirements.txt /app/requirements.txt RUN pip install --no-cache-dir -r /app/requirements.txt COPY app /app WORKDIR /app/ CMD exec python app/main.py ================================================ FILE: test/apis/task/iris-classifier-trainer/submit.py ================================================ """ Typical usage example: python submit.py """ import sys import json import requests import cortex def main(): # parse args if len(sys.argv) != 3: print("usage: python submit.py ") sys.exit(1) env_name = sys.argv[1] dest_s3_dir = sys.argv[2] # get task endpoint cx = cortex.client(env_name) task_endpoint = cx.get_api("iris-classifier-trainer")["endpoint"] # submit job job_spec = {"config": {"dest_s3_dir": dest_s3_dir}} response = requests.post(task_endpoint, json=job_spec) print(json.dumps(response.json(), indent=2)) if __name__ == "__main__": main() ================================================ FILE: test/apis/trafficsplitter/hello-world/.dockerignore ================================================ *.dockerfile README.md sample.json *.pyc *.pyo *.pyd __pycache__ .pytest_cache ================================================ FILE: test/apis/trafficsplitter/hello-world/cortex_cpu.yaml ================================================ - name: hello-world-a kind: RealtimeAPI pod: port: 8080 max_concurrency: 1 containers: - name: api image: quay.io/cortexlabs-test/realtime-hello-world-cpu:latest env: RESPONSE: "hello from API A" readiness_probe: http_get: path: "/healthz" port: 8080 compute: cpu: 200m mem: 128Mi - name: hello-world-b kind: RealtimeAPI pod: port: 8080 max_concurrency: 1 containers: - name: api image: quay.io/cortexlabs-test/realtime-hello-world-cpu:latest env: RESPONSE: "hello from API B" readiness_probe: http_get: path: "/healthz" port: 8080 compute: cpu: 200m mem: 128Mi - name: hello-world-shadow kind: RealtimeAPI pod: port: 8080 max_concurrency: 1 containers: - name: api image: quay.io/cortexlabs-test/realtime-hello-world-cpu:latest env: RESPONSE: "hello from shadow API" readiness_probe: http_get: path: "/healthz" port: 8080 compute: cpu: 200m mem: 128Mi - name: hello-world kind: TrafficSplitter apis: - name: hello-world-a weight: 30 - name: hello-world-b weight: 70 - name: hello-world-shadow shadow: true weight: 100 ================================================ FILE: test/apis/trafficsplitter/hello-world/sample.json ================================================ {} ================================================ FILE: test/e2e/README.md ================================================ # End-to-end Tests ## Dependencies Install the `e2e` package, from the project directory: ```shell pip install -e test/e2e ``` This only needs to be installed once (not on every code change). _note: you may need to run `pip3 uninstall cortex` and `pip3 install -e python/client/` before the command above_ ## Running the tests Before running tests, instruct the Python client to use your development CLI binary: ```shell export CORTEX_CLI_PATH=/bin/cortex ``` From an existing cluster: ```shell pytest test/e2e/tests --env ``` Using a new cluster, created for testing only and deleted afterwards: ```shell pytest test/e2e/tests --config ``` **Note:** For the BatchAPI tests, the `--s3-path` option should be provided with an S3 bucket for testing purposes. It is more convenient however to define this bucket through an environment variable, see [configuration](#configuration) . ### Skip GPU Tests It is possible to skip GPU tests by passing the `--skip-gpus` flag to the pytest command. ### Skip Inferentia Tests It is possible to skip Inferentia tests by passing the `--skip-infs` flag to the pytest command. ### Skip Autoscaling Test It is possible to skip the autoscaling test by passing the `--skip-autoscaling` flag to the pytest command. ### Skip Load Test It is possible to skip the load tests by passing the `--skip-load` flag to the pytest command. ### Skip Long Running Test It is possible to skip the long running test by passing the `--skip-long-running` flag to the pytest command. ## Configuration It is possible to configure the behaviour of the tests by defining environment variables or a `.env` file at the project directory. ```dotenv # .env file CORTEX_TEST_REALTIME_DEPLOY_TIMEOUT=120 CORTEX_TEST_BATCH_DEPLOY_TIMEOUT=60 CORTEX_TEST_BATCH_JOB_TIMEOUT=120 CORTEX_TEST_BATCH_S3_PATH=s3:///test/jobs ``` ================================================ FILE: test/e2e/e2e/__init__.py ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from .cluster import create_cluster, delete_cluster __all__ = ["create_cluster", "delete_cluster"] ================================================ FILE: test/e2e/e2e/cluster.py ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import subprocess import sys import yaml from e2e.exceptions import ClusterCreationException, ClusterDeletionException def create_cluster(cluster_config: str): """Create a cortex cluster from a cluster config""" with open(cluster_config) as f: config = yaml.safe_load(f) p = subprocess.run( [ "cortex", "cluster", "up", cluster_config, "-y", "--configure-env", config["cluster_name"], ], stdout=sys.stdout, stderr=sys.stderr, ) if p.returncode != 0: raise ClusterCreationException(f"failed to create cluster with config: {cluster_config}") def delete_cluster(cluster_config: str): """Delete a cortex cluster from a cluster config""" with open(cluster_config) as f: config = yaml.safe_load(f) p = subprocess.run( ["cortex", "cluster", "down", "-y", "--config", cluster_config], stdout=sys.stdout, stderr=sys.stderr, ) if p.returncode != 0: raise ClusterDeletionException(f"failed to delete cluster with config: {cluster_config}") ================================================ FILE: test/e2e/e2e/exceptions.py ================================================ class ClusterCreationException(Exception): pass class ClusterDeletionException(Exception): pass class ExpectationsValidationException(Exception): pass class GeneratorValidationException(Exception): pass ================================================ FILE: test/e2e/e2e/expectations.py ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import pathlib import types from typing import Dict, Any import jsonschema import requests import yaml from jsonschema import Draft7Validator from e2e.exceptions import ExpectationsValidationException CONTENT_TO_ATTR = {"text": "text", "json": "json", "binary": "content"} def assert_response_expectations(response: requests.Response, expectations: Dict[str, Any]): content_type = expectations["content_type"] expected = expectations.get("expected") if expected: output = _get_response_content(response, content_type) assert output == expected, f"unexpected response: got {output}, expected {expected}" expected_json_schema = expectations.get("json_schema") if expected_json_schema: output = _get_response_content(response, content_type) jsonschema.validate(output, schema=expected_json_schema) def assert_json_expectations(response_json: Dict[str, Any], expectations: Dict[str, Any]): expected_json_schema = expectations.get("json_schema") if expected_json_schema: jsonschema.validate(response_json, schema=expected_json_schema) def parse_expectations(expectations_file: str) -> Dict[str, Any]: with open(expectations_file) as f: expectations = yaml.safe_load(f) validate_expectations(expectations) return expectations def validate_expectations(expectations): if "response" in expectations: validate_response_expectations(expectations["response"]) def validate_response_expectations(expectations: Dict[str, Any]): if not expectations["content_type"] in CONTENT_TO_ATTR.keys(): raise ExpectationsValidationException( f"response.content_type should be one of {CONTENT_TO_ATTR.keys()}" ) if "expected" in expectations and "json_schema" in expectations: raise ExpectationsValidationException("expected and json_schema are mutually exclusive") if "json_schema" in expectations: if expectations["content_type"] != "json": raise ExpectationsValidationException( "json_schema is only valid when content_type is set to json" ) try: Draft7Validator.check_schema(schema=expectations["json_schema"]) except Exception as e: raise ExpectationsValidationException("json_schema is invalid") from e if "grpc" in expectations: grpc = expectations["grpc"] required_fields = [ "proto_module_pb2", "proto_module_pb2_grpc", "stub_service_name", "input_spec", "output_spec", ] for required_field in required_fields: if required_field not in grpc: raise ExpectationsValidationException(f"missing grpc.{required_field} field") p1 = str(pathlib.Path(grpc["proto_module_pb2"]).parent) p2 = str(pathlib.Path(grpc["proto_module_pb2_grpc"]).parent) if p1 != p2: raise ExpectationsValidationException( "the parent directories of proto_module_pb2 and proto_module_pb2_grpc don't match" ) input_spec = grpc["input_spec"] if "class_name" not in input_spec: raise ExpectationsValidationException("missing grpc.input_spec.class_name field") if "input" not in input_spec: raise ExpectationsValidationException("missing grpc.input_spec.input field") output_spec = grpc["output_spec"] if "class_name" not in output_spec: raise ExpectationsValidationException("missing grpc.output_spec.class_name field") if "stream" not in output_spec: raise ExpectationsValidationException("missing grpc.output_spec.stream field") def _get_response_content(response: requests.Response, content_type: str) -> str: attr = CONTENT_TO_ATTR.get(content_type, "content") content = getattr(response, attr) if isinstance(content, types.MethodType): return content() return content ================================================ FILE: test/e2e/e2e/generator.py ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import importlib import pathlib from typing import Any, Callable, List import sys import inspect from e2e.exceptions import GeneratorValidationException def load_generator(sample_generator: pathlib.Path) -> Callable[[], List[int]]: api_dir = str(sample_generator.parent) sys.path.append(api_dir) sample_generator_module = importlib.import_module(str(pathlib.Path(sample_generator).stem)) sys.path.pop() validate_module(sample_generator_module) return sample_generator_module.generate_sample def validate_module(sample_generator_module: Any): if not hasattr(sample_generator_module, "generate_sample"): raise GeneratorValidationException( "sample generator module doesn't have a function called 'generate_sample'" ) if not inspect.isfunction(getattr(sample_generator_module, "generate_sample")): raise GeneratorValidationException("'generate_sample' is not a function") if inspect.getfullargspec(getattr(sample_generator_module, "generate_sample")).args != []: raise GeneratorValidationException( "'generate_sample' function must not have any parameters" ) ================================================ FILE: test/e2e/e2e/tests.py ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import json import math import os import re import threading as td import time from http import HTTPStatus from pathlib import Path from typing import Callable, Dict, Any, List, Union import boto3 import cortex as cx import requests import yaml from e2e.expectations import ( parse_expectations, assert_response_expectations, assert_json_expectations, ) from e2e.generator import load_generator from e2e.utils import ( apis_ready, api_updated, wait_on_event, wait_on_futures, endpoint_ready, post_request, get_request, job_done, jobs_done, request_batch_prediction, request_task, retrieve_async_result, make_requests_concurrently, check_futures_healthy, retrieve_results_concurrently, stream_api_logs, stream_job_logs, wait_for, ) TEST_APIS_DIR = Path(__file__).parent.parent.parent / "apis" def delete_apis(client: cx.Client, api_names: List[str]): for name in api_names: client.delete(name) def test_realtime_api( printer: Callable, client: cx.Client, api: str, timeout: int = None, api_config_name: str = "cortex_cpu.yaml", node_groups: List[str] = [], extra_path: str = "", method: str = "POST", ): api_dir = TEST_APIS_DIR / api with open(str(api_dir / api_config_name)) as f: api_specs = yaml.safe_load(f) assert len(api_specs) == 1 if len(node_groups) > 0: api_specs[0]["node_groups"] = node_groups expectations = None expectations_file = api_dir / "expectations.yaml" if expectations_file.exists(): expectations = parse_expectations(str(expectations_file)) api_name = api_specs[0]["name"] for api_spec in api_specs: client.deploy(api_spec=api_spec) try: assert apis_ready( client=client, api_names=[api_name], timeout=timeout ), f"apis {api_name} not ready" with open(str(api_dir / "sample.json")) as f: payload = json.load(f) if method == "POST": response = post_request(client, api_name, payload, extra_path) else: response = get_request(client, api_name, payload, extra_path) assert ( response.status_code == HTTPStatus.OK ), f"status code: got {response.status_code}, expected {HTTPStatus.OK}" if expectations and "response" in expectations: assert_response_expectations(response, expectations["response"]) except: # best effort try: api_info = client.get_api(api_name) printer(json.dumps(api_info, indent=2)) td.Thread(target=lambda: stream_api_logs(client, api_name), daemon=True).start() time.sleep(5) finally: raise finally: delete_apis(client, [api_name]) def test_batch_api( printer: Callable, client: cx.Client, api: str, test_s3_path: str, deploy_timeout: int = None, job_timeout: int = None, retry_attempts: int = 0, api_config_name: str = "cortex_cpu.yaml", node_groups: List[str] = [], local_operator: bool = False, ): api_dir = TEST_APIS_DIR / api with open(str(api_dir / api_config_name)) as f: api_specs = yaml.safe_load(f) assert len(api_specs) == 1 if len(node_groups) > 0: api_specs[0]["node_groups"] = node_groups api_name = api_specs[0]["name"] client.deploy(api_spec=api_specs[0]) try: endpoint_override = f"http://localhost:8888/batch/{api_name}" if local_operator else None assert endpoint_ready( client=client, api_name=api_name, timeout=deploy_timeout, endpoint_override=endpoint_override, ), f"api {api_name} not ready" with open(str(api_dir / "sample.json")) as f: payload = json.load(f) response = None for _ in range(retry_attempts + 1): response = request_batch_prediction( client, api_name, item_list=payload, batch_size=2, config={"dest_s3_dir": test_s3_path}, local_operator=local_operator, ) if response.status_code == HTTPStatus.OK: break time.sleep(1) assert ( response.status_code == HTTPStatus.OK ), f"status code: got {response.status_code}, expected {HTTPStatus.OK} ({response.text})" job_spec = response.json() # monitor job progress job_id = job_spec["job_id"] endpoint_override = ( f"http://localhost:8888/batch/{api_name}?jobID={job_id}" if local_operator else None ) assert job_done( client=client, api_name=api_name, job_id=job_id, timeout=job_timeout, endpoint_override=endpoint_override, ), f"job did not succeed (api_name: {api_name}, job_id: {job_spec['job_id']})" except: # best effort try: api_info = client.get_api(api_name) printer(json.dumps(api_info, indent=2)) job_status = client.get_job(api_name, job_spec["job_id"]) printer(json.dumps(job_status, indent=2)) td.Thread( target=lambda: stream_job_logs(client, api_name, job_spec["job_id"]), daemon=True, ).start() time.sleep(5) finally: raise finally: delete_apis(client, [api_name]) def test_async_api( printer: Callable, client: cx.Client, api: str, deploy_timeout: int = None, poll_retries: int = 5, poll_sleep_seconds: int = 1, api_config_name: str = "cortex_cpu.yaml", node_groups: List[str] = [], ): api_dir = TEST_APIS_DIR / api with open(str(api_dir / api_config_name)) as f: api_specs = yaml.safe_load(f) assert len(api_specs) == 1 if len(node_groups) > 0: api_specs[0]["node_groups"] = node_groups expectations = None expectations_file = api_dir / "expectations.yaml" if expectations_file.exists(): expectations = parse_expectations(str(expectations_file)) api_name = api_specs[0]["name"] client.deploy(api_spec=api_specs[0]) try: assert apis_ready( client=client, api_names=[api_name], timeout=deploy_timeout ), f"apis {api_name} not ready" with open(str(api_dir / "sample.json")) as f: payload = json.load(f) response = post_request(client, api_name, payload) assert ( response.status_code == HTTPStatus.OK ), f"workload submission status code: got {response.status_code}, expected {HTTPStatus.OK}" response_json = response.json() assert "id" in response_json request_id = response_json["id"] result_response = None for _ in range(poll_retries + 1): result_response = retrieve_async_result( client=client, api_name=api_name, request_id=request_id ) if result_response.status_code != HTTPStatus.OK: time.sleep(poll_sleep_seconds) continue result_response_json = result_response.json() assert ( "id" in result_response_json ), f"id key was not present in result response (response: {result_response_json})" assert ( "status" in result_response_json ), f"status key was not present in result response (response: {result_response_json})" if result_response_json["status"] != "completed": time.sleep(poll_sleep_seconds) continue break assert ( result_response.status_code == HTTPStatus.OK ), f"result retrieval status code: got {result_response.status_code}, expected {HTTPStatus.OK}" # validate keys are in the result json response assert ( "result" in result_response_json ), f"result key was not present in result response (response: {result_response_json})" assert ( "timestamp" in result_response_json ), f"timestamp key was not present in result response (response: {result_response_json})" # validate result json response has valid values assert ( result_response_json["id"] == request_id ), f"result 'id' and request 'id' mismatch ({result_response_json['id']} != {request_id})" assert ( result_response_json["status"] == "completed" ), f"async workload did not complete (response: {result_response_json})" assert result_response_json["timestamp"] != "", "result 'timestamp' value was empty" assert result_response_json["result"] != "", "result 'result' value was empty" # assert result expectations if expectations: assert_json_expectations(result_response_json["result"], expectations["response"]) except: # best effort try: api_info = client.get_api(api_name) printer(json.dumps(api_info, indent=2)) printer(json.dumps(result_response_json, indent=2)) td.Thread( target=lambda: stream_api_logs(client, api_name), daemon=True, ).start() time.sleep(5) except: pass raise finally: delete_apis(client, [api_name]) def test_task_api( printer: Callable, client: cx.Client, api: str, deploy_timeout: int = None, job_timeout: int = None, retry_attempts: int = 0, api_config_name: str = "cortex_cpu.yaml", node_groups: List[str] = [], local_operator: bool = False, ): api_dir = TEST_APIS_DIR / api with open(str(api_dir / api_config_name)) as f: api_specs = yaml.safe_load(f) assert len(api_specs) == 1 if len(node_groups) > 0: api_specs[0]["node_groups"] = node_groups api_name = api_specs[0]["name"] client.deploy(api_spec=api_specs[0]) try: endpoint_override = f"http://localhost:8888/tasks/{api_name}" if local_operator else None assert endpoint_ready( client=client, api_name=api_name, timeout=deploy_timeout, endpoint_override=endpoint_override, ), f"api {api_name} not ready" response = None for _ in range(retry_attempts + 1): response = request_task(client, api_name, local_operator=local_operator) if response.status_code == HTTPStatus.OK: break time.sleep(1) job_spec = response.json() job_id = job_spec["job_id"] endpoint_override = ( f"http://localhost:8888/tasks/{api_name}?jobID={job_id}" if local_operator else None ) assert job_done( client=client, api_name=api_name, job_id=job_spec["job_id"], timeout=job_timeout, endpoint_override=endpoint_override, ), f"task job did not succeed (api_name: {api_name}, job_id: {job_spec['job_id']})" except: # best effort try: api_info = client.get_api(api_name) printer(json.dumps(api_info, indent=2)) job_status = client.get_job(api_name, job_spec["job_id"]) printer(json.dumps(job_status, indent=2)) td.Thread( target=lambda: stream_job_logs(client, api_name, job_spec["job_id"]), daemon=True ).start() time.sleep(5) except: pass raise finally: delete_apis(client, [api_name]) def test_autoscaling( printer: Callable, client: cx.Client, apis: Dict[str, Any], autoscaling_config: Dict[str, Union[int, float]], deploy_timeout: int = None, api_config_name: str = "cortex_cpu.yaml", node_groups: List[str] = [], ): max_replicas = autoscaling_config["max_replicas"] query_params = apis["query_params"] # increase the concurrency by 1 to ensure we get max_replicas replicas concurrency = max_replicas + 1 all_apis = [apis["primary"]] + apis["dummy"] all_api_names = [] for api in all_apis: api_dir = TEST_APIS_DIR / api with open(str(api_dir / api_config_name)) as f: api_specs = yaml.safe_load(f) assert len(api_specs) == 1 if len(node_groups) > 0: api_specs[0]["node_groups"] = node_groups api_specs[0]["autoscaling"] = { "max_replicas": max_replicas, "downscale_stabilization_period": "1m", } all_api_names.append(api_specs[0]["name"]) client.deploy(api_spec=api_specs[0]) primary_api_name = all_api_names[0] autoscaling = client.get_api(primary_api_name)["spec"]["autoscaling"] # controls the flow of requests request_stopper = td.Event() # determine upscale/downscale replica requests current_replicas = 1 # starting number of replicas test_timeout = 0 # measured in seconds while current_replicas < max_replicas: upscale_ceil = math.ceil(current_replicas * autoscaling["max_upscale_factor"]) if upscale_ceil > current_replicas + 1: current_replicas = upscale_ceil else: current_replicas += 1 if current_replicas > max_replicas: current_replicas = max_replicas test_timeout += int(autoscaling["upscale_stabilization_period"] / (1000 ** 3)) while current_replicas > 1: downscale_ceil = math.ceil(current_replicas * autoscaling["max_downscale_factor"]) if downscale_ceil < current_replicas - 1: current_replicas = downscale_ceil else: current_replicas -= 1 test_timeout += int(autoscaling["downscale_stabilization_period"] / (1000 ** 3)) # add overhead to the test timeout to account for the process of downloading images or adding nodes to the cluster test_timeout *= 2 try: assert apis_ready( client=client, api_names=all_api_names, timeout=deploy_timeout ), f"apis {all_api_names} not ready" threads_futures = make_requests_concurrently( client, primary_api_name, concurrency, request_stopper, query_params=query_params ) test_start_time = time.time() # upscale/downscale the api printer(f"scaling up to {max_replicas} replicas") while True: assert api_updated( client, primary_api_name, timeout=deploy_timeout ), "api didn't scale up to the desired number of replicas in time" current_replicas = client.get_api(primary_api_name)["status"]["requested"] # stop the requests from being made if current_replicas == max_replicas and not request_stopper.is_set(): printer(f"scaling back down to 1 replica") request_stopper.set() # check if the requesting threads are still healthy # if not, they'll raise an exception check_futures_healthy(threads_futures) # check if the test is taking too much time assert ( time.time() - test_start_time < test_timeout ), f"autoscaling test for api {primary_api_name} did not finish in {test_timeout}s; current number of replicas is {current_replicas}/{concurrency}" # stop the test if it has finished if current_replicas == 1 and request_stopper.is_set(): break # add some delay to reduce the number of gets time.sleep(1) except: # best effort try: api_info = client.get_api(primary_api_name) printer(json.dumps(api_info, indent=2)) finally: raise finally: request_stopper.set() delete_apis(client, all_api_names) def test_load_realtime( printer: Callable, client: cx.Client, api: str, load_config: Dict[str, Union[int, float]], deploy_timeout: int = None, api_config_name: str = "cortex_cpu.yaml", node_groups: List[str] = [], ): total_requests = load_config["total_requests"] desired_replicas = load_config["desired_replicas"] concurrency = load_config["concurrency"] status_code_timeout = load_config["status_code_timeout"] api_dir = TEST_APIS_DIR / api with open(str(api_dir / api_config_name)) as f: api_specs = yaml.safe_load(f) assert len(api_specs) == 1 api_specs[0]["autoscaling"] = { "min_replicas": desired_replicas, "max_replicas": desired_replicas, } if len(node_groups) > 0: api_specs[0]["node_groups"] = node_groups api_name = api_specs[0]["name"] client.deploy(api_spec=api_specs[0]) # controls the flow of requests request_stopper = td.Event() failed = False try: printer(f"getting {desired_replicas} replicas ready") assert apis_ready( client=client, api_names=[api_name], timeout=deploy_timeout ), f"api {api_name} not ready" # give the APIs some time to prevent getting high latency spikes in the beginning time.sleep(5) with open(str(api_dir / "sample.json")) as f: payload = json.load(f) printer("start making requests concurrently") threads_futures = make_requests_concurrently( client, api_name, concurrency, request_stopper, max_total_requests=total_requests, payload=payload, ) while not request_stopper.is_set(): # check if the requesting threads are still healthy # if not, they'll raise an exception check_futures_healthy(threads_futures) # don't stress the CPU too hard time.sleep(1) except: # best effort failed = True try: api_info = client.get_api(api_name) printer(json.dumps(api_info, indent=2)) finally: raise finally: request_stopper.set() delete_apis(client, [api_name]) if failed: time.sleep(30) def test_load_async( printer: Callable, client: cx.Client, api: str, load_config: Dict[str, Union[int, float]], deploy_timeout: int = None, poll_sleep_seconds: int = 1, api_config_name: str = "cortex_cpu.yaml", node_groups: List[str] = [], ): total_requests = load_config["total_requests"] desired_replicas = load_config["desired_replicas"] concurrency = load_config["concurrency"] submit_timeout = load_config["submit_timeout"] workload_timeout = load_config["workload_timeout"] api_dir = TEST_APIS_DIR / api with open(str(api_dir / api_config_name)) as f: api_specs = yaml.safe_load(f) assert len(api_specs) == 1 api_specs[0]["autoscaling"] = { "min_replicas": desired_replicas, "max_replicas": desired_replicas, } if len(node_groups) > 0: api_specs[0]["node_groups"] = node_groups api_name = api_specs[0]["name"] client.deploy(api_spec=api_specs[0]) request_stopper = td.Event() map_stopper = td.Event() responses: List[Dict[str, Any]] = [] failed = False try: printer(f"getting {desired_replicas} replicas ready") assert apis_ready( client=client, api_names=[api_name], timeout=deploy_timeout ), f"api {api_name} not ready" with open(str(api_dir / "sample.json")) as f: payload = json.load(f) printer("start making prediction requests concurrently") threads_futures = make_requests_concurrently( client, api_name, concurrency, request_stopper, responses=responses, max_total_requests=total_requests, payload=payload, ) assert wait_on_event( request_stopper, submit_timeout ), f"{total_requests} couldn't be submitted in {submit_timeout}s" check_futures_healthy(threads_futures) wait_on_futures(threads_futures) printer("finished making prediction requests") assert ( len(responses) == total_requests ), f"the submitted number of requests doesn't match the returned number of responses" job_ids = [] for response in responses: response_json = response.json() assert "id" in response_json job_ids.append(response_json["id"]) # assert the results printer("start retrieving the async results concurrently") results = [] retrieve_results_concurrently( client, api_name, concurrency, map_stopper, job_ids, results, poll_sleep_seconds, workload_timeout, ) for request_id, json_result in results: # validate keys are in the result json response assert ( "id" in json_result ), f"id key was not present in result response (response: {json_result})" assert ( "result" in json_result ), f"result key was not present in result response (response: {json_result})" assert ( "timestamp" in json_result ), f"timestamp key was not present in result response (response: {json_result})" # validate result json response has valid values assert ( json_result["id"] == request_id ), f"result 'id' and request 'id' mismatch ({json_result['id']} != {request_id})" assert ( json_result["status"] == "completed" ), f"async workload did not complete (response: {json_result})" assert json_result["timestamp"] != "", "result 'timestamp' value was empty" assert json_result["result"] != "", "result 'result' value was empty" except: # best effort failed = True try: api_info = client.get_api(api_name) printer(json.dumps(api_info, indent=2)) finally: raise finally: if "results" in vars() and len(results) < total_requests: printer(f"{len(results)}/{total_requests} have been successfully retrieved") map_stopper.set() delete_apis(client, [api_name]) if failed: time.sleep(30) def test_load_batch( printer: Callable, client: cx.Client, api: str, test_s3_path: str, load_config: Dict[str, Union[int, float]], deploy_timeout: int = None, retry_attempts: int = 0, api_config_name: str = "cortex_cpu.yaml", node_groups: List[str] = [], ): jobs = load_config["jobs"] workers_per_job = load_config["workers_per_job"] items_per_job = load_config["items_per_job"] batch_size = load_config["batch_size"] workload_timeout = load_config["workload_timeout"] bucket, key = re.match("s3://(.+?)/(.+)", test_s3_path).groups() s3 = boto3.client("s3") paginator = s3.get_paginator("list_objects_v2") api_dir = TEST_APIS_DIR / api with open(str(api_dir / api_config_name)) as f: api_specs = yaml.safe_load(f) assert len(api_specs) == 1 if len(node_groups) > 0: api_specs[0]["node_groups"] = node_groups sample_generator_path = api_dir / "sample_generator.py" assert ( sample_generator_path.exists() ), "sample_generator.py must be present for the batch load test" sample_generator = load_generator(sample_generator_path) api_name = api_specs[0]["name"] client.deploy(api_spec=api_specs[0]) api_endpoint = client.get_api(api_name)["endpoint"] failed = False try: assert endpoint_ready( client=client, api_name=api_name, timeout=deploy_timeout ), f"api {api_name} not ready" # submit jobs printer(f"submitting {jobs} jobs") job_specs = [] for _ in range(jobs): for _ in range(retry_attempts + 1): response = request_batch_prediction( client, api_name, item_list=[sample_generator() for _ in range(items_per_job)], batch_size=batch_size, workers=workers_per_job, config={"dest_s3_dir": test_s3_path}, ) if response.status_code == HTTPStatus.OK: break time.sleep(1) # retries are only required once retry_attempts = 0 assert ( response.status_code == HTTPStatus.OK ), f"status code: got {response.status_code}, expected {HTTPStatus.OK} ({response.text})" job_specs.append(response.json()) # wait the jobs to finish printer("waiting on the jobs") assert jobs_done( client, api_name, [job_spec["job_id"] for job_spec in job_specs], workload_timeout ), f"not all jobs succeed in {workload_timeout}s" # assert jobs printer("checking the jobs' responses") for job_spec in job_specs: job_id: str = job_spec["job_id"] job = requests.get(f"{api_endpoint}?jobID={job_id}").json() job_status = job["job_status"] job_metrics = job["metrics"] assert ( job_status["batches_in_queue"] == 0 ), f"there are still batches in queue ({job_status['batches_in_queue']}) for job ID {job_id}" assert job_metrics["succeeded"] == math.ceil(items_per_job / batch_size) num_objects = 0 for page in paginator.paginate(Bucket=bucket, Prefix=os.path.join(key, job_id)): num_objects += len(page["Contents"]) assert num_objects == 1 except: # best effort failed = True try: api_info = client.get_api(api_name) # only get the last 10 job statuses if "batch_job_statuses" in api_info and len(api_info["batch_job_statuses"]) > 10: api_info["batch_job_statuses"] = api_info["batch_job_statuses"][-10:] printer(json.dumps(api_info, indent=2)) finally: raise finally: delete_apis(client, [api_name]) if failed: time.sleep(30) def test_long_running_realtime( printer: Callable, client: cx.Client, api: str, long_running_config: Dict[str, Union[int, float]], deploy_timeout: int = None, api_config_name: str = "cortex_cpu.yaml", node_groups: List[str] = [], ): api_dir = TEST_APIS_DIR / api with open(str(api_dir / api_config_name)) as f: api_specs = yaml.safe_load(f) assert len(api_specs) == 1 time_to_run = long_running_config["time_to_run"] if len(node_groups) > 0: api_specs[0]["node_groups"] = node_groups expectations = None expectations_file = api_dir / "expectations.yaml" if expectations_file.exists(): expectations = parse_expectations(str(expectations_file)) api_name = api_specs[0]["name"] for api_spec in api_specs: client.deploy(api_spec=api_spec) try: assert apis_ready( client=client, api_names=[api_name], timeout=deploy_timeout ), f"apis {api_name} not ready" with open(str(api_dir / "sample.json")) as f: payload = json.load(f) counter = 0 start_time = time.time() while time.time() - start_time <= time_to_run: response = post_request(client, api_name, payload) assert ( response.status_code == HTTPStatus.OK ), f"status code: got {response.status_code}, expected {HTTPStatus.OK}" if expectations and "response" in expectations: assert_response_expectations(response, expectations["response"]) counter += 1 except: # best effort try: api_info = client.get_api(api_name) printer(json.dumps(api_info, indent=2)) td.Thread(target=lambda: stream_api_logs(client, api_name), daemon=True).start() time.sleep(5) finally: raise finally: delete_apis(client, [api_name]) def test_realtime_scale_to_zero( client: cx.Client, api: str, timeout: int = None, api_config_name: str = "cortex_cpu.yaml", ): api_dir = TEST_APIS_DIR / api with open(str(api_dir / api_config_name)) as f: api_specs = yaml.safe_load(f) api_name = api_specs[0]["name"] for api_spec in api_specs: client.deploy(api_spec=api_spec) try: assert apis_ready( client=client, api_names=[api_name], timeout=timeout, greater_or_equal_to=0 ), f"apis {api_name} not ready" api_info = client.get_api(api_name) endpoint = api_info["endpoint"] response = requests.post(endpoint, json={}, timeout=30) # make first request, which should go to activator assert ( response.status_code == HTTPStatus.OK ), f"status code: got {response.status_code}, expected {HTTPStatus.OK}" assert response.headers.get("x-cortex-origin") == "activator" def _make_request() -> bool: res = requests.post(endpoint, json={}, timeout=30) # make second request, which should go directly to the api assert ( res.status_code == HTTPStatus.OK ), f"status code: got {res.status_code}, expected {HTTPStatus.OK}" return res.headers.get("x-cortex-origin") == "api" assert wait_for(_make_request, timeout=60) finally: delete_apis(client, [api_name]) ================================================ FILE: test/e2e/e2e/utils.py ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import threading as td import time from concurrent import futures from http import HTTPStatus from typing import Any, List, Optional, Tuple, Union, Dict, Callable import cortex as cx import requests import yaml def wait_for(fn: Callable[[], bool], timeout=None) -> bool: deadline = time.time() + timeout if timeout else None while True: if deadline is not None and time.time() > deadline: return False done = fn() if done: return True time.sleep(1) def apis_ready( client: cx.Client, api_names: List[str], timeout: Optional[int] = None, greater_or_equal_to: int = 1, ) -> bool: def _check_liveness(status): return ( status["requested"] >= greater_or_equal_to and status["requested"] == status["ready"] == status["up_to_date"] ) def _is_ready(): return all([_check_liveness(client.get_api(name)["status"]) for name in api_names]) return wait_for(_is_ready, timeout=timeout) def api_updated(client: cx.Client, api_name: str, timeout: Optional[int] = None) -> bool: def _is_ready(): status = client.get_api(api_name)["status"] return status["requested"] == status["ready"] return wait_for(_is_ready, timeout=timeout) def wait_on_event(event: td.Event, timeout: Optional[int] = None) -> bool: def _is_ready(): return event.is_set() return wait_for(_is_ready, timeout=timeout) def wait_on_futures(futures_list: List[futures.Future], timeout: Optional[int] = None): def _is_ready(): return all([future.done() for future in futures_list]) return wait_for(_is_ready, timeout=timeout) def endpoint_ready( client: cx.Client, api_name: str, timeout: int = None, endpoint_override: str = None ) -> bool: def _is_ready(): if endpoint_override: endpoint = endpoint_override else: endpoint = client.get_api(api_name)["endpoint"] response = requests.post(endpoint) return response.status_code == HTTPStatus.BAD_REQUEST return wait_for(_is_ready, timeout=timeout) def job_done( client: cx.Client, api_name: str, job_id: str, timeout: int = None, endpoint_override: str = None, ) -> bool: def _is_ready(): if endpoint_override: job_info = requests.get(endpoint_override) job_info = job_info.json() return job_info["job_status"]["status"] == "succeeded" job_info = client.get_job(api_name, job_id) return job_info["job_status"]["status"] == "succeeded" return wait_for(_is_ready, timeout=timeout) def jobs_done(client: cx.Client, api_name: str, job_ids: List[str], timeout: int = None) -> bool: exec = futures.ThreadPoolExecutor(10) def _runnable(job_id): return job_done(client, api_name, job_id) try: for _ in exec.map(_runnable, job_ids, timeout=timeout): pass except: return False return True def post_request( client: cx.Client, api_name: str, payload: Union[List, Dict], extra_path: Optional[str] = None, ) -> requests.Response: api_info = client.get_api(api_name) endpoint = api_info["endpoint"] if extra_path and extra_path != "": endpoint = os.path.join(endpoint, extra_path) response = requests.post(endpoint, json=payload) return response def get_request( client: cx.Client, api_name: str, payload: Union[List, Dict], extra_path: Optional[str] = None, ) -> requests.Response: api_info = client.get_api(api_name) endpoint = api_info["endpoint"] if extra_path and extra_path != "": endpoint = os.path.join(endpoint, extra_path) response = requests.get(endpoint, json=payload) return response def retrieve_async_result(client: cx.Client, api_name: str, request_id: str) -> requests.Response: api_info = client.get_api(api_name) response = requests.get(f"{api_info['endpoint']}/{request_id}") return response def request_batch_prediction( client: cx.Client, api_name: str, item_list: List, batch_size: int, workers: int = 1, config: Dict = None, local_operator: bool = False, ) -> requests.Response: if local_operator: endpoint = f"http://localhost:8888/batch/{api_name}" else: api_info = client.get_api(api_name) endpoint = api_info["endpoint"] batch_payload = { "workers": workers, "item_list": {"items": item_list, "batch_size": batch_size}, "config": config, } response = requests.post(endpoint, json=batch_payload) return response def request_task( client: cx.Client, api_name: str, config: Dict = None, timeout: int = None, local_operator: bool = False, ): if local_operator: endpoint = f"http://localhost:8888/tasks/{api_name}" else: api_info = client.get_api(api_name) endpoint = api_info["endpoint"] payload = {} if config is not None: payload["config"] = config if timeout is not None: payload["timeout"] = timeout response = requests.post(endpoint, json=payload) return response _make_requests_concurrently_max_total_requests: int = 0 def make_requests_concurrently( client: cx.Client, api_name: str, concurrency: int, event_stopper: td.Event, latencies: Optional[List[float]] = None, responses: Optional[List[Dict[str, Any]]] = None, max_total_requests: Optional[int] = None, payload: Optional[Union[List, Dict]] = None, query_params: Dict[str, str] = {}, ) -> List[futures.Future]: lock = td.RLock() thread_local = td.local() start_sync = td.Barrier(concurrency) end_sync = td.Barrier(concurrency) executor = futures.ThreadPoolExecutor(concurrency) api_info = client.get_api(api_name) endpoint = api_info["endpoint"] global _make_requests_concurrently_max_total_requests _make_requests_concurrently_max_total_requests = max_total_requests def get_session() -> requests.Session: if not hasattr(thread_local, "session"): thread_local.session = requests.Session() return thread_local.session def runnable(): session = get_session() global _make_requests_concurrently_max_total_requests start_sync.wait() while not event_stopper.is_set(): if _make_requests_concurrently_max_total_requests is not None: with lock: if _make_requests_concurrently_max_total_requests == 0: break _make_requests_concurrently_max_total_requests -= 1 start = time.time() response = session.post(endpoint, json=payload, params=query_params) assert ( response.status_code == HTTPStatus.OK ), f"status code: got {response.status_code}, expected {HTTPStatus.OK}" if latencies is not None: latencies.append(time.time() - start) if responses is not None: responses.append(response) if _make_requests_concurrently_max_total_requests is not None: end_sync.wait() event_stopper.set() futures_list = [] for _ in range(concurrency): future = executor.submit(runnable) futures_list.append(future) return futures_list def retrieve_results_concurrently( client: cx.Client, api_name: str, concurrency: int, event_stopper: td.Event, job_ids: List[str], responses: List[Tuple[str, Dict[str, Any]]] = [], poll_sleep_seconds: int = 1, timeout: Optional[int] = None, ): api_info = client.get_api(api_name) task_kind = api_info["spec"]["kind"] == "TaskAPI" async_kind = api_info["spec"]["kind"] == "AsyncAPI" if not task_kind and not async_kind: raise ValueError("function can only be called for TaskAPI/AsyncAPI kinds") exec = futures.ThreadPoolExecutor(concurrency) thread_local = td.local() def _get_session() -> requests.Session: if not hasattr(thread_local, "session"): thread_local.session = requests.Session() return thread_local.session def _retriever(request_id: str): session = _get_session() while not event_stopper.is_set(): if task_kind: result_response = session.get(f"{api_info['endpoint']}?jobID={request_id}") if async_kind: result_response = session.get(f"{api_info['endpoint']}/{request_id}") if result_response.status_code != HTTPStatus.OK: content = result_response.content.decode("utf-8") if "error" in content: event_stopper.set() raise RuntimeError( f"received {result_response.status_code} status code with the following message: {content}" ) time.sleep(poll_sleep_seconds) continue result_response_json = result_response.json() if async_kind and "status" in result_response_json: if result_response_json["status"] == "completed": break if result_response_json["status"] not in ["in_progress", "in_queue"]: raise RuntimeError( f"status for request ID {request_id} got set to {result_response_json['status']}" ) if ( task_kind and "job_status" in result_response_json and "status" in result_response_json["job_status"] ): if result_response_json["job_status"]["status"] == "succeeded": break if result_response_json["job_status"]["status"] not in [ "pending", "enqueuing", "running", ]: raise RuntimeError( f"status for job ID {request_id} got set to {result_response_json['job_status']['status']}" ) if event_stopper.is_set(): return responses.append((request_id, result_response_json)) # will throw an exception if something failed in any thread for _ in exec.map(_retriever, job_ids, timeout=timeout): pass def check_futures_healthy(futures_list: List[futures.Future]): for future in futures_list: has_exception = None try: has_exception = future.exception(timeout=0.0) except futures.TimeoutError: pass if has_exception: future.result() def client_from_config(config_path: str) -> cx.Client: with open(config_path) as f: config = yaml.safe_load(f) cluster_name = config["cluster_name"] return cx.client(f"{cluster_name}") def stream_api_logs(client: cx.Client, api_name: str): cx.run_cli(["logs", api_name, "--random-pod", "-e", client.env_name]) def stream_job_logs(client: cx.Client, api_name: str, job_id: str): cx.run_cli(["logs", api_name, job_id, "--random-pod", "-e", client.env_name]) ================================================ FILE: test/e2e/pytest.ini ================================================ # pytest.ini [pytest] minversion = 6.0 addopts = -s -v -r sxf ================================================ FILE: test/e2e/setup.py ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from pathlib import Path from setuptools import setup, find_packages root = Path(__file__).parent.absolute() cortex_client_dir = root.parent.parent / "python" / "client" if not cortex_client_dir.exists(): raise ModuleNotFoundError(f"cortex client not found in {cortex_client_dir}") setup( name="e2e", version="master", # CORTEX_VERSION packages=find_packages(exclude=["tests"]), url="https://github.com/cortexlabs/cortex", license="Apache License 2.0", python_requires=">=3.6", install_requires=[ "requests==2.24.0", "jsonschema==3.2.0", "pytest==6.1.*", "pytest-print==0.2.1", "python-dotenv==0.15.0", "pyyaml>=5.3.1", "boto3>=1.14.53", "cortex", ], dependency_links=[f"file://{cortex_client_dir}#egg=cortex"], author="Cortex Labs", author_email="dev@cortexlabs.com", description="Cortex E2E tests package", ) ================================================ FILE: test/e2e/tests/__init__.py ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ================================================ FILE: test/e2e/tests/aws/__init__.py ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ================================================ FILE: test/e2e/tests/aws/conftest.py ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import cortex as cx import pytest import e2e from e2e.utils import client_from_config @pytest.fixture def client(config): env_name = config["aws"]["env"] if env_name: return cx.client(env_name) config_path = config["aws"]["config"] if config_path is not None: return client_from_config(config_path) pytest.skip("--env or --config must be passed to run tests") def pytest_configure(config): cluster_config = config.getoption("--config") if cluster_config: e2e.create_cluster(cluster_config) def pytest_unconfigure(config): cluster_config = config.getoption("--config") if cluster_config: e2e.delete_cluster(cluster_config) ================================================ FILE: test/e2e/tests/aws/test_async.py ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from typing import Callable, Dict import cortex as cx import pytest import e2e.tests TEST_APIS = ["async/text-generator"] TEST_APIS_GPU = ["async/text-generator"] @pytest.mark.usefixtures("client") @pytest.mark.parametrize("api", TEST_APIS) def test_async_api(printer: Callable, config: Dict, client: cx.Client, api: str): e2e.tests.test_async_api( printer=printer, client=client, api=api, deploy_timeout=config["global"]["async_deploy_timeout"], poll_retries=config["global"]["async_workload_timeout"], node_groups=config["aws"]["x86_nodegroups"], ) @pytest.mark.usefixtures("client") @pytest.mark.parametrize("api", TEST_APIS_GPU) def test_async_api_gpu(printer: Callable, config: Dict, client: cx.Client, api: str): skip_gpus = config["global"].get("skip_gpus", False) if skip_gpus: pytest.skip("--skip-gpus flag detected, skipping GPU tests") e2e.tests.test_async_api( printer=printer, client=client, api=api, deploy_timeout=config["global"]["async_deploy_timeout"], poll_retries=config["global"]["async_workload_timeout"], api_config_name="cortex_gpu.yaml", node_groups=config["aws"]["x86_nodegroups"], ) ================================================ FILE: test/e2e/tests/aws/test_autoscaling.py ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from typing import Any, Callable, Dict import cortex as cx import pytest import e2e.tests TEST_APIS = [ { "primary": "realtime/sleep", "dummy": ["realtime/prime-generator"], "query_params": { "sleep": "1.0", }, } ] @pytest.mark.usefixtures("client") @pytest.mark.parametrize("apis", TEST_APIS, ids=[api["primary"] for api in TEST_APIS]) def test_autoscaling(printer: Callable, config: Dict, client: cx.Client, apis: Dict[str, Any]): skip_autoscaling_test = config["global"].get("skip_autoscaling", False) if skip_autoscaling_test: pytest.skip("--skip-autoscaling flag detected, skipping autoscaling tests") e2e.tests.test_autoscaling( printer, client, apis, autoscaling_config=config["global"]["autoscaling_test_config"], deploy_timeout=config["global"]["realtime_deploy_timeout"], node_groups=config["aws"]["x86_nodegroups"], ) ================================================ FILE: test/e2e/tests/aws/test_batch.py ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from typing import Callable, Dict import cortex as cx import pytest import e2e.tests TEST_APIS = ["batch/image-classifier-alexnet"] TEST_APIS_GPU = ["batch/image-classifier-alexnet"] @pytest.mark.usefixtures("client") @pytest.mark.parametrize("api", TEST_APIS) def test_batch_api(printer: Callable, config: Dict, client: cx.Client, api: str): s3_path = config["aws"].get("s3_path") if not s3_path: pytest.skip( "--s3-path option is required to run batch tests (alternatively set the " "CORTEX_TEST_BATCH_S3_PATH env var) )" ) e2e.tests.test_batch_api( printer, client, api, test_s3_path=s3_path, deploy_timeout=config["global"]["batch_deploy_timeout"], job_timeout=config["global"]["batch_job_timeout"], retry_attempts=5, node_groups=config["aws"]["x86_nodegroups"], local_operator=config["global"]["local_operator"], ) @pytest.mark.usefixtures("client") @pytest.mark.parametrize("api", TEST_APIS_GPU) def test_batch_api_gpu(printer: Callable, config: Dict, client: cx.Client, api: str): skip_gpus = config["global"].get("skip_gpus", False) if skip_gpus: pytest.skip("--skip-gpus flag detected, skipping GPU tests") s3_path = config["aws"].get("s3_path") if not s3_path: pytest.skip( "--s3-path option is required to run batch tests (alternatively set the " "CORTEX_TEST_BATCH_S3_PATH env var) )" ) e2e.tests.test_batch_api( printer=printer, client=client, api=api, test_s3_path=s3_path, deploy_timeout=config["global"]["batch_deploy_timeout"], job_timeout=config["global"]["batch_job_timeout"], retry_attempts=5, api_config_name="cortex_gpu.yaml", node_groups=config["aws"]["x86_nodegroups"], local_operator=config["global"]["local_operator"], ) ================================================ FILE: test/e2e/tests/aws/test_load.py ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from typing import Callable, Dict import cortex as cx import pytest import e2e.tests TEST_APIS_REALTIME = ["realtime/prime-generator"] TEST_APIS_ASYNC = ["async/text-generator"] TEST_APIS_BATCH = ["batch/sum"] @pytest.mark.usefixtures("client") @pytest.mark.parametrize("api", TEST_APIS_REALTIME) def test_load_realtime(printer: Callable, config: Dict, client: cx.Client, api: str): skip_load_test = config["global"].get("skip_load", False) if skip_load_test: pytest.skip("--skip-load flag detected, skipping load tests") e2e.tests.test_load_realtime( printer, client, api, load_config=config["global"]["load_test_config"]["realtime"], deploy_timeout=config["global"]["realtime_deploy_timeout"], node_groups=config["aws"]["x86_nodegroups"], ) @pytest.mark.usefixtures("client") @pytest.mark.parametrize("api", TEST_APIS_ASYNC) def test_load_async(printer: Callable, config: Dict, client: cx.Client, api: str): skip_load_test = config["global"].get("skip_load", False) if skip_load_test: pytest.skip("--skip-load flag detected, skipping load tests") e2e.tests.test_load_async( printer, client, api, load_config=config["global"]["load_test_config"]["async"], deploy_timeout=config["global"]["async_deploy_timeout"], node_groups=config["aws"]["x86_nodegroups"], ) @pytest.mark.usefixtures("client") @pytest.mark.parametrize("api", TEST_APIS_BATCH) def test_load_batch(printer: Callable, config: Dict, client: cx.Client, api: str): skip_load_test = config["global"].get("skip_load", False) if skip_load_test: pytest.skip("--skip-load flag detected, skipping load tests") s3_path = config["aws"].get("s3_path") if not s3_path: pytest.skip( "--s3-path option is required to run batch tests (alternatively set the " "CORTEX_TEST_BATCH_S3_PATH env var) )" ) e2e.tests.test_load_batch( printer, client, api, test_s3_path=s3_path, load_config=config["global"]["load_test_config"]["batch"], deploy_timeout=config["global"]["batch_deploy_timeout"], node_groups=config["aws"]["x86_nodegroups"], ) ================================================ FILE: test/e2e/tests/aws/test_long_running.py ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from typing import Callable, Dict import cortex as cx import pytest import e2e.tests TEST_APIS = ["realtime/text-generator"] @pytest.mark.usefixtures("client") @pytest.mark.parametrize("api", TEST_APIS) def test_long_running_realtime(printer: Callable, config: Dict, client: cx.Client, api: str): skip_load_test = config["global"].get("skip_long_running", False) if skip_load_test: pytest.skip("--skip-long-running flag detected, skipping long-running test") e2e.tests.test_long_running_realtime( printer, client, api, long_running_config=config["global"]["long_running_test_config"], deploy_timeout=config["global"]["realtime_deploy_timeout"], node_groups=config["aws"]["x86_nodegroups"], ) ================================================ FILE: test/e2e/tests/aws/test_realtime.py ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from typing import Callable, Dict import cortex as cx import pytest import e2e.tests TEST_APIS = [ { "name": "realtime/image-classifier-resnet50", "extra_path": "v1/models/resnet50:predict", }, { "name": "realtime/prime-generator", "extra_path": "", }, { "name": "realtime/text-generator", "extra_path": "", }, ] TEST_APIS_ARM = [ { "name": "realtime/hello-world", "extra_path": "", }, ] TEST_APIS_GPU = [ { "name": "realtime/image-classifier-resnet50", "extra_path": "v1/models/resnet50:predict", }, { "name": "realtime/text-generator", "extra_path": "", }, ] @pytest.mark.usefixtures("client") @pytest.mark.parametrize("api", TEST_APIS, ids=[api["name"] for api in TEST_APIS]) def test_realtime_api(printer: Callable, config: Dict, client: cx.Client, api: Dict[str, str]): e2e.tests.test_realtime_api( printer=printer, client=client, api=api["name"], timeout=config["global"]["realtime_deploy_timeout"], node_groups=config["aws"]["x86_nodegroups"], extra_path=api["extra_path"], ) @pytest.mark.usefixtures("client") @pytest.mark.parametrize("api", TEST_APIS_ARM, ids=[api["name"] for api in TEST_APIS_ARM]) def test_realtime_api_arm(printer: Callable, config: Dict, client: cx.Client, api: Dict[str, str]): e2e.tests.test_realtime_api( printer=printer, client=client, api=api["name"], timeout=config["global"]["realtime_deploy_timeout"], api_config_name="cortex_cpu_arm64.yaml", node_groups=config["aws"]["arm_nodegroups"], extra_path=api["extra_path"], method="GET", ) @pytest.mark.usefixtures("client") @pytest.mark.parametrize("api", TEST_APIS_GPU, ids=[api["name"] for api in TEST_APIS_GPU]) def test_realtime_api_gpu(printer: Callable, config: Dict, client: cx.Client, api: Dict[str, str]): skip_gpus = config["global"].get("skip_gpus", False) if skip_gpus: pytest.skip("--skip-gpus flag detected, skipping GPU tests") e2e.tests.test_realtime_api( printer=printer, client=client, api=api["name"], timeout=config["global"]["realtime_deploy_timeout"], api_config_name="cortex_gpu.yaml", node_groups=config["aws"]["x86_nodegroups"], extra_path=api["extra_path"], ) ================================================ FILE: test/e2e/tests/aws/test_scale_to_zero.py ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from typing import Dict import pytest import cortex as cx import e2e.tests TEST_APIS = [{"api": "realtime/hello-world", "config": "cortex_scale_to_zero.yaml"}] @pytest.mark.parametrize("api", TEST_APIS, ids=[api["api"] for api in TEST_APIS]) @pytest.mark.usefixtures("client") def test_scale_to_zero_realtime(config: Dict, client: cx.Client, api: Dict[str, str]): e2e.tests.test_realtime_scale_to_zero( client=client, api=api["api"], timeout=config["global"]["realtime_deploy_timeout"], api_config_name=api["config"], ) ================================================ FILE: test/e2e/tests/aws/test_task.py ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from typing import Callable, Dict import cortex as cx import pytest import e2e.tests TEST_APIS = ["task/iris-classifier-trainer"] @pytest.mark.usefixtures("client") @pytest.mark.parametrize("api", TEST_APIS) def test_task_api(printer: Callable, config: Dict, client: cx.Client, api: str): e2e.tests.test_task_api( printer, client, api, retry_attempts=5, deploy_timeout=config["global"]["task_deploy_timeout"], job_timeout=config["global"]["task_job_timeout"], node_groups=config["aws"]["x86_nodegroups"], local_operator=config["global"]["local_operator"], ) ================================================ FILE: test/e2e/tests/conftest.py ================================================ # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import pytest import yaml from dotenv import load_dotenv def pytest_addoption(parser): parser.addoption( "--env", action="store", default=None, help="set cortex environment, to test on an existing cluster", ) parser.addoption( "--config", action="store", default=None, help="set cortex cluster config, to test on a new cluster", ) parser.addoption( "--s3-path", action="store", default=None, help="set s3 path where batch jobs results will be stored", ) parser.addoption( "--skip-gpus", action="store_true", help="skip GPU tests", ) parser.addoption( "--skip-infs", action="store_true", help="skip Inferentia tests", ) parser.addoption( "--skip-autoscaling", action="store_true", help="skip autoscaling tests", ) parser.addoption( "--skip-load", action="store_true", help="skip load tests", ) parser.addoption( "--skip-long-running", action="store_true", help="skip long-running test", ) parser.addoption( "--local-operator", action="store_true", help="enable for using testing against BatchAPI with a local operator", ) parser.addoption( "--arm-nodegroups", action="store", default=None, help="arm nodegroups to run the arm tests on", ) parser.addoption( "--x86-nodegroups", action="store", default=None, help="x86 nodegroups to run the x86 tests on", ) def pytest_configure(config): load_dotenv(".env") s3_path = os.environ.get("CORTEX_TEST_BATCH_S3_PATH") s3_path = config.getoption("--s3-path") if not s3_path else s3_path arm_nodegroups = [] if config.getoption("--arm-nodegroups"): arm_nodegroups = config.getoption("--arm-nodegroups").split(",") x86_nodegroups = [] if config.getoption("--x86-nodegroups"): x86_nodegroups = config.getoption("--x86-nodegroups").split(",") configuration = { "aws": { "env": config.getoption("--env"), "config": config.getoption("--config"), "s3_path": s3_path, "arm_nodegroups": arm_nodegroups, "x86_nodegroups": x86_nodegroups, }, "global": { "local_operator": config.getoption("--local-operator"), "realtime_deploy_timeout": int( os.environ.get("CORTEX_TEST_REALTIME_DEPLOY_TIMEOUT", 320) ), "batch_deploy_timeout": int(os.environ.get("CORTEX_TEST_BATCH_DEPLOY_TIMEOUT", 150)), "batch_job_timeout": int(os.environ.get("CORTEX_TEST_BATCH_JOB_TIMEOUT", 200)), "async_deploy_timeout": int(os.environ.get("CORTEX_TEST_ASYNC_DEPLOY_TIMEOUT", 320)), "async_workload_timeout": int( os.environ.get("CORTEX_TEST_ASYNC_WORKLOAD_TIMEOUT", 200) ), "task_deploy_timeout": int(os.environ.get("CORTEX_TEST_TASK_DEPLOY_TIMEOUT", 75)), "task_job_timeout": int(os.environ.get("CORTEX_TEST_TASK_JOB_TIMEOUT", 200)), "skip_gpus": config.getoption("--skip-gpus"), "skip_infs": config.getoption("--skip-infs"), "skip_autoscaling": config.getoption("--skip-autoscaling"), "skip_long_running": config.getoption("--skip-long-running"), "skip_load": config.getoption("--skip-load"), "autoscaling_test_config": { "max_replicas": 20, }, "load_test_config": { "realtime": { "total_requests": 10 ** 5, "desired_replicas": 50, "concurrency": 50, "status_code_timeout": 60, # measured in seconds }, "async": { "total_requests": 10 ** 3, "desired_replicas": 20, "concurrency": 10, "submit_timeout": 120, # measured in seconds "workload_timeout": 120, # measured in seconds }, "batch": { "jobs": 10, "workers_per_job": 10, "items_per_job": 10 ** 5, "batch_size": 10 * 2, "workload_timeout": 300, # measured in seconds }, }, "long_running_test_config": { "time_to_run": 5 * 24 * 3600, # measured in seconds "status_code_timeout": 60, # measured in seconds }, }, } class Config: @pytest.fixture(autouse=True) def config(self): return configuration config.pluginmanager.register(Config()) print("\n----- Test Configuration -----\n") print(yaml.dump(configuration, indent=2)) if configuration["aws"]["env"] and configuration["aws"]["config"]: raise ValueError("--env and --config are mutually exclusive") ================================================ FILE: test/utils/README.md ================================================ ## Throughput tester [throughput_test.py](throughput_test.py) is a Python CLI that can be used to test the throughput of your deployed API. The throughput will vary depending on your API's configuration (specified in your `cortex.yaml` file), your local machine's resources (mostly CPU, since it has to spawn many concurrent requests), and the internet connection on your local machine. ```bash Usage: throughput_test.py [OPTIONS] ENDPOINT PAYLOAD Program for testing the throughput of Cortex-deployed APIs. Options: -w, --processes INTEGER Number of processes for prediction requests. [default: 1] -t, --threads INTEGER Number of threads per process for prediction requests. [default: 1] -s, --samples INTEGER Number of samples to run per thread. [default: 10] -i, --time-based FLOAT How long the thread making predictions will run for in seconds. If set, -s option will be ignored. --help Show this message and exit. ``` `ENDPOINT` is the API's endpoint, which you can get by running `cortex get `. This argument can also be exported as an environment variable instead of being passed to the CLI. `PAYLOAD` can either be a local file or an URL resource that points to a file. The allowed extension types for the file are `json` and `jpg`. This argument can also be exported as an environment variable instead of being passed to the CLI. * `json` files are generally `sample.json`s as they are found in most Cortex examples. Each of these is attached to the request as payload. The content type of the request is `"application/json"`. * `jpg` images are read as numpy arrays and then are converted to a bytes object using `cv2.imencode` function. The content type of the request is `"application/octet-stream"`. The same payload `PAYLOAD` is attached to all requests the script makes. ### Dependencies The [throughput_test.py](throughput_test.py) CLI has been tested with Python 3.6.9. To install the CLI's dependencies, run the following: ```bash pip install requests click opencv-contrib-python numpy validator-collection imageio ``` ================================================ FILE: test/utils/build-all.sh ================================================ #!/bin/bash # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # usage: ./build-all.sh [REGISTRY] [--skip-push] # REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test set -eo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/../.. >/dev/null && pwd)" for f in $(find $ROOT/test/apis -type f -name 'build-*.sh'); do "$f" "$@" done ================================================ FILE: test/utils/build.sh ================================================ #!/bin/bash # Copyright 2022 Cortex Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # note: this only meant to be called from the build*.sh files in each test api directory # usage: ./build.sh BUILDER_PATH IMAGE_NAME [REGISTRY] [--skip-push] # PATH is e.g. /home/ubuntu/src/github.com/cortexlabs/cortex/test/apis/realtime/sleep/build-cpu.sh # REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test set -eo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/../.. >/dev/null && pwd)" source $ROOT/dev/util.sh function registry_login() { login_url=$1 # e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs/realtime-sleep-cpu region=$2 blue_echo "\nLogging in to ECR" aws ecr get-login-password --region $region | docker login --username AWS --password-stdin $login_url green_echo "\nSuccess" } function create_ecr_repo() { repo_name=$1 # e.g. cortexlabs/realtime-sleep-cpu region=$2 blue_echo "\nCreating ECR repo $repo_name" aws ecr create-repository --repository-name=$repo_name --region=$region green_echo "\nSuccess" } should_skip_push="false" positional_args=() while [[ $# -gt 0 ]]; do key="$1" case $key in -s|--skip-push) should_skip_push="true" shift ;; *) positional_args+=("$1") shift ;; esac done set -- "${positional_args[@]}" builder_path="$1" # e.g. /home/ubuntu/src/github.com/cortexlabs/cortex/test/apis/realtime/hello-world/build-cpu.sh image_name="$2" # e.g. realtime-hello-world-cpu registry=${3:-$CORTEX_DEV_DEFAULT_IMAGE_REGISTRY} # e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs if [ -z "$registry" ]; then error_echo "registry must be provided as a positional arg, or $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY must be set" fi image_url="${registry}/${image_name}" # e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs/realtime-sleep-cpu dockerfile_name="$(echo "$builder_path" | sed 's/.*build-//' | sed 's/\..*//')" # e.g. cpu api_dir="$(dirname $builder_path)" # e.g. /home/ubuntu/src/github.com/cortexlabs/cortex/test/apis/realtime/hello-world dockerfile_path="${api_dir}/${dockerfile_name}.Dockerfile" # e.g. /home/ubuntu/src/github.com/cortexlabs/cortex/test/apis/realtime/hello-world/cpu.Dockerfile if [[ "$registry" == *".ecr."* ]]; then login_url="$(echo "$registry" | sed 's/\/.*//')" # e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com repo_name="$(echo $image_url | sed 's/[^\/]*\///')" # e.g. cortexlabs/realtime-sleep-cpu region="$(echo "$registry" | sed 's/.*\.ecr\.//' | sed 's/\..*//')" # e.g. us-west-2 fi if [ ! -f "$dockerfile_path" ]; then error_echo "$dockerfile_path does not exist" exit 1 fi blue_echo "Building $image_url:latest\n" docker build "$api_dir" -f "$dockerfile_path" -t "$image_url" green_echo "\nBuilt $image_url:latest" if [ "$should_skip_push" = "true" ]; then exit 0 fi while true; do blue_echo "\nPushing $image_url:latest" exec 5>&1 set +e out=$(docker push $image_url 2>&1 | tee /dev/fd/5; exit ${PIPESTATUS[0]}) exit_code=$? set -e if [ $exit_code -ne 0 ]; then if [[ "$image_url" != *".ecr."* ]]; then exit $exit_code else if [[ "$out" == *"authorization token has expired"* ]] || [[ "$out" == *"no basic auth credentials"* ]]; then registry_login $login_url $region continue elif [[ "$out" == *"repository with name"*"does not exist"* ]]; then create_ecr_repo $repo_name $region continue else exit $exit_code fi fi fi green_echo "\nPushed $image_url:latest" break done # update api config find $api_dir -type f -name 'cortex_*.yaml' \ -exec sed -i "s|quay.io/cortexlabs-test/${image_name}|${image_url}|g" {} \; ================================================ FILE: test/utils/throughput_test.py ================================================ import os import sys import click import concurrent.futures import requests import imageio import json import time import itertools import cv2 import numpy as np from validator_collection import checkers @click.command(help="Program for testing the throughput of Cortex-deployed APIs.") @click.argument("endpoint", type=str, envvar="ENDPOINT") @click.argument("payload", type=str, envvar="PAYLOAD") @click.option( "--processes", "-p", type=int, default=1, show_default=True, help="Number of processes for requests.", ) @click.option( "--threads", "-t", type=int, default=1, show_default=True, help="Number of threads per process for requests.", ) @click.option( "--samples", "-s", type=int, default=10, show_default=True, help="Number of samples to run per thread.", ) @click.option( "--time-based", "-i", type=float, default=0.0, help="How long the thread making requests will run for in seconds. If set, -s option will be ignored.", ) def main(payload, endpoint, processes, threads, samples, time_based): file_type = None if checkers.is_url(payload): if payload.lower().endswith(".json"): file_type = "json" payload_data = requests.get(payload).json() elif payload.lower().endswith(".jpg"): file_type = "jpg" payload_data = imageio.imread(payload) elif checkers.is_file(payload): if payload.lower().endswith(".json"): file_type = "json" with open(payload, "r") as f: payload_data = json.load(f) elif payload.lower().endswith(".jpg"): file_type = "jpg" payload_data = cv2.imread(payload, cv2.IMREAD_COLOR) else: print(f"'{payload}' isn't an URL resource, nor is it a local file") sys.exit(1) if file_type is None: print(f"'{payload}' doesn't point to a jpg image or to a json file") sys.exit(1) if file_type == "jpg": data = image_to_jpeg_bytes(payload_data) if file_type == "json": data = json.dumps(payload_data) print("Starting the inference throughput test...") results = [] start = time.time() with concurrent.futures.ProcessPoolExecutor(max_workers=processes) as executor: results = executor_submitter( executor, processes, process_worker, threads, data, endpoint, samples, time_based ) end = time.time() elapsed = end - start total_requests = sum(results) print(f"A total of {total_requests} requests have been served in {elapsed} seconds") print(f"Avg number of inferences/sec is {total_requests / elapsed}") print(f"Avg time spent on an inference is {elapsed / total_requests} seconds") def process_worker(threads, data, endpoint, samples, time_based): results = [] with concurrent.futures.ThreadPoolExecutor(max_workers=threads) as executor: results = executor_submitter(executor, threads, task, data, endpoint, samples, time_based) return results def executor_submitter(executor, workers, *args, **kwargs): futures = [] for worker in range(workers): future = executor.submit(*args, **kwargs) futures.append(future) results = [future.result() for future in futures] results = list(itertools.chain.from_iterable(results)) return results def task(data, endpoint, samples, time_based): timeout = 60 if isinstance(data, str): headers = {"content-type": "application/json"} elif isinstance(data, bytes): headers = {"content-type": "application/octet-stream"} else: return if time_based == 0.0: for i in range(samples): try: resp = requests.post( endpoint, data=data, headers=headers, timeout=timeout, ) except Exception as e: print(e) break time.sleep(0.1) return [samples] else: start = time.time() counter = 0 while start + time_based >= time.time(): try: resp = requests.post( endpoint, data=data, headers=headers, timeout=timeout, ) except Exception as e: print(e) break time.sleep(0.1) counter += 1 return [counter] def image_to_jpeg_nparray(image, quality=[int(cv2.IMWRITE_JPEG_QUALITY), 95]): """ Convert numpy image to jpeg numpy vector. """ is_success, im_buf_arr = cv2.imencode(".jpg", image, quality) return im_buf_arr def image_to_jpeg_bytes(image, quality=[int(cv2.IMWRITE_JPEG_QUALITY), 95]): """ Convert numpy image to bytes-encoded jpeg image. """ buf = image_to_jpeg_nparray(image, quality) byte_im = buf.tobytes() return byte_im if __name__ == "__main__": main()