Repository: tensorchord/openmodelz Branch: main Commit: b574e2bd838c Files: 413 Total size: 1.1 MB Directory structure: gitextract_h_b9vahq/ ├── .all-contributorsrc ├── .github/ │ └── workflows/ │ ├── CI.yaml │ └── publish.yaml ├── .gitignore ├── .goreleaser.yaml ├── LICENSE ├── MANIFEST.in ├── README.md ├── agent/ │ ├── .gitignore │ ├── Dockerfile │ ├── Makefile │ ├── README.md │ ├── api/ │ │ └── types/ │ │ ├── build.go │ │ ├── error.go │ │ ├── event.go │ │ ├── inference_deployment.go │ │ ├── inference_deployment_instance.go │ │ ├── inference_status.go │ │ ├── info.go │ │ ├── log.go │ │ ├── modelz_cloud.go │ │ ├── namespace.go │ │ ├── queue.go │ │ ├── requests.go │ │ ├── secret.go │ │ └── server.go │ ├── client/ │ │ ├── build.go │ │ ├── client.go │ │ ├── const.go │ │ ├── errors.go │ │ ├── hijack.go │ │ ├── image_cache_create.go │ │ ├── inference_create.go │ │ ├── inference_get.go │ │ ├── inference_list.go │ │ ├── inference_remove.go │ │ ├── inference_scale.go │ │ ├── inference_update.go │ │ ├── info.go │ │ ├── instance_exec.go │ │ ├── instance_list.go │ │ ├── log.go │ │ ├── modelz_cloud.go │ │ ├── namespace_create.go │ │ ├── namespace_delete.go │ │ ├── options.go │ │ ├── request.go │ │ ├── server_label_create.go │ │ ├── server_list.go │ │ ├── server_node_delete.go │ │ ├── transport.go │ │ └── utils.go │ ├── cmd/ │ │ └── agent/ │ │ └── main.go │ ├── errdefs/ │ │ ├── defs.go │ │ ├── doc.go │ │ ├── helpers.go │ │ ├── http_helpers.go │ │ └── is.go │ ├── pkg/ │ │ ├── app/ │ │ │ ├── config.go │ │ │ └── root.go │ │ ├── config/ │ │ │ └── config.go │ │ ├── consts/ │ │ │ └── consts.go │ │ ├── docs/ │ │ │ └── docs.go │ │ ├── event/ │ │ │ ├── event.go │ │ │ ├── fake.go │ │ │ ├── suite_test.go │ │ │ ├── username.go │ │ │ └── util.go │ │ ├── k8s/ │ │ │ ├── convert_inference.go │ │ │ ├── convert_inference_test.go │ │ │ ├── convert_job.go │ │ │ ├── convert_pod.go │ │ │ ├── convert_pod_test.go │ │ │ ├── generate_image_cache.go │ │ │ ├── generate_job.go │ │ │ ├── managed_cluster.go │ │ │ ├── resolver.go │ │ │ └── suite_test.go │ │ ├── log/ │ │ │ ├── factory.go │ │ │ ├── k8s.go │ │ │ └── loki.go │ │ ├── metrics/ │ │ │ ├── exporter.go │ │ │ └── metrics.go │ │ ├── prom/ │ │ │ └── prometheus_query.go │ │ ├── runtime/ │ │ │ ├── build.go │ │ │ ├── cluster_info_get.go │ │ │ ├── image_cache.go │ │ │ ├── inference_create.go │ │ │ ├── inference_delete.go │ │ │ ├── inference_exec.go │ │ │ ├── inference_get.go │ │ │ ├── inference_instance.go │ │ │ ├── inference_list.go │ │ │ ├── inference_replicas.go │ │ │ ├── inference_update.go │ │ │ ├── mock/ │ │ │ │ └── mock.go │ │ │ ├── namespace.go │ │ │ ├── node.go │ │ │ ├── runtime.go │ │ │ ├── server_delete.go │ │ │ ├── server_label_create.go │ │ │ ├── server_list.go │ │ │ ├── util_domain.go │ │ │ └── util_resource.go │ │ ├── scaling/ │ │ │ ├── function_scaler.go │ │ │ ├── ranges.go │ │ │ ├── retry.go │ │ │ ├── service_query.go │ │ │ └── util.go │ │ ├── server/ │ │ │ ├── error.go │ │ │ ├── handler_build_create.go │ │ │ ├── handler_build_get.go │ │ │ ├── handler_build_list.go │ │ │ ├── handler_build_logs.go │ │ │ ├── handler_gradio_proxy.go │ │ │ ├── handler_healthz.go │ │ │ ├── handler_healthz_test.go │ │ │ ├── handler_image_cache.go │ │ │ ├── handler_inference_create.go │ │ │ ├── handler_inference_create_test.go │ │ │ ├── handler_inference_delete.go │ │ │ ├── handler_inference_delete_test.go │ │ │ ├── handler_inference_get.go │ │ │ ├── handler_inference_get_test.go │ │ │ ├── handler_inference_instance.go │ │ │ ├── handler_inference_instance_exec.go │ │ │ ├── handler_inference_list.go │ │ │ ├── handler_inference_logs.go │ │ │ ├── handler_inference_proxy.go │ │ │ ├── handler_inference_scale.go │ │ │ ├── handler_inference_update.go │ │ │ ├── handler_info.go │ │ │ ├── handler_mosec_proxy.go │ │ │ ├── handler_namespace_create.go │ │ │ ├── handler_namespace_delete.go │ │ │ ├── handler_namespace_delete_test.go │ │ │ ├── handler_namespace_list.go │ │ │ ├── handler_other_proxy.go │ │ │ ├── handler_root.go │ │ │ ├── handler_server_delete.go │ │ │ ├── handler_server_label_create.go │ │ │ ├── handler_server_list.go │ │ │ ├── handler_streamlit_proxy.go │ │ │ ├── middleware_callid.go │ │ │ ├── proxy_auth.go │ │ │ ├── server_factory.go │ │ │ ├── server_handlerfunc.go │ │ │ ├── server_init_kubernetes.go │ │ │ ├── server_init_logs.go │ │ │ ├── server_init_metrics.go │ │ │ ├── server_init_modelz_cloud.go │ │ │ ├── server_init_route.go │ │ │ ├── server_run.go │ │ │ ├── server_websocket.go │ │ │ ├── static/ │ │ │ │ ├── index.html │ │ │ │ ├── landing.go │ │ │ │ └── page_loading.go │ │ │ ├── suite_test.go │ │ │ ├── user.go │ │ │ └── validator/ │ │ │ └── validator.go │ │ └── version/ │ │ └── version.go │ └── sqlc.yaml ├── autoscaler/ │ ├── .gitignore │ ├── Dockerfile │ ├── Makefile │ ├── cmd/ │ │ └── autoscaler/ │ │ └── main.go │ └── pkg/ │ ├── autoscaler/ │ │ ├── factory.go │ │ ├── inferencecache.go │ │ ├── loadcache.go │ │ └── scaler.go │ ├── autoscalerapp/ │ │ └── root.go │ ├── prom/ │ │ ├── prom.go │ │ └── types.go │ ├── server/ │ │ └── status.go │ └── version/ │ └── version.go ├── go.mod ├── go.sum ├── ingress-operator/ │ ├── .DEREK.yml │ ├── .dockerignore │ ├── .gitignore │ ├── .tools/ │ │ ├── README.md │ │ ├── code-generator.mod │ │ └── code-generator.sum │ ├── .vscode/ │ │ └── settings.json │ ├── Dockerfile │ ├── LICENSE │ ├── Makefile │ ├── artifacts/ │ │ ├── .gitignore │ │ ├── crds/ │ │ │ └── tensorchord.ai_inferenceingresses.yaml │ │ ├── operator-amd64.yaml │ │ └── operator-rbac.yaml │ ├── cmd/ │ │ └── ingress-operator/ │ │ └── main.go │ ├── hack/ │ │ ├── boilerplate.go.txt │ │ ├── custom-boilerplate.go.txt │ │ ├── print-codegen-version.sh │ │ ├── update-codegen.sh │ │ ├── update-crds.sh │ │ └── verify-codegen.sh │ ├── pkg/ │ │ ├── apis/ │ │ │ └── modelzetes/ │ │ │ ├── register.go │ │ │ └── v1/ │ │ │ ├── doc.go │ │ │ ├── register.go │ │ │ ├── types.go │ │ │ └── zz_generated.deepcopy.go │ │ ├── app/ │ │ │ ├── config.go │ │ │ └── root.go │ │ ├── client/ │ │ │ ├── clientset/ │ │ │ │ └── versioned/ │ │ │ │ ├── clientset.go │ │ │ │ ├── doc.go │ │ │ │ ├── fake/ │ │ │ │ │ ├── clientset_generated.go │ │ │ │ │ ├── doc.go │ │ │ │ │ └── register.go │ │ │ │ ├── scheme/ │ │ │ │ │ ├── doc.go │ │ │ │ │ └── register.go │ │ │ │ └── typed/ │ │ │ │ └── modelzetes/ │ │ │ │ └── v1/ │ │ │ │ ├── doc.go │ │ │ │ ├── fake/ │ │ │ │ │ ├── doc.go │ │ │ │ │ ├── fake_inferenceingress.go │ │ │ │ │ └── fake_modelzetes_client.go │ │ │ │ ├── generated_expansion.go │ │ │ │ ├── inferenceingress.go │ │ │ │ └── modelzetes_client.go │ │ │ ├── informers/ │ │ │ │ └── externalversions/ │ │ │ │ ├── factory.go │ │ │ │ ├── generic.go │ │ │ │ ├── internalinterfaces/ │ │ │ │ │ └── factory_interfaces.go │ │ │ │ └── modelzetes/ │ │ │ │ ├── interface.go │ │ │ │ └── v1/ │ │ │ │ ├── inferenceingress.go │ │ │ │ └── interface.go │ │ │ └── listers/ │ │ │ └── modelzetes/ │ │ │ └── v1/ │ │ │ ├── expansion_generated.go │ │ │ └── inferenceingress.go │ │ ├── config/ │ │ │ └── config.go │ │ ├── consts/ │ │ │ └── consts.go │ │ ├── controller/ │ │ │ ├── core.go │ │ │ ├── core_test.go │ │ │ └── v1/ │ │ │ ├── controller.go │ │ │ ├── controller_factory.go │ │ │ ├── controller_test.go │ │ │ └── docs.go │ │ ├── signals/ │ │ │ ├── signal.go │ │ │ ├── signal_posix.go │ │ │ └── signal_windows.go │ │ └── version/ │ │ └── version.go │ └── vendor.go ├── mdz/ │ ├── .gitignore │ ├── Makefile │ ├── README.md │ ├── cmd/ │ │ └── mdz/ │ │ └── main.go │ ├── docs/ │ │ ├── cli/ │ │ │ ├── mdz.md │ │ │ ├── mdz_delete.md │ │ │ ├── mdz_deploy.md │ │ │ ├── mdz_exec.md │ │ │ ├── mdz_list.md │ │ │ ├── mdz_list_instance.md │ │ │ ├── mdz_logs.md │ │ │ ├── mdz_port-forward.md │ │ │ ├── mdz_scale.md │ │ │ ├── mdz_server.md │ │ │ ├── mdz_server_delete.md │ │ │ ├── mdz_server_destroy.md │ │ │ ├── mdz_server_join.md │ │ │ ├── mdz_server_label.md │ │ │ ├── mdz_server_list.md │ │ │ ├── mdz_server_start.md │ │ │ ├── mdz_server_stop.md │ │ │ └── mdz_version.md │ │ └── macOS-quickstart.md │ ├── examples/ │ │ └── bloomz-560m-openai/ │ │ └── README.md │ ├── hack/ │ │ └── cli-doc-gen/ │ │ └── main.go │ └── pkg/ │ ├── agentd/ │ │ ├── runtime/ │ │ │ ├── create.go │ │ │ ├── delete.go │ │ │ ├── label.go │ │ │ ├── list.go │ │ │ ├── proxy.go │ │ │ └── runtime.go │ │ └── server/ │ │ ├── error.go │ │ ├── handler_healthz.go │ │ ├── handler_inference_create.go │ │ ├── handler_inference_delete.go │ │ ├── handler_inference_get.go │ │ ├── handler_inference_list.go │ │ ├── handler_inference_logs.go │ │ ├── handler_inference_proxy.go │ │ ├── handler_info.go │ │ ├── middleware_callid.go │ │ ├── server_factory.go │ │ ├── server_handlerfunc.go │ │ ├── server_init_route.go │ │ └── server_run.go │ ├── cmd/ │ │ ├── delete.go │ │ ├── deploy.go │ │ ├── exec.go │ │ ├── exec_stream.go │ │ ├── ioutils/ │ │ │ └── reader.go │ │ ├── list.go │ │ ├── list_instance.go │ │ ├── localagent.go │ │ ├── logs.go │ │ ├── portforward.go │ │ ├── root.go │ │ ├── scale.go │ │ ├── server.go │ │ ├── server_delete.go │ │ ├── server_destroy.go │ │ ├── server_join.go │ │ ├── server_label.go │ │ ├── server_list.go │ │ ├── server_start.go │ │ ├── server_stop.go │ │ ├── streams/ │ │ │ ├── in.go │ │ │ ├── out.go │ │ │ └── stream.go │ │ └── version.go │ ├── server/ │ │ ├── agentd_run.go │ │ ├── engine.go │ │ ├── gpu-resource.yaml │ │ ├── gpu_install.go │ │ ├── k3s-install.sh │ │ ├── k3s_destroy.go │ │ ├── k3s_install.go │ │ ├── k3s_join.go │ │ ├── k3s_killall.go │ │ ├── k3s_prepare.go │ │ ├── nginx-dep.yaml │ │ ├── nginx_install.go │ │ ├── openmodelz.yaml │ │ ├── openmodelz_install.go │ │ └── registries.yaml │ ├── telemetry/ │ │ └── telemetry.go │ ├── term/ │ │ ├── interrupt.go │ │ └── term.go │ └── version/ │ └── version.go ├── modelzetes/ │ ├── .dockerignore │ ├── .gitattributes │ ├── .gitignore │ ├── Dockerfile │ ├── LICENSE │ ├── Makefile │ ├── artifacts/ │ │ ├── crds/ │ │ │ └── tensorchord.ai_inferences.yaml │ │ └── samples/ │ │ └── v2alpha1.yaml │ ├── buf.yaml │ ├── cmd/ │ │ └── modelzetes/ │ │ └── main.go │ ├── hack/ │ │ ├── boilerplate.go.txt │ │ ├── print-codegen-version.sh │ │ ├── update-codegen.sh │ │ ├── update-crds.sh │ │ └── verify-codegen.sh │ ├── pkg/ │ │ ├── apis/ │ │ │ └── modelzetes/ │ │ │ ├── register.go │ │ │ └── v2alpha1/ │ │ │ ├── doc.go │ │ │ ├── register.go │ │ │ ├── types.go │ │ │ └── zz_generated.deepcopy.go │ │ ├── app/ │ │ │ ├── config.go │ │ │ └── root.go │ │ ├── client/ │ │ │ ├── clientset/ │ │ │ │ └── versioned/ │ │ │ │ ├── clientset.go │ │ │ │ ├── doc.go │ │ │ │ ├── fake/ │ │ │ │ │ ├── clientset_generated.go │ │ │ │ │ ├── doc.go │ │ │ │ │ └── register.go │ │ │ │ ├── scheme/ │ │ │ │ │ ├── doc.go │ │ │ │ │ └── register.go │ │ │ │ └── typed/ │ │ │ │ └── modelzetes/ │ │ │ │ └── v2alpha1/ │ │ │ │ ├── doc.go │ │ │ │ ├── fake/ │ │ │ │ │ ├── doc.go │ │ │ │ │ ├── fake_inference.go │ │ │ │ │ └── fake_modelzetes_client.go │ │ │ │ ├── generated_expansion.go │ │ │ │ ├── inference.go │ │ │ │ └── modelzetes_client.go │ │ │ ├── informers/ │ │ │ │ └── externalversions/ │ │ │ │ ├── factory.go │ │ │ │ ├── generic.go │ │ │ │ ├── internalinterfaces/ │ │ │ │ │ └── factory_interfaces.go │ │ │ │ └── modelzetes/ │ │ │ │ ├── interface.go │ │ │ │ └── v2alpha1/ │ │ │ │ ├── inference.go │ │ │ │ └── interface.go │ │ │ └── listers/ │ │ │ └── modelzetes/ │ │ │ └── v2alpha1/ │ │ │ ├── expansion_generated.go │ │ │ └── inference.go │ │ ├── config/ │ │ │ └── config.go │ │ ├── consts/ │ │ │ └── consts.go │ │ ├── controller/ │ │ │ ├── annotations_test.go │ │ │ ├── controller.go │ │ │ ├── deployment.go │ │ │ ├── deployment_test.go │ │ │ ├── deployment_update_test.go │ │ │ ├── factory.go │ │ │ ├── framework_test.go │ │ │ ├── fromconfig.go │ │ │ ├── replicas_test.go │ │ │ ├── secrets.go │ │ │ ├── secrets_test.go │ │ │ ├── service.go │ │ │ └── service_test.go │ │ ├── k8s/ │ │ │ ├── config.go │ │ │ ├── errors.go │ │ │ ├── factory.go │ │ │ ├── factory_test.go │ │ │ ├── instance.go │ │ │ ├── instance_test.go │ │ │ ├── log.go │ │ │ ├── logs.go │ │ │ ├── probes.go │ │ │ ├── probes_test.go │ │ │ ├── proxy.go │ │ │ ├── proxy_test.go │ │ │ ├── secrets.go │ │ │ ├── secrets_factory_test.go │ │ │ ├── securityContext.go │ │ │ ├── securityContext_test.go │ │ │ └── utils.go │ │ ├── pointer/ │ │ │ └── ptr.go │ │ ├── signals/ │ │ │ ├── signal.go │ │ │ ├── signal_posix.go │ │ │ └── signal_windows.go │ │ └── version/ │ │ └── version.go │ └── vendor.go ├── pyproject.toml ├── setup.py └── typos.toml ================================================ FILE CONTENTS ================================================ ================================================ FILE: .all-contributorsrc ================================================ { "projectName": "openmodelz", "projectOwner": "tensorchord", "repoType": "github", "repoHost": "https://github.com", "files": [ "README.md" ], "imageSize": 70, "commit": true, "commitConvention": "angular", "contributorsSortAlphabetically": true, "contributors": [ { "login": "gaocegege", "name": "Ce Gao", "avatar_url": "https://avatars.githubusercontent.com/u/5100735?v=4", "profile": "https://github.com/gaocegege", "contributions": [ "code", "review", "tutorial" ] }, { "login": "tddschn", "name": "Teddy Xinyuan Chen", "avatar_url": "https://avatars.githubusercontent.com/u/45612704?v=4", "profile": "https://github.com/tddschn", "contributions": [ "doc" ] }, { "login": "VoVAllen", "name": "Jinjing Zhou", "avatar_url": "https://avatars.githubusercontent.com/u/8686776?v=4", "profile": "https://github.com/VoVAllen", "contributions": [ "question", "bug", "ideas" ] }, { "login": "kemingy", "name": "Keming", "avatar_url": "https://avatars.githubusercontent.com/u/12974685?v=4", "profile": "https://blog.mapotofu.org/", "contributions": [ "code", "design", "infra" ] }, { "login": "cutecutecat", "name": "cutecutecat", "avatar_url": "https://avatars.githubusercontent.com/u/19801166?v=4", "profile": "https://github.com/cutecutecat", "contributions": [ "ideas" ] }, { "login": "xieydd", "name": "xieydd", "avatar_url": "https://avatars.githubusercontent.com/u/20329697?v=4", "profile": "https://xieydd.github.io/", "contributions": [ "ideas" ] }, { "login": "Xuanwo", "name": "Xuanwo", "avatar_url": "https://avatars.githubusercontent.com/u/5351546?v=4", "profile": "https://xuanwo.io/", "contributions": [ "content", "design", "ideas" ] }, { "login": "Zheaoli", "name": "Nadeshiko Manju", "avatar_url": "https://avatars.githubusercontent.com/u/7054676?v=4", "profile": "http://manjusaka.itscoder.com/", "contributions": [ "bug", "design", "ideas" ] }, { "login": "zwpaper", "name": "Wei Zhang", "avatar_url": "https://avatars.githubusercontent.com/u/3764335?v=4", "profile": "https://page.codespaper.com", "contributions": [ "code" ] } ], "contributorsPerLine": 7, "commitType": "docs" } ================================================ FILE: .github/workflows/CI.yaml ================================================ name: CI on: push: branches: - main paths: - '.github/workflows/**' - '**.go' - '**/Makefile' - 'go.**' pull_request: paths: - '.github/workflows/**' - '**.go' - '**/Makefile' - 'go.**' concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: typos-check: name: Spell Check with Typos runs-on: ubuntu-latest steps: - name: Checkout Actions Repository uses: actions/checkout@v3 - name: Check spelling with custom config file uses: crate-ci/typos@v1.16.2 with: config: ./typos.toml test: name: test strategy: matrix: os: [ubuntu-latest] dir: ["agent", "autoscaler", "ingress-operator", "mdz", "modelzetes"] runs-on: ${{ matrix.os }} steps: - name: Check out code uses: actions/checkout@v3 - name: Setup Go uses: actions/setup-go@v4 with: go-version: 1.19 - uses: actions/cache@v3 with: path: | ~/.cache/go-build ~/go/pkg/mod key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} restore-keys: | ${{ runner.os }}-go- - name: test run: | cd ${{ matrix.dir }} make fmt git diff --exit-code || (echo 'Please run "make fmt" to format code' && exit 1); make go test -race -coverprofile=${{ matrix.dir }}.out -covermode=atomic ./... - name: Upload coverage report uses: actions/upload-artifact@v3 with: name: ${{ matrix.dir }}-out path: ${{ matrix.dir }}/${{ matrix.dir }}.out report: needs: - test - typos-check runs-on: ubuntu-latest steps: - name: Check out code uses: actions/checkout@v3 - name: Setup Go uses: actions/setup-go@v4 with: go-version: 1.19 - name: Install bins run: | go install github.com/mattn/goveralls@latest - name: Get agent coverage report uses: actions/download-artifact@v3 with: name: agent-out path: merge - name: Get autoscaler coverage report uses: actions/download-artifact@v3 with: name: autoscaler-out path: merge - name: Get ingress-operator coverage report uses: actions/download-artifact@v3 with: name: ingress-operator-out path: merge - name: Get mdz coverage report uses: actions/download-artifact@v3 with: name: mdz-out path: merge - name: Get modelzetes coverage report uses: actions/download-artifact@v3 with: name: modelzetes-out path: merge - name: Merge all coverage reports uses: cutecutecat/go-cover-merge@v1 with: input_dir: merge output_file: final.out - name: Send coverage env: COVERALLS_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | goveralls -coverprofile=final.out -service=github ================================================ FILE: .github/workflows/publish.yaml ================================================ name: release on: release: types: [published] pull_request: paths: - '.github/workflows/release.yml' - '.goreleaser/' - '.goreleaser.yaml' jobs: goreleaser: if: github.repository == 'tensorchord/openmodelz' runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v3 with: fetch-depth: 0 - name: Set up Go uses: actions/setup-go@v4 with: go-version: 1.19 - name: Docker Login uses: docker/login-action@v2 with: username: ${{ secrets.DOCKERIO_USERNAME }} password: ${{ secrets.DOCKERIO_TOKEN }} - name: Run GoReleaser uses: goreleaser/goreleaser-action@v4 with: distribution: goreleaser version: latest args: release --rm-dist env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: upload gobin uses: actions/upload-artifact@v3 with: name: gobin_${{ github.event.release.tag_name }} retention-days: 1 path: | dist/mdz_linux_amd64_v1/mdz if-no-files-found: error pypi_publish: needs: goreleaser # only trigger on main repo when tag starts with v if: github.repository == 'tensorchord/openmodelz' && startsWith(github.ref, 'refs/tags/v') runs-on: ${{ matrix.os }} timeout-minutes: 20 strategy: matrix: os: [ubuntu-20.04] steps: - uses: actions/checkout@v3 - name: Get gobin uses: actions/download-artifact@v3 with: name: gobin_${{ github.event.release.tag_name }} path: dist/ - name: Configure linux build environment if: runner.os == 'Linux' run: | mkdir -p mdz/bin mv dist/mdz mdz/bin/mdz chmod +x mdz/bin/mdz - name: Build wheels uses: pypa/cibuildwheel@v2.14.1 - name: Build source distribution if: runner.os == 'Linux' # Only release source under linux to avoid conflict run: | python -m pip install wheel setuptools_scm python setup.py sdist mv dist/*.tar.gz wheelhouse/ - name: Upload to PyPI env: TWINE_USERNAME: __token__ TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} run: | python -m pip install --upgrade pip python -m pip install twine python -m twine upload wheelhouse/* ================================================ FILE: .gitignore ================================================ # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] *$py.class # C extensions *.so # Distribution / packaging .Python build/ develop-eggs/ dist/ downloads/ eggs/ .eggs/ lib/ lib64/ parts/ sdist/ var/ wheels/ share/python-wheels/ *.egg-info/ .installed.cfg *.egg MANIFEST _version.txt _version.py wheelhouse/ # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest *.spec # Installer logs pip-log.txt pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ .nox/ .coverage .coverage.* .cache .ruff_cache/ nosetests.xml coverage.xml *.cover *.py,cover .hypothesis/ .pytest_cache/ cover/ # Translations *.mo *.pot # Django stuff: *.log local_settings.py db.sqlite3 db.sqlite3-journal # Flask stuff: instance/ .webassets-cache # Scrapy stuff: .scrapy # Sphinx documentation docs/_build/ # PyBuilder .pybuilder/ target/ # Jupyter Notebook .ipynb_checkpoints # IPython profile_default/ ipython_config.py # pyenv # For a library or package, you might want to ignore these files since the code is # intended to run in multiple environments; otherwise, check them in: # .python-version # pipenv # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. # However, in case of collaboration, if having platform-specific dependencies or dependencies # having no cross-platform support, pipenv may install dependencies that don't work, or not # install all needed dependencies. #Pipfile.lock # poetry # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. # This is especially recommended for binary packages to ensure reproducibility, and is more # commonly ignored for libraries. # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control #poetry.lock # pdm # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. #pdm.lock # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it # in version control. # https://pdm.fming.dev/#use-with-ide .pdm.toml # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm __pypackages__/ # Celery stuff celerybeat-schedule celerybeat.pid # SageMath parsed files *.sage.py # Environments .env .venv env/ venv/ ENV/ env.bak/ venv.bak/ # Spyder project settings .spyderproject .spyproject # Rope project settings .ropeproject # mkdocs documentation /site # mypy .mypy_cache/ .dmypy.json dmypy.json # Pyre type checker .pyre/ # pytype static type analyzer .pytype/ # Cython debug symbols cython_debug/ # PyCharm # JetBrains specific template is maintained in a separate JetBrains.gitignore that can # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. .idea/ # If you prefer the allow list template instead of the deny list, see community template: # https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore # # Binaries for programs and plugins *.exe *.exe~ *.dll *.so *.dylib # Test binary, built with `go test -c` *.test # Output of the go coverage tool, specifically when used with LiteIDE *.out # Dependency directories (remove the comment below to include it) # vendor/ # Go workspace file go.work ================================================ FILE: .goreleaser.yaml ================================================ project_name: openmodelz builds: - env: - CGO_ENABLED=0 goos: - linux goarch: - amd64 id: modelzetes main: ./cmd/modelzetes/main.go dir: ./modelzetes binary: modelzetes ldflags: - -s -w - -X github.com/tensorchord/openmodelz/modelzetes/pkg/version.version={{ .Version }} - -X github.com/tensorchord/openmodelz/modelzetes/pkg/version.buildDate={{ .Date }} - -X github.com/tensorchord/openmodelz/modelzetes/pkg/version.gitCommit={{ .Commit }} - -X github.com/tensorchord/openmodelz/modelzetes/pkg/version.gitTreeState=clean - -X github.com/tensorchord/openmodelz/modelzetes/pkg/version.gitTag={{ .Tag }} - env: - CGO_ENABLED=0 goos: - linux goarch: - amd64 id: agent main: ./cmd/agent/main.go dir: ./agent binary: agent ldflags: - -s -w - -X github.com/tensorchord/openmodelz/agent/pkg/version.version={{ .Version }} - -X github.com/tensorchord/openmodelz/agent/pkg/version.buildDate={{ .Date }} - -X github.com/tensorchord/openmodelz/agent/pkg/version.gitCommit={{ .Commit }} - -X github.com/tensorchord/openmodelz/agent/pkg/version.gitTreeState=clean - -X github.com/tensorchord/openmodelz/agent/pkg/version.gitTag={{ .Tag }} - env: - CGO_ENABLED=0 goos: - linux goarch: - amd64 id: mdz main: ./cmd/mdz/main.go dir: ./mdz binary: mdz ldflags: - -s -w - -X github.com/tensorchord/openmodelz/mdz/pkg/version.version={{ .Version }} - -X github.com/tensorchord/openmodelz/mdz/pkg/version.buildDate={{ .Date }} - -X github.com/tensorchord/openmodelz/mdz/pkg/version.gitCommit={{ .Commit }} - -X github.com/tensorchord/openmodelz/mdz/pkg/version.gitTreeState=clean - -X github.com/tensorchord/openmodelz/mdz/pkg/version.gitTag={{ .Tag }} - env: - CGO_ENABLED=0 goos: - linux goarch: - amd64 id: autoscaler main: ./cmd/autoscaler/main.go dir: ./autoscaler binary: autoscaler ldflags: - -s -w - -X github.com/tensorchord/openmodelz/autoscaler/pkg/version.version={{ .Version }} - -X github.com/tensorchord/openmodelz/autoscaler/pkg/version.buildDate={{ .Date }} - -X github.com/tensorchord/openmodelz/autoscaler/pkg/version.gitCommit={{ .Commit }} - -X github.com/tensorchord/openmodelz/autoscaler/pkg/version.gitTreeState=clean - -X github.com/tensorchord/openmodelz/autoscaler/pkg/version.gitTag={{ .Tag }} - env: - CGO_ENABLED=0 goos: - linux goarch: - amd64 id: ingress-operator main: ./cmd/ingress-operator/main.go dir: ./ingress-operator binary: ingress-operator ldflags: - -s -w - -X github.com/tensorchord/openmodelz/ingress-operator/pkg/version.version={{ .Version }} - -X github.com/tensorchord/openmodelz/ingress-operator/pkg/version.buildDate={{ .Date }} - -X github.com/tensorchord/openmodelz/ingress-operator/pkg/version.gitCommit={{ .Commit }} - -X github.com/tensorchord/openmodelz/ingress-operator/pkg/version.gitTreeState=clean - -X github.com/tensorchord/openmodelz/ingress-operator/pkg/version.gitTag={{ .Tag }} archives: - id: mdz format: binary builds: - mdz name_template: >- {{ .Binary }}_{{ .Version }}_{{- title .Os }}_ {{- if eq .Arch "amd64" }}x86_64 {{- else if eq .Arch "386" }}i386 {{- else }}{{ .Arch }}{{ end }} checksum: name_template: 'checksums.txt' snapshot: name_template: "{{ incpatch .Version }}-next" changelog: use: github sort: asc groups: - title: 'Exciting New Features 🎉' regexp: "^.*feat.*" order: 0 - title: 'Bug Fix 🛠' regexp: "^.*(Fix|fix|bug).*" order: 1 - title: 'Refactor 🏗️' regexp: "^.*refact.*" order: 2 - title: 'Documentation 🖊️' regexp: "^.*docs.*" order: 3 - title: 'Others:' order: 999 dockers: - image_templates: - "modelzai/openmodelz-modelzetes:v{{ .Version }}-amd64" use: buildx dockerfile: modelzetes/Dockerfile ids: - modelzetes build_flag_templates: - "--platform=linux/amd64" # - image_templates: # - "modelzai/modelzetes:v{{ .Version }}-arm64v8" # use: buildx # goarch: arm64 # ids: # - modelzetes # dockerfile: modelzetes/Dockerfile # build_flag_templates: # - "--platform=linux/arm64/v8" - image_templates: - "modelzai/openmodelz-agent:v{{ .Version }}-amd64" use: buildx dockerfile: agent/Dockerfile ids: - agent build_flag_templates: - "--platform=linux/amd64" # - image_templates: # - "modelzai/modelz-agent:v{{ .Version }}-arm64v8" # use: buildx # goarch: arm64 # ids: # - agent # dockerfile: agent/Dockerfile # build_flag_templates: # - "--platform=linux/arm64/v8" - image_templates: - "modelzai/openmodelz-autoscaler:v{{ .Version }}-amd64" use: buildx dockerfile: autoscaler/Dockerfile ids: - autoscaler build_flag_templates: - "--platform=linux/amd64" # - image_templates: # - "modelzai/modelz-autoscaler:v{{ .Version }}-arm64v8" # use: buildx # goarch: arm64 # ids: # - autoscaler # dockerfile: autoscaler/Dockerfile # build_flag_templates: # - "--platform=linux/arm64/v8" - image_templates: - "modelzai/openmodelz-ingress-operator:v{{ .Version }}-amd64" use: buildx dockerfile: ingress-operator/Dockerfile ids: - ingress-operator build_flag_templates: - "--platform=linux/amd64" docker_manifests: - name_template: modelzai/openmodelz-modelzetes:v{{ .Version }} image_templates: - modelzai/openmodelz-modelzetes:v{{ .Version }}-amd64 # - modelzai/modelzetes:v{{ .Version }}-arm64v8 - name_template: modelzai/openmodelz-agent:v{{ .Version }} image_templates: - modelzai/openmodelz-agent:v{{ .Version }}-amd64 # - modelzai/modelz-agent:v{{ .Version }}-arm64v8 - name_template: modelzai/openmodelz-autoscaler:v{{ .Version }} image_templates: - modelzai/openmodelz-autoscaler:v{{ .Version }}-amd64 # - modelzai/modelz-autoscaler:v{{ .Version }}-arm64v8 - name_template: modelzai/openmodelz-ingress-operator:v{{ .Version }} image_templates: - modelzai/openmodelz-ingress-operator:v{{ .Version }}-amd64 # - modelzai/ingress-operator:v{{ .Version }}-arm64v8 ================================================ FILE: LICENSE ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: MANIFEST.in ================================================ prune autoscaler prune ingress-operator prune modelzetes prune .github include LICENSE include README.md include .goreleaser.yaml include mdz/Makefile mdz/go.mod mdz/go.sum mdz/LICENSE graft mdz/pkg graft mdz/cmd prune mdz/bin prune mdz/docs prune mdz/examples graft agent/pkg prune agent/bin ================================================ FILE: README.md ================================================
# OpenModelZ

discord invitation link trackgit-views docs all-contributors CI PyPI version Coverage Status

## What is OpenModelZ? OpenModelZ ( `mdz` ) is tool to deploy your models to any cluster (GCP, AWS, Lambda labs, your home lab, or even a single machine). Getting models into production is hard for data scientists and SREs. You need to configure the monitoring, logging, and scaling infrastructure, with the right security and permissions. And then setup the domain, SSL, and load balancer. This can take weeks or months of work even for a single model deployment. You can now use mdz deploy to effortlessly deploy your models. OpenModelZ handles all the infrastructure setup for you. Each deployment gets a public subdomain, like `http://jupyter-9pnxd.2.242.22.143.modelz.live`, making it easily accessible.

OpenModelZ

## Benefits OpenModelZ provides the following features out-of-the-box: - 📈 **Auto-scaling from 0**: The number of inference servers could be scaled based on the workload. You could start from 0 and scale it up to 10+ replicas easily. - 📦 **Support any machine learning framework**: You could deploy any machine learning framework (e.g. [vLLM](https://github.com/vllm-project/vllm)/[triton-inference-server](https://github.com/triton-inference-server/server)/[mosec](https://github.com/mosecorg/mosec) etc.) with a single command. Besides, you could also deploy your own custom inference server. - 🔬 **Gradio/Streamlit/Jupyter support**: We provide a robust prototyping environment with support for [Gradio](https://gradio.app), [Streamlit](https://streamlit.io/), [jupyter](https://jupyter.org/) and so on. You could visualize your model's performance and debug it easily in the notebook, or deploy a web app for your model with a single command. - 🏃 **Start from a single machine to a cluster of machines**: You could start from a single machine and scale it up to a cluster of machines without any hassle, with a single command `mdz server start`. - 🚀 **Public accessible subdomain for each deployment** ( optional ) : We provision a separate subdomain for each deployment without any extra cost and effort, making each deployment easily accessible from the outside. OpenModelZ is the foundational component of the ModelZ platform available at [modelz.ai](https://modelz.ai). ## How it works Get a server (could be a cloud VM, a home lab, or even a single machine) and run the `mdz server start` command. OpenModelZ will bootstrap the server for you. ```text $ mdz server start 🚧 Creating the server... 🚧 Initializing the load balancer... 🚧 Initializing the GPU resource... 🚧 Initializing the server... 🚧 Waiting for the server to be ready... 🐋 Checking if the server is running... 🐳 The server is running at http://146.235.213.84.modelz.live 🎉 You could set the environment variable to get started! export MDZ_URL=http://146.235.213.84.modelz.live $ export MDZ_URL=http://146.235.213.84.modelz.live ``` Then you could deploy your model with a single command `mdz deploy` and get the endpoint: ``` $ mdz deploy --image modelzai/gradio-stable-diffusion:23.03 --name sdw --port 7860 --gpu 1 Inference sd is created $ mdz list NAME ENDPOINT STATUS INVOCATIONS REPLICAS sdw http://sdw-qh2n0y28ybqc36oc.146.235.213.84.modelz.live Ready 174 1/1 http://146.235.213.84.modelz.live/inference/sdw.default ``` ## Quick Start 🚀 ### Install `mdz` You can install OpenModelZ using the following command: ```text copy pip install openmodelz ``` You could verify the installation by running the following command: ```text copy mdz ``` Once you've installed the `mdz` you can start deploying models and experimenting with them. ### Bootstrap `mdz` It's super easy to bootstrap the `mdz` server. You just need to find a server (could be a cloud VM, a home lab, or even a single machine) and run the `mdz server start` command. > Notice: We may require the root permission to bootstrap the `mdz` server on port 80. ``` $ mdz server start 🚧 Creating the server... 🚧 Initializing the load balancer... 🚧 Initializing the GPU resource... 🚧 Initializing the server... 🚧 Waiting for the server to be ready... 🐋 Checking if the server is running... Agent: Version: v0.0.13 Build Date: 2023-07-19T09:12:55Z Git Commit: 84d0171640453e9272f78a63e621392e93ef6bbb Git State: clean Go Version: go1.19.10 Compiler: gc Platform: linux/amd64 🐳 The server is running at http://192.168.71.93.modelz.live 🎉 You could set the environment variable to get started! export MDZ_URL=http://192.168.71.93.modelz.live ``` The internal IP address will be used as the default endpoint of your deployments. You could provide the public IP address of your server to the `mdz server start` command to make it accessible from the outside world. ```bash # Provide the public IP as an argument $ mdz server start 1.2.3.4 ``` You could also specify the registry mirror to speed up the image pulling process. Here is an example: ```bash /--mirror-endpoints/ $ mdz server start --mirror-endpoints https://docker.mirrors.sjtug.sjtu.edu.cn ``` ### Create your first UI-based deployment Once you've bootstrapped the `mdz` server, you can start deploying your first applications. We will use jupyter notebook as an example in this tutorial. You could use any docker image as your deployment. ```text $ mdz deploy --image jupyter/minimal-notebook:lab-4.0.3 --name jupyter --port 8888 --command "jupyter notebook --ip='*' --NotebookApp.token='' --NotebookApp.password=''" Inference jupyter is created $ mdz list NAME ENDPOINT STATUS INVOCATIONS REPLICAS jupyter http://jupyter-9pnxdkeb6jsfqkmq.192.168.71.93.modelz.live Ready 488 1/1 http://192.168.71.93/inference/jupyter.default ``` You could access the deployment by visiting the endpoint URL. The endpoint will be automatically generated for each deployment with the following format: `-..modelz.live`. It is `http://jupyter-9pnxdkeb6jsfqkmq.192.168.71.93.modelz.live` in this case. The endpoint could be accessed from the outside world as well if you've provided the public IP address of your server to the `mdz server start` command. ![jupyter notebook](./images/jupyter.png) ### Create your first OpenAI compatible API server You could also create API-based deployments. We will use [OpenAI compatible API server with Bloomz 560M](https://github.com/tensorchord/modelz-llm#run-the-self-hosted-api-server) as an example in this tutorial. ```text $ mdz deploy --image modelzai/llm-bloomz-560m:23.07.4 --name simple-server Inference simple-server is created $ mdz list NAME ENDPOINT STATUS INVOCATIONS REPLICAS jupyter http://jupyter-9pnxdkeb6jsfqkmq.192.168.71.93.modelz.live Ready 488 1/1 http://192.168.71.93/inference/jupyter.default simple-server http://simple-server-lagn8m9m8648q6kx.192.168.71.93.modelz.live Ready 0 1/1 http://192.168.71.93/inference/simple-server.default ``` You could use OpenAI python package and the endpoint `http://simple-server-lagn8m9m8648q6kx.192.168.71.93.modelz.live` in this case, to interact with the deployment. ```python import openai openai.api_base="http://simple-server-lagn8m9m8648q6kx.192.168.71.93.modelz.live" openai.api_key="any" # create a chat completion chat_completion = openai.ChatCompletion.create(model="bloomz", messages=[ {"role": "user", "content": "Who are you?"}, {"role": "assistant", "content": "I am a student"}, {"role": "user", "content": "What do you learn?"}, ], max_tokens=100) ``` ### Scale your deployment You could scale your deployment by using the `mdz scale` command. ```text /scale/ $ mdz scale simple-server --replicas 3 ``` The requests will be load balanced between the replicas of your deployment. You could also tell the `mdz` to **autoscale your deployment** based on the inflight requests. Please check out the [Autoscaling](https://docs.open.modelz.ai/deployment/autoscale) documentation for more details. ### Debug your deployment Sometimes you may want to debug your deployment. You could use the `mdz logs` command to get the logs of your deployment. ```text /logs/ $ mdz logs simple-server simple-server-6756dd67ff-4bf4g: 10.42.0.1 - - [27/Jul/2023 02:32:16] "GET / HTTP/1.1" 200 - simple-server-6756dd67ff-4bf4g: 10.42.0.1 - - [27/Jul/2023 02:32:16] "GET / HTTP/1.1" 200 - simple-server-6756dd67ff-4bf4g: 10.42.0.1 - - [27/Jul/2023 02:32:17] "GET / HTTP/1.1" 200 - ``` You could also use the `mdz exec` command to execute a command in the container of your deployment. You do not need to ssh into the server to do that. ```text /exec/ $ mdz exec simple-server ps PID USER TIME COMMAND 1 root 0:00 /usr/bin/dumb-init /bin/sh -c python3 -m http.server 80 7 root 0:00 /bin/sh -c python3 -m http.server 80 8 root 0:00 python3 -m http.server 80 9 root 0:00 ps ``` ```text /exec/ $ mdz exec simple-server -ti bash bash-4.4# ``` Or you could port-forward the deployment to your local machine and debug it locally. ```text /port-forward/ $ mdz port-forward simple-server 7860 Forwarding inference simple-server to local port 7860 ``` ### Add more servers You could add more servers to your cluster by using the `mdz server join` command. The `mdz` server will be bootstrapped on the server and join the cluster automatically. ```text /join/ $ mdz server join $ mdz server list NAME PHASE ALLOCATABLE CAPACITY node1 Ready cpu: 16 cpu: 16 mem: 32784748Ki mem: 32784748Ki gpu: 1 gpu: 1 node2 Ready cpu: 16 cpu: 16 mem: 32784748Ki mem: 32784748Ki gpu: 1 gpu: 1 ``` ### Label your servers You could label your servers to deploy your models to specific servers. For example, you could label your servers with `gpu=true` and deploy your models to servers with GPUs. ```text /--node-labels gpu=true,type=nvidia-a100/ $ mdz server label node3 gpu=true type=nvidia-a100 $ mdz deploy ... --node-labels gpu=true,type=nvidia-a100 ``` ## Architecture OpenModelZ is inspired by the [k3s](https://github.com/k3s-io/k3s) and [OpenFaaS](https://github.com/openfaas), but designed specifically for machine learning deployment. We keep the core of the system **simple, and easy to extend**. You do not need to read this section if you just want to deploy your models. But if you want to understand how OpenModelZ works, this section is for you.

OpenModelZ

OpenModelZ is composed of two components: - Data Plane: The data plane is responsible for the servers. You could use `mdz server` to manage the servers. The data plane is designed to be **stateless** and **scalable**. You could easily scale the data plane by adding more servers to the cluster. It uses k3s under the hood, to support VMs, bare-metal, and IoT devices (in the future). You could also deploy OpenModelZ on a existing kubernetes cluster. - Control Plane: The control plane is responsible for the deployments. It manages the deployments and the underlying resources. A request will be routed to the inference servers by the load balancer. And the autoscaler will scale the number of inference servers based on the workload. We provide a domain `*.modelz.live` by default, with the help of a [wildcard DNS server](https://github.com/cunnie/sslip.io) to support the public accessible subdomain for each deployment. You could also use your own domain. You could check out the [architecture](https://docs.open.modelz.ai/architecture) documentation for more details. ## Roadmap 🗂️ Please checkout [ROADMAP](https://docs.open.modelz.ai/community). ## Contribute 😊 We welcome all kinds of contributions from the open-source community, individuals, and partners. - Join our [discord community](https://discord.gg/KqswhpVgdU)! ## Contributors ✨
Ce Gao
Ce Gao
💻 👀 ✅		 Jinjing Zhou
Jinjing Zhou
💬 🐛 🤔		 Keming
Keming
💻 🎨 🚇		 Nadeshiko Manju
Nadeshiko Manju
🐛 🎨 🤔		 Teddy Xinyuan Chen
Teddy Xinyuan Chen
📖		 Wei Zhang
Wei Zhang
💻		 Xuanwo
Xuanwo
🖋 🎨 🤔
cutecutecat
cutecutecat
🤔		 xieydd
xieydd
🤔
## Acknowledgements 🙏 - [K3s](https://github.com/k3s-io/k3s) for the single control-plane binary and process. - [OpenFaaS](https://github.com/openfaas) for their work on serverless function services. It laid the foundation for OpenModelZ. - [sslip.io](https://github.com/cunnie/sslip.io) for the wildcard DNS service. It makes it possible to access the server from the outside world without any setup. ================================================ FILE: agent/.gitignore ================================================ # If you prefer the allow list template instead of the deny list, see community template: # https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore # # Binaries for programs and plugins *.exe *.exe~ *.dll *.so *.dylib # Test binary, built with `go test -c` *.test # Output of the go coverage tool, specifically when used with LiteIDE *.out *.report # Dependency directories (remove the comment below to include it) vendor/ # Go workspace file go.work .vscode/* .idea # Local History for Visual Studio Code .history/ # Built Visual Studio Code Extensions *.vsix __debug_bin bin/ debug-bin/ /build.envd .ipynb_checkpoints/ cover.html cmd/test/ # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] *$py.class # Distribution / packaging .Python build/ develop-eggs/ dist/ downloads/ eggs/ .eggs/ lib/ lib64/ parts/ sdist/ var/ wheels/ wheelhouse/ pip-wheel-metadata/ share/python-wheels/ *.egg-info/ .installed.cfg *.egg MANIFEST .demo/ pkg/docs/swagger.* ================================================ FILE: agent/Dockerfile ================================================ FROM ubuntu:22.04 LABEL maintainer="modelz-support@tensorchord.ai" RUN apt-get -qq update \ && apt-get -qq install -y --no-install-recommends ca-certificates curl COPY agent /usr/bin/agent ENTRYPOINT ["/usr/bin/agent"] ================================================ FILE: agent/Makefile ================================================ # Copyright 2022 TensorChord Inc. # # The old school Makefile, following are required targets. The Makefile is written # to allow building multiple binaries. You are free to add more targets or change # existing implementations, as long as the semantics are preserved. # # make - default to 'build' target # make lint - code analysis # make test - run unit test (or plus integration test) # make build - alias to build-local target # make build-local - build local binary targets # make build-linux - build linux binary targets # make container - build containers # $ docker login registry -u username -p xxxxx # make push - push containers # make clean - clean up targets # # Not included but recommended targets: # make e2e-test # # The makefile is also responsible to populate project version information. # # # Tweak the variables based on your project. # # This repo's root import path (under GOPATH). ROOT := github.com/tensorchord/openmodelz/agent # Target binaries. You can build multiple binaries for a single project. TARGETS := agent # Container image prefix and suffix added to targets. # The final built images are: # $[REGISTRY]/$[IMAGE_PREFIX]$[TARGET]$[IMAGE_SUFFIX]:$[VERSION] # $[REGISTRY] is an item from $[REGISTRIES], $[TARGET] is an item from $[TARGETS]. IMAGE_PREFIX ?= $(strip ) IMAGE_SUFFIX ?= $(strip ) # Container registries. REGISTRY ?= ghcr.io/tensorchord # Container registry for base images. BASE_REGISTRY ?= docker.io BASE_REGISTRY_USER ?= modelzai # Disable CGO by default. CGO_ENABLED ?= 0 # # These variables should not need tweaking. # # It's necessary to set this because some environments don't link sh -> bash. export SHELL := bash # It's necessary to set the errexit flags for the bash shell. export SHELLOPTS := errexit PACKAGE_NAME := github.com/tensorchord/openmodelz/agent GOLANG_CROSS_VERSION ?= v1.17.6 # Project main package location (can be multiple ones). CMD_DIR := ./cmd # Project output directory. OUTPUT_DIR := ./bin DEBUG_DIR := ./debug-bin # Build directory. BUILD_DIR := ./build # Current version of the project. VERSION ?= $(shell git describe --match 'v[0-9]*' --always --tags --abbrev=0) BUILD_DATE=$(shell date -u +'%Y-%m-%dT%H:%M:%SZ') GIT_COMMIT=$(shell git rev-parse HEAD) GIT_TAG=$(shell if [ -z "`git status --porcelain`" ]; then git describe --exact-match --tags HEAD 2>/dev/null; fi) GIT_TREE_STATE=$(shell if [ -z "`git status --porcelain`" ]; then echo "clean" ; else echo "dirty"; fi) GITSHA ?= $(shell git rev-parse --short HEAD) # Track code version with Docker Label. DOCKER_LABELS ?= git-describe="$(shell date -u +v%Y%m%d)-$(shell git describe --tags --always --dirty)" # Golang standard bin directory. GOPATH ?= $(shell go env GOPATH) GOROOT ?= $(shell go env GOROOT) BIN_DIR := $(GOPATH)/bin GOLANGCI_LINT := $(BIN_DIR)/golangci-lint # check if we need embed the dashboard DASHBOARD_BUILD ?= debug # Default golang flags used in build and test # -mod=vendor: force go to use the vendor files instead of using the `$GOPATH/pkg/mod` # -p: the number of programs that can be run in parallel # -count: run each test and benchmark 1 times. Set this flag to disable test cache export GOFLAGS ?= -count=1 # # Define all targets. At least the following commands are required: # # All targets. .PHONY: help lint test build container push addlicense debug debug-local build-local generate clean test-local addlicense-install release build-image .DEFAULT_GOAL:=build build: build-local ## Build the release version help: ## Display this help @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z0-9_-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) debug: debug-local ## Build the debug version # more info about `GOGC` env: https://github.com/golangci/golangci-lint#memory-usage-of-golangci-lint lint: $(GOLANGCI_LINT) ## Lint GO code @$(GOLANGCI_LINT) run $(GOLANGCI_LINT): curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $$(go env GOPATH)/bin mockgen-install: go install github.com/golang/mock/mockgen@v1.6.0 addlicense-install: go install github.com/google/addlicense@latest # https://github.com/swaggo/swag/pull/1322, we should use master instead of latest for now. swag-install: go install github.com/swaggo/swag/cmd/swag@v1.8.7 build-local: @for target in $(TARGETS); do \ CGO_ENABLED=$(CGO_ENABLED) go build -tags $(DASHBOARD_BUILD) -trimpath -v -o $(OUTPUT_DIR)/$${target} \ -ldflags "-s -w -X $(ROOT)/pkg/version.version=$(VERSION) -X $(ROOT)/pkg/version.buildDate=$(BUILD_DATE) -X $(ROOT)/pkg/version.gitCommit=$(GIT_COMMIT) -X $(ROOT)/pkg/version.gitTreeState=$(GIT_TREE_STATE)" \ $(CMD_DIR)/$${target}; \ done # It is used by vscode to attach into the process. debug-local: @for target in $(TARGETS); do \ CGO_ENABLED=$(CGO_ENABLED) go build -tags $(DASHBOARD_BUILD) -trimpath \ -v -o $(DEBUG_DIR)/$${target} \ -gcflags='all=-N -l' \ $(CMD_DIR)/$${target}; \ done addlicense: addlicense-install ## Add license to GO code files addlicense -l mpl -c "TensorChord Inc." $$(find . -type f -name '*.go' | grep -v pkg/docs/docs.go) test-local: @go test -tags=$(DASHBOARD_BUILD) -v -race -coverprofile=coverage.out ./... test: ## Run the tests @go test -tags=$(DASHBOARD_BUILD) -race -coverprofile=coverage.out ./... @go tool cover -func coverage.out | tail -n 1 | awk '{ print "Total coverage: " $$3 }' clean: ## Clean the outputs and artifacts @-rm -vrf ${OUTPUT_DIR} @-rm -vrf ${DEBUG_DIR} @-rm -vrf build dist .eggs *.egg-info fmt: swag-install ## Run go fmt against code. go fmt ./... swag fmt vet: ## Run go vet against code. go vet ./... swag: swag-install swag init -g ./cmd/agent/main.go --parseDependency --output ./pkg/docs build-image: build-local docker build -t ${BASE_REGISTRY}/${BASE_REGISTRY_USER}/openmodelz-agent:dev -f Dockerfile ./bin docker push ${BASE_REGISTRY}/${BASE_REGISTRY_USER}/openmodelz-agent:dev release: @if [ ! -f ".release-env" ]; then \ echo "\033[91m.release-env is required for release\033[0m";\ exit 1;\ fi docker run \ --rm \ --privileged \ -e CGO_ENABLED=1 \ --env-file .release-env \ -v /var/run/docker.sock:/var/run/docker.sock \ -v `pwd`:/go/src/$(PACKAGE_NAME) \ -v `pwd`/sysroot:/sysroot \ -w /go/src/$(PACKAGE_NAME) \ goreleaser/goreleaser-cross:${GOLANG_CROSS_VERSION} \ release --rm-dist generate: mockgen-install swag @mockgen -source pkg/runtime/runtime.go -destination pkg/runtime/mock/mock.go -package mock ================================================ FILE: agent/README.md ================================================
# OpenModelZ Agent

discord invitation link trackgit-views

## Installation ``` pip install openmodelz ``` ## Architecture Please check out [Architecture](https://docs.open.modelz.ai/architecture) documentation. ================================================ FILE: agent/api/types/build.go ================================================ package types type Build struct { Spec BuildSpec `json:"spec"` Status BuildStatus `json:"status,omitempty"` } type BuildSpec struct { Name string `json:"name,omitempty"` Namespace string `json:"namespace,omitempty"` GitRepositorySource `json:",inline,omitempty"` DockerSource `json:",inline,omitempty"` BuildTarget BuildTarget `json:",inline,omitempty"` } type DockerSource struct { ArtifactImage string `json:"image,omitempty"` ArtifactImageTag string `json:"image_tag,omitempty"` AuthN AuthN `json:"authn,omitempty"` SecretID string `json:"secret_id,omitempty"` } type BuildTarget struct { // directory is the target directory name. // Must not contain or start with '..'. If '.' is supplied, the volume directory will be the // git repository. Otherwise, if specified, the volume will contain the git repository in // the subdirectory with the given name. // +optional Directory string `json:"directory,omitempty"` Builder BuilderType `json:"builder,omitempty"` ArtifactImage string `json:"image,omitempty"` ArtifactImageTag string `json:"image_tag,omitempty"` Digest string `json:"digest,omitempty"` Duration string `json:"duration,omitempty"` Registry string `json:"registry,omitempty"` RegistryToken string `json:"registry_token,omitempty"` } type AuthN struct { Username string `json:"username,omitempty"` Password string `json:"password,omitempty"` Token string `json:"token,omitempty"` } type BuildStatus struct { Phase BuildPhase `json:"phase,omitempty"` } type BuildPhase string const ( BuildPhasePending BuildPhase = "Pending" BuildPhaseRunning BuildPhase = "Running" BuildPhaseSucceeded BuildPhase = "Succeeded" BuildPhaseFailed BuildPhase = "Failed" ) type BuilderType string const ( BuilderTypeDockerfile BuilderType = "Dockerfile" BuilderTypeENVD BuilderType = "envd" BuilderTypeImage BuilderType = "image" ) type GitRepositorySource struct { // repository is the URL Repository string `json:"repository"` Branch string `json:"branch,omitempty"` // revision is the commit hash for the specified revision. // +optional Revision string `json:"revision,omitempty"` } ================================================ FILE: agent/api/types/error.go ================================================ package types type ErrorResponse struct { Message string `json:"message"` } ================================================ FILE: agent/api/types/event.go ================================================ package types import "time" const ( DeploymentCreateEvent = "deployment-create" DeploymentUpdateEvent = "deployment-update" DeploymentDeleteEvent = "deployment-delete" DeploymentScaleUpEvent = "deployment-scale-up" DeploymentScaleDownEvent = "deployment-scale-down" DeploymentScaleBlockEvent = "deployment-scale-block" PodCreateEvent = "pod-create" PodReadyEvent = "pod-ready" PodTimeoutEvent = "pod-timeout" ) type DeploymentEvent struct { ID string `json:"id"` CreatedAt time.Time `json:"created_at"` UserID string `json:"user_id"` DeploymentID string `json:"deployment_id"` EventType string `json:"event_type"` Message string `json:"message"` } ================================================ FILE: agent/api/types/inference_deployment.go ================================================ package types // InferenceDeployment represents a request to create or update a Model. type InferenceDeployment struct { Spec InferenceDeploymentSpec `json:"spec"` Status InferenceDeploymentStatus `json:"status,omitempty"` } type InferenceDeploymentSpec struct { // Name is the name of the inference. Name string `json:"name"` // Namespace for the inference. Namespace string `json:"namespace,omitempty"` // Scaling is the scaling configuration for the inference. Scaling *ScalingConfig `json:"scaling,omitempty"` // Framework is the inference framework. Framework Framework `json:"framework,omitempty"` // Image is a fully-qualified container image Image string `json:"image"` // Port is the port exposed by the inference. Port *int32 `json:"port,omitempty"` // HTTPProbePath is the path of the http probe. HTTPProbePath *string `json:"http_probe_path,omitempty"` // Command to run when starting the Command *string `json:"command,omitempty"` // EnvVars can be provided to set environment variables for the inference runtime. EnvVars map[string]string `json:"envVars,omitempty"` // Constraints are the constraints for the inference. Constraints []string `json:"constraints,omitempty"` // Secrets list of secrets to be made available to inference. Secrets []string `json:"secrets,omitempty"` // Labels are key-value pairs that may be attached to the inference. Labels map[string]string `json:"labels,omitempty"` // Annotations are key-value pairs that may be attached to the inference. Annotations map[string]string `json:"annotations,omitempty"` // Resources are the compute resource requirements. Resources *ResourceRequirements `json:"resources,omitempty"` } // Framework is the inference framework. It is only used to set the default port // and command. For example, if the framework is "gradio", the default port is // 7860 and the default command is "python app.py". You could override these // defaults by setting the port and command fields and framework to `other`. type Framework string const ( FrameworkGradio Framework = "gradio" FrameworkStreamlit Framework = "streamlit" FrameworkMosec Framework = "mosec" FrameworkOther Framework = "other" ) type ScalingConfig struct { // MinReplicas is the lower limit for the number of replicas to which the // autoscaler can scale down. It defaults to 0. MinReplicas *int32 `json:"min_replicas,omitempty"` // MaxReplicas is the upper limit for the number of replicas to which the // autoscaler can scale up. It cannot be less that minReplicas. It defaults // to 1. MaxReplicas *int32 `json:"max_replicas,omitempty"` // TargetLoad is the target load. In capacity mode, it is the expected number of the inflight requests per replica. TargetLoad *int32 `json:"target_load,omitempty"` // Type is the scaling type. It can be either "capacity" or "rps". Default is "capacity". Type *ScalingType `json:"type,omitempty"` // ZeroDuration is the duration (in seconds) of zero load before scaling down to zero. Default is 5 minutes. ZeroDuration *int32 `json:"zero_duration,omitempty"` // StartupDuration is the duration (in seconds) of startup time. StartupDuration *int32 `json:"startup_duration,omitempty"` } type ScalingType string const ( ScalingTypeCapacity ScalingType = "capacity" ScalingTypeRPS ScalingType = "rps" ) // ResourceRequirements describes the compute resource requirements. type ResourceRequirements struct { // Limits describes the maximum amount of compute resources allowed. // More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ // +optional Limits ResourceList `json:"limits,omitempty" protobuf:"bytes,1,rep,name=limits,casttype=ResourceList,castkey=ResourceName"` // Requests describes the minimum amount of compute resources required. // If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, // otherwise to an implementation-defined value. // More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ // +optional Requests ResourceList `json:"requests,omitempty" protobuf:"bytes,2,rep,name=requests,casttype=ResourceList,castkey=ResourceName"` } // ResourceList is a set of (resource name, quantity) pairs. type ResourceList map[ResourceName]Quantity type ResourceName string const ( ResourceCPU ResourceName = "cpu" ResourceMemory ResourceName = "memory" ResourceGPU ResourceName = "gpu" ) type Quantity string const ( RuntimeClassNvidia string = "nvidia" ) type ImageCache struct { // Name is the name of the inference. Name string `json:"name"` Namespace string `json:"namespace"` Image string `json:"image"` ForceFullCache bool `json:"force_full_cache"` NodeSelector string `json:"node_selector"` } ================================================ FILE: agent/api/types/inference_deployment_instance.go ================================================ package types import "time" type InferenceDeploymentInstance struct { Spec InferenceDeploymentInstanceSpec `json:"spec,omitempty"` Status InferenceDeploymentInstanceStatus `json:"status,omitempty"` } type InferenceDeploymentInstanceSpec struct { Namespace string `json:"namespace,omitempty"` Name string `json:"name,omitempty"` OwnerReference string `json:"owner_reference,omitempty"` } type InferenceDeploymentInstanceStatus struct { Phase InstancePhase `json:"phase,omitempty"` StartTime time.Time `json:"createdAt,omitempty"` Reason string `json:"reason,omitempty"` Message string `json:"message,omitempty"` } type InstancePhase string const ( InstancePhaseScheduling InstancePhase = "Scheduling" InstancePhasePending InstancePhase = "Pending" InstancePhaseRunning InstancePhase = "Running" InstancePhaseFailed InstancePhase = "Failed" InstancePhaseSucceeded InstancePhase = "Succeeded" InstancePhaseUnknown InstancePhase = "Unknown" InstancePhaseCreating InstancePhase = "Creating" InstancePhaseInitializing InstancePhase = "Initializing" ) ================================================ FILE: agent/api/types/inference_status.go ================================================ package types import "time" // InferenceDeploymentStatus exported for system/inferences endpoint type InferenceDeploymentStatus struct { Phase Phase `json:"phase,omitempty"` // InvocationCount count of invocations InvocationCount int32 `json:"invocationCount,omitempty"` // Replicas desired within the cluster Replicas int32 `json:"replicas,omitempty"` // AvailableReplicas is the count of replicas ready to receive // invocations as reported by the faas-provider AvailableReplicas int32 `json:"availableReplicas,omitempty"` // CreatedAt is the time read back from the faas backend's // data store for when the function or its container was created. CreatedAt *time.Time `json:"createdAt,omitempty"` // Usage represents CPU and RAM used by all of the // functions' replicas. Divide by AvailableReplicas for an // average value per replica. Usage *InferenceUsage `json:"usage,omitempty"` // EventMessage record human readable message indicating details about the event of deployment. EventMessage string `json:"eventMessage,omitempty"` } type Phase string const ( // PhaseReady is the state of an inference when it is ready to // receive invocations. PhaseReady Phase = "Ready" // PhaseScaling is the state of an inference when scales. PhaseScaling Phase = "Scaling" PhaseTerminating Phase = "Terminating" PhaseNoReplicas Phase = "NoReplicas" PhaseNotReady Phase = "NotReady" PhaseBuilding Phase = "Building" PhaseOptimizing Phase = "Optimizing" ) // InferenceUsage represents CPU and RAM used by all of the // functions' replicas. // // CPU is measured in seconds consumed since the last measurement // RAM is measured in total bytes consumed type InferenceUsage struct { // CPU is the increase in CPU usage since the last measurement // equivalent to Kubernetes' concept of millicores. CPU float64 `json:"cpu,omitempty"` //TotalMemoryBytes is the total memory usage in bytes. TotalMemoryBytes float64 `json:"totalMemoryBytes,omitempty"` GPU float64 `json:"gpu,omitempty"` } ================================================ FILE: agent/api/types/info.go ================================================ package types // ProviderInfo provides information about the configured provider type ProviderInfo struct { Name string `json:"provider"` Version *VersionInfo `json:"version"` Orchestration string `json:"orchestration"` } // VersionInfo provides the commit message, sha and release version number type VersionInfo struct { Version string `json:"version,omitempty"` BuildDate string `json:"build_date,omitempty"` GitCommit string `json:"git_commit,omitempty"` GitTag string `json:"git_tag,omitempty"` GitTreeState string `json:"git_tree_state,omitempty"` GoVersion string `json:"go_version,omitempty"` Compiler string `json:"compiler,omitempty"` Platform string `json:"platform,omitempty"` } ================================================ FILE: agent/api/types/log.go ================================================ package types import "time" type LogRequest struct { Namespace string `form:"namespace" json:"namespace,omitempty"` Name string `form:"name" json:"name,omitempty"` // Instance is the optional pod name, that allows you to request logs from a specific instance Instance string `form:"instance" json:"instance,omitempty"` // Follow is allows the user to request a stream of logs until the timeout Follow bool `form:"follow" json:"follow,omitempty"` // Tail sets the maximum number of log messages to return, <=0 means unlimited Tail int `form:"tail" json:"tail,omitempty"` Since string `form:"since" json:"since,omitempty"` // End is the end time of the log stream End string `form:"end" json:"end,omitempty"` } // Message is a specific log message from a function container log stream type Message struct { // Name is the function name Name string `json:"name"` Namespace string `json:"namespace"` // instance is the name/id of the specific function instance Instance string `json:"instance"` // Timestamp is the timestamp of when the log message was recorded Timestamp time.Time `json:"timestamp"` // Text is the raw log message content Text string `json:"text"` } ================================================ FILE: agent/api/types/modelz_cloud.go ================================================ package types import "time" const ( ClusterStatusInit = "init" ClusterStatusActive = "active" ClusterStatusUnknown = "unknown" ) const ( DailEndPointSuffix = "/api/v1/clusteragent/connect" ) type AgentToken struct { UID string `json:"uid,omitempty"` Token string `json:"token,omitempty"` ClusterName string `json:"cluster_name,omitempty"` } type ManagedCluster struct { Name string `json:"name,omitempty"` ID string `json:"id,omitempty"` TokenID string `json:"token_id,omitempty"` Version string `json:"version,omitempty"` KubernetesVersion string `json:"kubernetes_version,omitempty"` Platform string `json:"platform,omitempty"` Status string `json:"status,omitempty"` CreatedAt time.Time `json:"created_at,omitempty"` UpdatedAt time.Time `json:"updated_at,omitempty"` Region string `json:"region,omitempty"` ServerResources string `json:"server_resources,omitempty"` PrometheusURL string `json:"prometheus_url,omitempty"` } type APIKeyMap map[string]string type NamespaceList struct { Items []string `json:"items,omitempty"` } ================================================ FILE: agent/api/types/namespace.go ================================================ package types type NamespaceRequest struct { Name string `json:"name,omitempty"` } ================================================ FILE: agent/api/types/queue.go ================================================ package types import ( "net/http" "net/url" ) // Request for asynchronous processing type QueueRequest struct { // Header from HTTP request Header http.Header // Host from HTTP request Host string // Body from HTTP request to use for invocation Body []byte // Method from HTTP request Method string // Path from HTTP request Path string // QueryString from HTTP request QueryString string // Function name to invoke Function string // QueueName to publish the request to, leave blank // for default. QueueName string // Used by queue worker to submit a result CallbackURL *url.URL `json:"CallbackUrl"` } // RequestQueuer can public a request to be executed asynchronously type RequestQueuer interface { Queue(req *QueueRequest) error } ================================================ FILE: agent/api/types/requests.go ================================================ // Copyright (c) Alex Ellis 2017. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. package types // ScaleServiceRequest scales the service to the requested replcia count. type ScaleServiceRequest struct { ServiceName string `json:"serviceName"` Replicas uint64 `json:"replicas"` EventMessage string `json:"eventMessage"` Attempt int `json:"attempt"` } // DeleteFunctionRequest delete a deployed function type DeleteFunctionRequest struct { FunctionName string `json:"functionName"` } ================================================ FILE: agent/api/types/secret.go ================================================ package types // Secret for underlying orchestrator type Secret struct { // Name of the secret Name string `json:"name"` // Namespace if applicable for the secret Namespace string `json:"namespace,omitempty"` // Value is a string representing the string's value Value string `json:"value,omitempty"` // RawValue can be used to provide binary data when // Value is not set RawValue []byte `json:"rawValue,omitempty"` } ================================================ FILE: agent/api/types/server.go ================================================ package types type Server struct { Spec ServerSpec `json:"spec,omitempty"` Status ServerStatus `json:"status,omitempty"` } type ServerSpec struct { Name string `json:"name,omitempty"` Labels map[string]string `json:"labels,omitempty"` } type ServerStatus struct { Allocatable ResourceList `json:"allocatable,omitempty"` Capacity ResourceList `json:"capacity,omitempty"` Phase string `json:"phase,omitempty"` System NodeSystemInfo `json:"system,omitempty"` } // NodeSystemInfo is a set of ids/uuids to uniquely identify the node. type NodeSystemInfo struct { // MachineID reported by the node. For unique machine identification // in the cluster this field is preferred. Learn more from man(5) // machine-id: http://man7.org/linux/man-pages/man5/machine-id.5.html MachineID string `json:"machineID" protobuf:"bytes,1,opt,name=machineID"` // Kernel Version reported by the node from 'uname -r' (e.g. 3.16.0-0.bpo.4-amd64). KernelVersion string `json:"kernelVersion" protobuf:"bytes,4,opt,name=kernelVersion"` // OS Image reported by the node from /etc/os-release (e.g. Debian GNU/Linux 7 (wheezy)). OSImage string `json:"osImage" protobuf:"bytes,5,opt,name=osImage"` // The Operating System reported by the node OperatingSystem string `json:"operatingSystem" protobuf:"bytes,9,opt,name=operatingSystem"` // The Architecture reported by the node Architecture string `json:"architecture" protobuf:"bytes,10,opt,name=architecture"` // The Resource Type reported by the node ResourceType string `json:"resourceType" protobuf:"bytes,11,opt,name=resourceType"` } ================================================ FILE: agent/client/build.go ================================================ package client import ( "context" "encoding/json" "fmt" "net/url" "github.com/sirupsen/logrus" "github.com/tensorchord/openmodelz/agent/api/types" ) func (cli *Client) BuildCreate(ctx context.Context, namespace string, build types.Build) error { build.Spec.Namespace = namespace logrus.Debugf("create new build: %s", build) val := url.Values{} resp, err := cli.post(ctx, gatewayBuildControlPlanePath, val, build, nil) defer ensureReaderClosed(resp) if err != nil { return wrapResponseError(err, resp, "build", build.Spec.Name) } return nil } func (cli *Client) BuildGet(ctx context.Context, namespace, name string) (types.Build, error) { val := url.Values{} val.Add("namespace", namespace) build := types.Build{} resp, err := cli.get( ctx, fmt.Sprintf(gatewayBuildInstanceControlPlanePath, name), val, nil) defer ensureReaderClosed(resp) if err != nil { logrus.Infof("failed to query build.get: %s", err) return build, wrapResponseError(err, resp, "build", name) } err = json.NewDecoder(resp.body).Decode(&build) if err != nil { logrus.Infof("failed to decode build: %s", err) return build, wrapResponseError(err, resp, "build", name) } return build, nil } func (cli *Client) BuildList(ctx context.Context, namespace string) ([]types.Build, error) { val := url.Values{} val.Add("namespace", namespace) resp, err := cli.get(ctx, gatewayBuildControlPlanePath, val, nil) defer ensureReaderClosed(resp) if err != nil { logrus.Infof("failed to query build.list: %s", err) return nil, wrapResponseError(err, resp, "build", namespace) } var builds []types.Build err = json.NewDecoder(resp.body).Decode(&builds) if err != nil { logrus.Infof("failed to decode builds: %s", err) return nil, wrapResponseError(err, resp, "build", namespace) } return builds, nil } ================================================ FILE: agent/client/client.go ================================================ // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. package client import ( "context" "net/http" "net/url" "path" "strings" "github.com/cockroachdb/errors" "github.com/docker/go-connections/sockets" ) // Refer to github.com/docker/docker/client // ErrRedirect is the error returned by checkRedirect when the request is non-GET. var ErrRedirect = errors.New("unexpected redirect in response") // Client is the API client that performs all operations // against a docker server. type Client struct { // scheme sets the scheme for the client scheme string // host holds the server address to connect to host string // proto holds the client protocol i.e. unix. proto string // addr holds the client address. addr string // basePath holds the path to prepend to the requests. basePath string // client used to send and receive http requests. client *http.Client // version of the server to talk to. version string // custom http headers configured by users. customHTTPHeaders map[string]string // manualOverride is set to true when the version was set by users. manualOverride bool // negotiateVersion indicates if the client should automatically negotiate // the API version to use when making requests. API version negotiation is // performed on the first request, after which negotiated is set to "true" // so that subsequent requests do not re-negotiate. negotiateVersion bool // negotiated indicates that API version negotiation took place negotiated bool } // NewClientWithOpts initializes a new API client with a default HTTPClient, and // default API host and version. It also initializes the custom HTTP headers to // add to each request. // // It takes an optional list of Opt functional arguments, which are applied in // the order they're provided, which allows modifying the defaults when creating // the client. For example, the following initializes a client that configures // itself with values from environment variables (client.FromEnv), and has // automatic API version negotiation enabled (client.WithAPIVersionNegotiation()). // // cli, err := client.NewClientWithOpts( // client.FromEnv, // client.WithAPIVersionNegotiation(), // ) func NewClientWithOpts(ops ...Opt) (*Client, error) { client, err := defaultHTTPClient(DefaultModelzGatewayHost) if err != nil { return nil, err } c := &Client{ host: DefaultModelzGatewayHost, version: "", client: client, proto: defaultProto, addr: defaultAddr, basePath: apiBasePath, } for _, op := range ops { if err := op(c); err != nil { return nil, err } } if c.scheme == "" { c.scheme = "http" tlsConfig := resolveTLSConfig(c.client.Transport) if tlsConfig != nil { // TODO(stevvooe): This isn't really the right way to write clients in Go. // `NewClient` should probably only take an `*http.Client` and work from there. // Unfortunately, the model of having a host-ish/url-thingy as the connection // string has us confusing protocol and transport layers. We continue doing // this to avoid breaking existing clients but this should be addressed. c.scheme = "https" } } return c, nil } func defaultHTTPClient(host string) (*http.Client, error) { hostURL, err := ParseHostURL(host) if err != nil { return nil, err } transport := &http.Transport{} _ = sockets.ConfigureTransport(transport, hostURL.Scheme, hostURL.Host) return &http.Client{ Transport: transport, CheckRedirect: CheckRedirectKeepHeader, }, nil } // CheckRedirect specifies the policy for dealing with redirect responses: // If the request is non-GET return ErrRedirect, otherwise use the last response. // // Go 1.8 changes behavior for HTTP redirects (specifically 301, 307, and 308) // in the client. The envd client (and by extension envd API client) can be // made to send a request like POST /containers//start where what would normally // be in the name section of the URL is empty. This triggers an HTTP 301 from // the daemon. // // In go 1.8 this 301 will be converted to a GET request, and ends up getting // a 404 from the daemon. This behavior change manifests in the client in that // before, the 301 was not followed and the client did not generate an error, // but now results in a message like Error response from daemon: page not found. func CheckRedirect(req *http.Request, via []*http.Request) error { if via[0].Method == http.MethodGet { return http.ErrUseLastResponse } return ErrRedirect } func CheckRedirectKeepHeader(req *http.Request, via []*http.Request) error { req.Header = via[0].Header.Clone() return nil } // DaemonHost returns the host address used by the client func (cli *Client) DaemonHost() string { return cli.host } // HTTPClient returns a copy of the HTTP client bound to the server func (cli *Client) HTTPClient() *http.Client { c := *cli.client return &c } // ParseHostURL parses a url string, validates the string is a host url, and // returns the parsed URL func ParseHostURL(host string) (*url.URL, error) { protoAddrParts := strings.SplitN(host, "://", 2) if len(protoAddrParts) == 1 { return nil, errors.Errorf("unable to parse docker host `%s`", host) } var basePath string proto, addr := protoAddrParts[0], protoAddrParts[1] if proto == "tcp" { parsed, err := url.Parse("tcp://" + addr) if err != nil { return nil, err } addr = parsed.Host basePath = parsed.Path } return &url.URL{ Scheme: proto, Host: addr, Path: basePath, }, nil } // Close the transport used by the client func (cli *Client) Close() error { if t, ok := cli.client.Transport.(*http.Transport); ok { t.CloseIdleConnections() } return nil } // getAPIPath returns the versioned request path to call the api. // It appends the query parameters to the path if they are not empty. func (cli *Client) getAPIPath(ctx context.Context, p string, query url.Values) string { var apiPath string if cli.version != "" { v := strings.TrimPrefix(cli.version, "v") apiPath = path.Join(cli.basePath, "/v"+v, p) } else { apiPath = path.Join(cli.basePath, p) } return (&url.URL{Path: apiPath, RawQuery: query.Encode()}).String() } ================================================ FILE: agent/client/const.go ================================================ // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. package client const DefaultModelzGatewayHost = "http://0.0.0.0:8080" const defaultProto = "http" const defaultAddr = "0.0.0.0:8080" // Base path for api, distinguish from frontend pages const apiBasePath = "" const ( gatewayInferControlPlanePath = "/system/inferences" gatewayInferScaleControlPath = "/system/scale-inference" gatewayInferInstanceControlPlanePath = "/system/inference/%s/instances" gatewayInferInstanceExecControlPlanePath = "/system/inference/%s/instance/%s/exec" gatewayServerControlPlanePath = "/system/servers" gatewayServerLabelCreateControlPlanePath = "/system/server/%s/labels" gatewayServerNodeDeleteControlPlanePath = "/system/server/%s/delete" gatewayNamespaceControlPlanePath = "/system/namespaces" gatewayBuildControlPlanePath = "/system/build" gatewayBuildInstanceControlPlanePath = "/system/build/%s" gatewayImageCacheControlPlanePath = "/system/image-cache" modelzCloudClusterControlPlanePath = "/api/v1/users/%s/clusters/%s" modelzCloudClusterWithUserControlPlanePath = "/api/v1/users/%s/clusters" modelzCloudClusterAPIKeyControlPlanePath = "/api/v1/users/%s/clusters/%s/api_keys" modelzCloudClusterNamespaceControlPlanePath = "/api/v1/users/%s/clusters/%s/namespaces" modelzCloudClusterDeploymentControlPlanePath = "/api/v1/users/%s/clusters/%s/deployments/%s/agent" modelzCloudClusterDeploymentEventControlPlanePath = "/api/v1/users/%s/clusters/%s/deployments/%s/event" ) const ( // EnvOverrideHost is the name of the environment variable that can be used // to override the default host to connect to (DefaultEnvdServerHost). // // This env-var is read by FromEnv and WithHostFromEnv and when set to a // non-empty value, takes precedence over the default host (which is platform // specific), or any host already set. EnvOverrideHost = "MODELZ_GATEWAY_HOST" // EnvOverrideCertPath is the name of the environment variable that can be // used to specify the directory from which to load the TLS certificates // (ca.pem, cert.pem, key.pem) from. These certificates are used to configure // the Client for a TCP connection protected by TLS client authentication. // // TLS certificate verification is enabled by default if the Client is configured // to use a TLS connection. Refer to EnvTLSVerify below to learn how to // disable verification for testing purposes. // // // For local access to the API, it is recommended to connect with the daemon // using the default local socket connection (on Linux), or the named pipe // (on Windows). // // If you need to access the API of a remote daemon, consider using an SSH // (ssh://) connection, which is easier to set up, and requires no additional // configuration if the host is accessible using ssh. EnvOverrideCertPath = "ENVD_SERVER_CERT_PATH" // EnvTLSVerify is the name of the environment variable that can be used to // enable or disable TLS certificate verification. When set to a non-empty // value, TLS certificate verification is enabled, and the client is configured // to use a TLS connection, using certificates from the default directories // (within `~/.envd`); refer to EnvOverrideCertPath above for additional // details. // // // Before setting up your client and daemon to use a TCP connection with TLS // client authentication, consider using one of the alternatives mentioned // in EnvOverrideCertPath above. // // Disabling TLS certificate verification (for testing purposes) // // TLS certificate verification is enabled by default if the Client is configured // to use a TLS connection, and it is highly recommended to keep verification // enabled to prevent machine-in-the-middle attacks. // // Set the "ENVD_SERVER_TLS_VERIFY" environment to an empty string ("") to // disable TLS certificate verification. Disabling verification is insecure, // so should only be done for testing purposes. From the Go documentation // (https://pkg.go.dev/crypto/tls#Config): // // InsecureSkipVerify controls whether a client verifies the server's // certificate chain and host name. If InsecureSkipVerify is true, crypto/tls // accepts any certificate presented by the server and any host name in that // certificate. In this mode, TLS is susceptible to machine-in-the-middle // attacks unless custom verification is used. This should be used only for // testing or in combination with VerifyConnection or VerifyPeerCertificate. EnvTLSVerify = "ENVD_SERVER_TLS_VERIFY" ) ================================================ FILE: agent/client/errors.go ================================================ // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. package client // import "github.com/docker/docker/client" import ( "fmt" "net/http" "github.com/cockroachdb/errors" "github.com/tensorchord/openmodelz/agent/errdefs" ) // errConnectionFailed implements an error returned when connection failed. type errConnectionFailed struct { host string } // Error returns a string representation of an errConnectionFailed func (err errConnectionFailed) Error() string { if err.host == "" { return "Cannot connect to the backend" } return fmt.Sprintf("Cannot connect at %s", err.host) } // IsErrConnectionFailed returns true if the error is caused by connection failed. func IsErrConnectionFailed(err error) bool { return errors.As(err, &errConnectionFailed{}) } // ErrorConnectionFailed returns an error with host in the error message when connection to docker daemon failed. func ErrorConnectionFailed(host string) error { return errConnectionFailed{host: host} } // Deprecated: use the errdefs.NotFound() interface instead. Kept for backward compatibility type notFound interface { error NotFound() bool } // IsErrNotFound returns true if the error is a NotFound error, which is returned // by the API when some object is not found. func IsErrNotFound(err error) bool { if errdefs.IsNotFound(err) { return true } var e notFound return errors.As(err, &e) } type objectNotFoundError struct { object string id string } func (e objectNotFoundError) NotFound() {} func (e objectNotFoundError) Error() string { return fmt.Sprintf("Error: No such %s: %s", e.object, e.id) } // IsErrUnauthorized returns true if the error is caused // when a remote registry authentication fails // // Deprecated: use errdefs.IsUnauthorized func IsErrUnauthorized(err error) bool { return errdefs.IsUnauthorized(err) } type pluginPermissionDenied struct { name string } func (e pluginPermissionDenied) Error() string { return "Permission denied while installing plugin " + e.name } // IsErrNotImplemented returns true if the error is a NotImplemented error. // This is returned by the API when a requested feature has not been // implemented. // // Deprecated: use errdefs.IsNotImplemented func IsErrNotImplemented(err error) bool { return errdefs.IsNotImplemented(err) } func wrapResponseError(err error, resp serverResponse, object, id string) error { switch { case err == nil: return nil case resp.statusCode == http.StatusNotFound: return objectNotFoundError{object: object, id: id} case resp.statusCode == http.StatusNotImplemented: return errdefs.NotImplemented(err) default: return err } } ================================================ FILE: agent/client/hijack.go ================================================ package client // import "docker.io/go-docker" import ( "net/url" "github.com/gorilla/websocket" "golang.org/x/net/context" ) // HijackedResponse holds connection information for a hijacked request. type HijackedResponse struct { Conn *websocket.Conn } // Close closes the hijacked connection and reader. func (h *HijackedResponse) Close() { h.Conn.Close() } // postHijacked sends a POST request and hijacks the connection. func (cli *Client) websocket(ctx context.Context, path string, query url.Values, headers map[string][]string) (HijackedResponse, error) { apiPath := cli.getAPIPath(ctx, path, nil) scheme := "ws" if cli.scheme == "https" { scheme = "wss" } apiURL := url.URL{ Scheme: scheme, Host: cli.addr, Path: apiPath, RawQuery: query.Encode(), } c, _, err := websocket.DefaultDialer.DialContext(ctx, apiURL.String(), nil) if err != nil { return HijackedResponse{}, err } return HijackedResponse{Conn: c}, err } func (h HijackedResponse) Read(p []byte) (int, error) { // Read message from websocket connection. tm := &TerminalMessage{} if err := h.Conn.ReadJSON(tm); err != nil { return 0, err } if tm.Op != "stdout" { return 0, nil } return copy(p, tm.Data), nil } func (h HijackedResponse) Write(p []byte) (int, error) { // Write message to websocket connection. tm := &TerminalMessage{ Op: "stdin", Data: string(p), } if err := h.Conn.WriteJSON(tm); err != nil { return 0, err } return len(p), nil } // TerminalMessage is the messaging protocol between ShellController and TerminalSession. // // OP DIRECTION FIELD(S) USED DESCRIPTION // --------------------------------------------------------------------- // bind fe->be SessionID Id sent back from TerminalResponse // stdin fe->be Data Keystrokes/paste buffer // resize fe->be Rows, Cols New terminal size // stdout be->fe Data Output from the process // toast be->fe Data OOB message to be shown to the user type TerminalMessage struct { ID string `json:"id,omitempty"` Op string `json:"op,omitempty"` Data string `json:"data,omitempty"` Rows uint16 `json:"rows,omitempty"` Cols uint16 `json:"cols,omitempty"` } ================================================ FILE: agent/client/image_cache_create.go ================================================ package client import ( "context" "net/url" "github.com/tensorchord/openmodelz/agent/api/types" ) func (cli *Client) ImageCacheCreate(ctx context.Context, namespace string, imageCache *types.ImageCache) error { urlValues := url.Values{} urlValues.Add("namespace", namespace) resp, err := cli.post(ctx, gatewayImageCacheControlPlanePath, urlValues, imageCache, nil) defer ensureReaderClosed(resp) if err != nil { return wrapResponseError(err, resp, "imagecache", imageCache.Name) } return wrapResponseError(err, resp, "imagecache", imageCache.Name) } ================================================ FILE: agent/client/inference_create.go ================================================ // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. package client import ( "context" "net/url" "github.com/tensorchord/openmodelz/agent/api/types" ) // InferenceCreate creates the inference. func (cli *Client) InferenceCreate(ctx context.Context, namespace string, inference types.InferenceDeployment) (types.InferenceDeployment, error) { urlValues := url.Values{} urlValues.Add("namespace", namespace) resp, err := cli.post(ctx, gatewayInferControlPlanePath, urlValues, inference, nil) defer ensureReaderClosed(resp) if err != nil { return inference, wrapResponseError(err, resp, "inference", inference.Spec.Name) } return inference, wrapResponseError(err, resp, "inference", inference.Spec.Name) } ================================================ FILE: agent/client/inference_get.go ================================================ // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. package client import ( "context" "encoding/json" "fmt" "net/url" "github.com/tensorchord/openmodelz/agent/api/types" ) // InferenceGet gets the inference. func (cli *Client) InferenceGet(ctx context.Context, namespace, name string) (types.InferenceDeployment, error) { urlValues := url.Values{} urlValues.Add("namespace", namespace) url := fmt.Sprintf("/system/inference/%s", name) resp, err := cli.get(ctx, url, urlValues, nil) defer ensureReaderClosed(resp) if err != nil { return types.InferenceDeployment{}, wrapResponseError(err, resp, "inference", name) } var inference types.InferenceDeployment err = json.NewDecoder(resp.body).Decode(&inference) if err != nil { return types.InferenceDeployment{}, wrapResponseError(err, resp, "inference", name) } return inference, wrapResponseError(err, resp, "inference", name) } ================================================ FILE: agent/client/inference_list.go ================================================ // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. package client import ( "context" "encoding/json" "net/url" "github.com/tensorchord/openmodelz/agent/api/types" ) // InferenceList lists the inferences. func (cli *Client) InferenceList(ctx context.Context, namespace string) ([]types.InferenceDeployment, error) { urlValues := url.Values{} urlValues.Add("namespace", namespace) resp, err := cli.get(ctx, gatewayInferControlPlanePath, urlValues, nil) defer ensureReaderClosed(resp) if err != nil { return nil, wrapResponseError(err, resp, "inferences with namespace", namespace) } var inferences []types.InferenceDeployment err = json.NewDecoder(resp.body).Decode(&inferences) return inferences, err } ================================================ FILE: agent/client/inference_remove.go ================================================ // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. package client import ( "context" "net/url" "github.com/tensorchord/openmodelz/agent/api/types" ) // InferenceRemove removes the inference. func (cli *Client) InferenceRemove(ctx context.Context, namespace string, name string) error { urlValues := url.Values{} urlValues.Add("namespace", namespace) req := types.DeleteFunctionRequest{ FunctionName: name, } resp, err := cli.delete(ctx, gatewayInferControlPlanePath, urlValues, req, nil) defer ensureReaderClosed(resp) return wrapResponseError(err, resp, "inference", name) } ================================================ FILE: agent/client/inference_scale.go ================================================ package client import ( "context" "net/url" "github.com/tensorchord/openmodelz/agent/api/types" ) // InferenceScale scales the inference. func (cli *Client) InferenceScale(ctx context.Context, namespace string, name string, replicas int, eventMessage string) error { urlValues := url.Values{} urlValues.Add("namespace", namespace) req := types.ScaleServiceRequest{ ServiceName: name, Replicas: uint64(replicas), EventMessage: eventMessage, } resp, err := cli.post(ctx, gatewayInferScaleControlPath, urlValues, req, nil) defer ensureReaderClosed(resp) return wrapResponseError(err, resp, "inference", name) } ================================================ FILE: agent/client/inference_update.go ================================================ // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. package client import ( "context" "net/url" "github.com/tensorchord/openmodelz/agent/api/types" ) // DeploymentUpdate creates the deployment. func (cli *Client) DeploymentUpdate(ctx context.Context, namespace string, inference types.InferenceDeployment) (types.InferenceDeployment, error) { urlValues := url.Values{} urlValues.Add("namespace", namespace) resp, err := cli.put(ctx, gatewayInferControlPlanePath, urlValues, inference, nil) defer ensureReaderClosed(resp) if err != nil { return inference, wrapResponseError(err, resp, "inference", inference.Spec.Name) } return inference, wrapResponseError(err, resp, "inference", inference.Spec.Name) } ================================================ FILE: agent/client/info.go ================================================ // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. package client import ( "context" "encoding/json" "github.com/tensorchord/openmodelz/agent/api/types" ) // InfoGet gets the agent info. func (cli *Client) InfoGet(ctx context.Context) (types.ProviderInfo, error) { resp, err := cli.get(ctx, "/system/info", nil, nil) defer ensureReaderClosed(resp) if err != nil { return types.ProviderInfo{}, wrapResponseError(err, resp, "info", "system") } var info types.ProviderInfo err = json.NewDecoder(resp.body).Decode(&info) if err != nil { return types.ProviderInfo{}, wrapResponseError(err, resp, "info", "system") } return info, wrapResponseError(err, resp, "info", "system") } ================================================ FILE: agent/client/instance_exec.go ================================================ // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. package client import ( "context" "fmt" "io" "net/url" "strings" ) // InstanceExec executes command in the instance. func (cli *Client) InstanceExec(ctx context.Context, namespace, inferenceName, instance string, command []string, tty bool) (string, error) { urlValues := url.Values{} urlValues.Add("namespace", namespace) urlValues.Add("tty", fmt.Sprintf("%v", tty)) urlValues.Add("command", strings.Join(command, ",")) urlPath := fmt.Sprintf(gatewayInferInstanceExecControlPlanePath, inferenceName, instance) resp, err := cli.get(ctx, urlPath, urlValues, nil) defer ensureReaderClosed(resp) if err != nil { return "", wrapResponseError(err, resp, "instances with namespace", namespace) } res, err := io.ReadAll(resp.body) if err != nil { return "", wrapResponseError(err, resp, "instances with namespace", namespace) } return string(res), wrapResponseError(err, resp, "instances with namespace", namespace) } // InstanceExec executes command in the instance. func (cli *Client) InstanceExecTTY(ctx context.Context, namespace, inferenceName, instance string, command []string, ) (HijackedResponse, error) { urlValues := url.Values{} urlValues.Add("namespace", namespace) urlValues.Add("tty", "true") urlValues.Add("command", strings.Join(command, ",")) urlPath := fmt.Sprintf(gatewayInferInstanceExecControlPlanePath, inferenceName, instance) resp, err := cli.websocket(ctx, urlPath, urlValues, nil) if err != nil { return HijackedResponse{}, wrapResponseError(err, serverResponse{}, "instances with namespace", namespace) } return resp, nil } ================================================ FILE: agent/client/instance_list.go ================================================ // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. package client import ( "context" "encoding/json" "fmt" "net/url" "github.com/tensorchord/openmodelz/agent/api/types" ) // InstanceList lists the deployment instances. func (cli *Client) InstanceList(ctx context.Context, namespace, inferenceName string) ([]types.InferenceDeploymentInstance, error) { urlValues := url.Values{} urlValues.Add("namespace", namespace) urlPath := fmt.Sprintf(gatewayInferInstanceControlPlanePath, inferenceName) resp, err := cli.get(ctx, urlPath, urlValues, nil) defer ensureReaderClosed(resp) if err != nil { return nil, wrapResponseError(err, resp, "instances with namespace", namespace) } var instances []types.InferenceDeploymentInstance err = json.NewDecoder(resp.body).Decode(&instances) return instances, wrapResponseError(err, resp, "instances with namespace", namespace) } ================================================ FILE: agent/client/log.go ================================================ // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. package client import ( "bufio" "context" "encoding/json" "fmt" "net/url" "strings" "github.com/sirupsen/logrus" "github.com/tensorchord/openmodelz/agent/api/types" ) const LogBufferSize = 128 // DeploymentLogGet gets the deployment logs. func (cli *Client) DeploymentLogGet(ctx context.Context, namespace, name string, since string, tail int, end string, follow bool) ( <-chan types.Message, error) { urlValues := url.Values{} urlValues.Add("namespace", namespace) urlValues.Add("name", name) if since != "" { urlValues.Add("since", since) } if end != "" { urlValues.Add("end", end) } if tail != 0 { urlValues.Add("tail", fmt.Sprintf("%d", tail)) } if follow { urlValues.Add("follow", "true") } resp, err := cli.get(ctx, "/system/logs/inference", urlValues, nil) if err != nil { return nil, wrapResponseError(err, resp, "deployment logs", name) } stream := make(chan types.Message, LogBufferSize) var log types.Message scanner := bufio.NewScanner(resp.body) go func() { defer ensureReaderClosed(resp) defer close(stream) for scanner.Scan() { err = json.Unmarshal(scanner.Bytes(), &log) if err != nil { logrus.Warnf("failed to decode %s log: %v | %s | [%s]", name, err, scanner.Text(), scanner.Err()) return // continue } stream <- log } }() return stream, err } func (cli *Client) BuildLogGet(ctx context.Context, namespace, name, since string, tail int) ([]types.Message, error) { urlValues := url.Values{} urlValues.Add("namespace", namespace) urlValues.Add("name", name) if since != "" { urlValues.Add("since", since) } if tail != 0 { urlValues.Add("tail", fmt.Sprintf("%d", tail)) } resp, err := cli.get(ctx, "/system/logs/build", urlValues, nil) defer ensureReaderClosed(resp) if err != nil { return nil, wrapResponseError(err, resp, "build logs", name) } var log types.Message logs := []types.Message{} scanner := bufio.NewScanner(resp.body) for scanner.Scan() { err = json.NewDecoder(strings.NewReader(scanner.Text())).Decode(&log) if err != nil { return nil, wrapResponseError(err, resp, "build logs", name) } logs = append(logs, log) } return logs, err } ================================================ FILE: agent/client/modelz_cloud.go ================================================ package client import ( "context" "encoding/json" "fmt" "net/http" "net/url" "time" "github.com/sirupsen/logrus" "github.com/tensorchord/openmodelz/agent/api/types" "github.com/tensorchord/openmodelz/agent/pkg/consts" "k8s.io/apimachinery/pkg/util/wait" ) func (cli *Client) WaitForAPIServerReady() error { err := wait.PollImmediateWithContext(context.Background(), time.Second, consts.DefaultAPIServerReadyTimeout, func(ctx context.Context) (bool, error) { err, healthStatus := cli.waitForAPIServerReady(ctx) if err != nil || healthStatus != http.StatusOK { logrus.Warn("APIServer isn't ready yet, Waiting a little while.") return false, err } return true, nil }) if err != nil { return fmt.Errorf("failed to wait for apiserver ready, %v", err) } return nil } func (cli *Client) waitForAPIServerReady(ctx context.Context) (error, int) { urlValues := url.Values{} resp, err := cli.get(ctx, "/healthz", urlValues, nil) if err != nil { return wrapResponseError(err, resp, "check apiserver is ready", ""), resp.statusCode } defer ensureReaderClosed(resp) return nil, resp.statusCode } func (cli *Client) RegisterAgent(ctx context.Context, token string, cluster *types.ManagedCluster) error { urlValues := url.Values{} agentToken, err := ParseAgentToken(token) if err != nil { return err } urlPath := fmt.Sprintf(modelzCloudClusterWithUserControlPlanePath, agentToken.UID) headers := make(map[string][]string) headers["Authorization"] = []string{"Bearer " + agentToken.Token} cluster.Name = agentToken.ClusterName resp, err := cli.post(ctx, urlPath, urlValues, cluster, headers) if err != nil { return wrapResponseError(err, resp, "register agent to modelz cloud", agentToken.UID) } defer ensureReaderClosed(resp) err = json.NewDecoder(resp.body).Decode(&cluster) if err != nil { return err } return nil } func (cli *Client) UpdateAgentStatus(ctx context.Context, apiServerReady <-chan struct{}, token string, cluster types.ManagedCluster) error { <-apiServerReady urlValues := url.Values{} agentToken, err := ParseAgentToken(token) if err != nil { return err } urlPath := fmt.Sprintf(modelzCloudClusterControlPlanePath, agentToken.UID, cluster.ID) headers := make(map[string][]string) headers["Authorization"] = []string{"Bearer " + agentToken.Token} resp, err := cli.put(ctx, urlPath, urlValues, cluster, headers) if err != nil { return wrapResponseError(err, resp, "update agent status to modelz cloud", agentToken.UID) } defer ensureReaderClosed(resp) if resp.statusCode == 200 { return nil } return fmt.Errorf("failed to update agent status to modelz cloud, status code: %d", resp.statusCode) } func (cli *Client) GetAPIKeys(ctx context.Context, apiServerReady <-chan struct{}, token string, cluster string) (types.APIKeyMap, error) { <-apiServerReady urlValues := url.Values{} agentToken, err := ParseAgentToken(token) keys := types.APIKeyMap{} if err != nil { return keys, err } headers := make(map[string][]string) headers["Authorization"] = []string{"Bearer " + agentToken.Token} urlPath := fmt.Sprintf(modelzCloudClusterAPIKeyControlPlanePath, agentToken.UID, cluster) resp, err := cli.get(ctx, urlPath, urlValues, headers) if err != nil { return keys, wrapResponseError(err, resp, "get api keys from modelz cloud", agentToken.UID) } defer ensureReaderClosed(resp) err = json.NewDecoder(resp.body).Decode(&keys) if err != nil { return keys, err } return keys, nil } func (cli *Client) GetNamespaces(ctx context.Context, apiServerReady <-chan struct{}, token string, cluster string) (types.NamespaceList, error) { <-apiServerReady urlValues := url.Values{} agentToken, err := ParseAgentToken(token) ns := types.NamespaceList{} if err != nil { return ns, err } urlValues.Add("login_name", agentToken.UID) headers := make(map[string][]string) headers["Authorization"] = []string{"Bearer " + agentToken.Token} resp, err := cli.get(ctx, fmt.Sprintf(modelzCloudClusterNamespaceControlPlanePath, agentToken.UID, cluster), urlValues, headers) if err != nil { return ns, wrapResponseError(err, resp, "get namespaces from modelz cloud", agentToken.UID) } defer ensureReaderClosed(resp) err = json.NewDecoder(resp.body).Decode(&ns) if err != nil { return ns, err } ns.Items = append(ns.Items, GetNamespaceByUserID(agentToken.UID)) return ns, nil } func (cli *Client) GetUIDFromDeploymentID(ctx context.Context, token string, cluster string, deployment string) (string, error) { urlValues := url.Values{} agentToken, err := ParseAgentToken(token) if err != nil { return "", err } headers := make(map[string][]string) headers["Authorization"] = []string{"Bearer " + agentToken.Token} urlPath := fmt.Sprintf(modelzCloudClusterDeploymentControlPlanePath, agentToken.UID, cluster, deployment) resp, err := cli.get(ctx, urlPath, urlValues, headers) if err != nil { return "", err } defer ensureReaderClosed(resp) var uid string err = json.NewDecoder(resp.body).Decode(&uid) if err != nil { return "", err } if resp.statusCode == 200 { return uid, nil } return "", fmt.Errorf("failed to get uid from deployment id, status code: %d", resp.statusCode) } func (cli *Client) CreateDeploymentEvent(ctx context.Context, token string, event types.DeploymentEvent) error { urlValues := url.Values{} agentToken, err := ParseAgentToken(token) if err != nil { return err } headers := make(map[string][]string) headers["Authorization"] = []string{"Bearer " + agentToken.Token} urlPath := fmt.Sprintf(modelzCloudClusterDeploymentEventControlPlanePath, agentToken.UID, agentToken.ClusterName, event.DeploymentID) resp, err := cli.post(ctx, urlPath, urlValues, event, headers) if err != nil { return err } defer ensureReaderClosed(resp) if resp.statusCode == http.StatusCreated { return nil } return fmt.Errorf("failed to create deployment event, status code: %d", resp.statusCode) } ================================================ FILE: agent/client/namespace_create.go ================================================ // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. package client import ( "context" "net/url" "github.com/tensorchord/openmodelz/agent/api/types" ) // NamespaceCreate creates the namespace. func (cli *Client) NamespaceCreate(ctx context.Context, namespace string) error { req := types.NamespaceRequest{ Name: namespace, } urlValues := url.Values{} resp, err := cli.post(ctx, gatewayNamespaceControlPlanePath, urlValues, req, nil) defer ensureReaderClosed(resp) return wrapResponseError(err, resp, "namespace", namespace) } ================================================ FILE: agent/client/namespace_delete.go ================================================ // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. package client import ( "context" "net/url" "github.com/tensorchord/openmodelz/agent/api/types" ) // NamespaceDelete deletes the namespace. func (cli *Client) NamespaceDelete(ctx context.Context, namespace string) error { req := types.NamespaceRequest{ Name: namespace, } urlValues := url.Values{} resp, err := cli.delete(ctx, gatewayNamespaceControlPlanePath, urlValues, req, nil) defer ensureReaderClosed(resp) return wrapResponseError(err, resp, "namespace", namespace) } ================================================ FILE: agent/client/options.go ================================================ // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. package client import ( "context" "net" "net/http" "os" "path/filepath" "time" "github.com/cockroachdb/errors" "github.com/docker/go-connections/sockets" "github.com/docker/go-connections/tlsconfig" ) // Opt is a configuration option to initialize a client type Opt func(*Client) error // FromEnv configures the client with values from environment variables. // // FromEnv uses the following environment variables: // // ENVD_SERVER_HOST (EnvOverrideHost) to set the URL to the docker server. // // ENVD_SERVER_CERT_PATH (EnvOverrideCertPath) to specify the directory from which to // load the TLS certificates (ca.pem, cert.pem, key.pem). // // ENVD_SERVER_TLS_VERIFY (EnvTLSVerify) to enable or disable TLS verification (off by // default). func FromEnv(c *Client) error { // TODO(gaocegege): Support: // ENVD_SERVER_API_VERSION (EnvOverrideAPIVersion) to set the version of the API to // use, leave empty for latest. // ops := []Opt{ WithTLSClientConfigFromEnv(), WithHostFromEnv(), } for _, op := range ops { if err := op(c); err != nil { return err } } return nil } // WithDialContext applies the dialer to the client transport. This can be // used to set the Timeout and KeepAlive settings of the client. func WithDialContext(dialContext func(ctx context.Context, network, addr string) (net.Conn, error)) Opt { return func(c *Client) error { if transport, ok := c.client.Transport.(*http.Transport); ok { transport.DialContext = dialContext return nil } return errors.Errorf("cannot apply dialer to transport: %T", c.client.Transport) } } // WithHost overrides the client host with the specified one. func WithHost(host string) Opt { return func(c *Client) error { hostURL, err := ParseHostURL(host) if err != nil { return err } c.host = host c.proto = hostURL.Scheme c.addr = hostURL.Host c.basePath = hostURL.Path if transport, ok := c.client.Transport.(*http.Transport); ok { return sockets.ConfigureTransport(transport, c.proto, c.addr) } return errors.Errorf("cannot apply host to transport: %T", c.client.Transport) } } // WithHostFromEnv overrides the client host with the host specified in the // DOCKER_HOST (EnvOverrideHost) environment variable. If DOCKER_HOST is not set, // or set to an empty value, the host is not modified. func WithHostFromEnv() Opt { return func(c *Client) error { if host := os.Getenv(EnvOverrideHost); host != "" { return WithHost(host)(c) } return nil } } // WithHTTPClient overrides the client http client with the specified one func WithHTTPClient(client *http.Client) Opt { return func(c *Client) error { if client != nil { c.client = client } return nil } } // WithTimeout configures the time limit for requests made by the HTTP client func WithTimeout(timeout time.Duration) Opt { return func(c *Client) error { c.client.Timeout = timeout return nil } } // WithHTTPHeaders overrides the client default http headers func WithHTTPHeaders(headers map[string]string) Opt { return func(c *Client) error { c.customHTTPHeaders = headers return nil } } // WithScheme overrides the client scheme with the specified one func WithScheme(scheme string) Opt { return func(c *Client) error { c.scheme = scheme return nil } } // WithTLSClientConfig applies a tls config to the client transport. func WithTLSClientConfig(cacertPath, certPath, keyPath string) Opt { return func(c *Client) error { opts := tlsconfig.Options{ CAFile: cacertPath, CertFile: certPath, KeyFile: keyPath, ExclusiveRootPools: true, } config, err := tlsconfig.Client(opts) if err != nil { return errors.Wrap(err, "failed to create tls config") } if transport, ok := c.client.Transport.(*http.Transport); ok { transport.TLSClientConfig = config return nil } return errors.Errorf("cannot apply tls config to transport: %T", c.client.Transport) } } // WithTLSClientConfigFromEnv configures the client's TLS settings with the // settings in the DOCKER_CERT_PATH and DOCKER_TLS_VERIFY environment variables. // If DOCKER_CERT_PATH is not set or empty, TLS configuration is not modified. // // WithTLSClientConfigFromEnv uses the following environment variables: // // DOCKER_CERT_PATH (EnvOverrideCertPath) to specify the directory from which to // load the TLS certificates (ca.pem, cert.pem, key.pem). // // DOCKER_TLS_VERIFY (EnvTLSVerify) to enable or disable TLS verification (off by // default). func WithTLSClientConfigFromEnv() Opt { return func(c *Client) error { dockerCertPath := os.Getenv(EnvOverrideCertPath) if dockerCertPath == "" { return nil } options := tlsconfig.Options{ CAFile: filepath.Join(dockerCertPath, "ca.pem"), CertFile: filepath.Join(dockerCertPath, "cert.pem"), KeyFile: filepath.Join(dockerCertPath, "key.pem"), InsecureSkipVerify: os.Getenv(EnvTLSVerify) == "", } tlsc, err := tlsconfig.Client(options) if err != nil { return err } c.client = &http.Client{ Transport: &http.Transport{TLSClientConfig: tlsc}, CheckRedirect: CheckRedirect, } return nil } } // WithVersion overrides the client version with the specified one. If an empty // version is specified, the value will be ignored to allow version negotiation. func WithVersion(version string) Opt { return func(c *Client) error { if version != "" { c.version = version c.manualOverride = true } return nil } } ================================================ FILE: agent/client/request.go ================================================ // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. package client import ( "bytes" "context" "encoding/json" "fmt" "io" "net" "net/http" "net/url" "os" "strings" "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "github.com/tensorchord/openmodelz/agent/errdefs" ) // serverResponse is a wrapper for http API responses. type serverResponse struct { body io.ReadCloser header http.Header statusCode int reqURL *url.URL } // head sends an http request to the docker API using the method HEAD. func (cli *Client) head(ctx context.Context, path string, query url.Values, headers map[string][]string) (serverResponse, error) { return cli.sendRequest(ctx, http.MethodHead, path, query, nil, headers) } // get sends an http request to the docker API using the method GET with a specific Go context. func (cli *Client) get(ctx context.Context, path string, query url.Values, headers map[string][]string) (serverResponse, error) { return cli.sendRequest(ctx, http.MethodGet, path, query, nil, headers) } // post sends an http request to the docker API using the method POST with a specific Go context. func (cli *Client) post(ctx context.Context, path string, query url.Values, obj interface{}, headers map[string][]string) (serverResponse, error) { body, headers, err := encodeBody(obj, headers) if err != nil { return serverResponse{}, err } return cli.sendRequest(ctx, http.MethodPost, path, query, body, headers) } func (cli *Client) postRaw(ctx context.Context, path string, query url.Values, body io.Reader, headers map[string][]string) (serverResponse, error) { return cli.sendRequest(ctx, http.MethodPost, path, query, body, headers) } func (cli *Client) put(ctx context.Context, path string, query url.Values, obj interface{}, headers map[string][]string) (serverResponse, error) { body, headers, err := encodeBody(obj, headers) if err != nil { return serverResponse{}, err } return cli.sendRequest(ctx, http.MethodPut, path, query, body, headers) } // putRaw sends an http request to the docker API using the method PUT. func (cli *Client) putRaw(ctx context.Context, path string, query url.Values, body io.Reader, headers map[string][]string) (serverResponse, error) { return cli.sendRequest(ctx, http.MethodPut, path, query, body, headers) } // delete sends an http request to the docker API using the method DELETE. func (cli *Client) delete(ctx context.Context, path string, query url.Values, obj interface{}, headers map[string][]string) (serverResponse, error) { body, headers, err := encodeBody(obj, headers) if err != nil { return serverResponse{}, err } return cli.sendRequest(ctx, http.MethodDelete, path, query, body, headers) } type headers map[string][]string func encodeBody(obj interface{}, headers headers) (io.Reader, headers, error) { if obj == nil { return nil, headers, nil } body, err := encodeData(obj) if err != nil { return nil, headers, err } if headers == nil { headers = make(map[string][]string) } headers["Content-Type"] = []string{"application/json"} return body, headers, nil } func (cli *Client) buildRequest(method, path string, body io.Reader, headers headers) (*http.Request, error) { expectedPayload := (method == http.MethodPost || method == http.MethodPut) if expectedPayload && body == nil { body = bytes.NewReader([]byte{}) } req, err := http.NewRequest(method, path, body) if err != nil { return nil, err } req = cli.addHeaders(req, headers) if cli.proto == "unix" || cli.proto == "npipe" { // For local communications, it doesn't matter what the host is. We just // need a valid and meaningful host name. req.Host = "modelz" } req.URL.Host = cli.addr req.URL.Scheme = cli.scheme if expectedPayload && req.Header.Get("Content-Type") == "" { req.Header.Set("Content-Type", "text/plain") } logrus.Debugf("Sending HTTP request to %s\n", req.URL.String()) logrus.Debugf("Request Headers: %v\n", req.Header) logrus.Debugf("Request Body: %v\n", body) return req, nil } func (cli *Client) sendRequest(ctx context.Context, method, path string, query url.Values, body io.Reader, headers headers) (serverResponse, error) { req, err := cli.buildRequest(method, cli.getAPIPath(ctx, path, query), body, headers) if err != nil { return serverResponse{}, errors.Wrap(err, "failed to build request") } resp, err := cli.doRequest(ctx, req) switch { case errors.Is(err, context.Canceled): return serverResponse{}, errdefs.Cancelled(err) case errors.Is(err, context.DeadlineExceeded): return serverResponse{}, errdefs.Deadline(err) case err == nil: err = cli.checkResponseErr(resp) } return resp, errdefs.FromStatusCode(err, resp.statusCode) } func (cli *Client) doRequest(ctx context.Context, req *http.Request) (serverResponse, error) { serverResp := serverResponse{statusCode: -1, reqURL: req.URL} req = req.WithContext(ctx) resp, err := cli.client.Do(req) if err != nil { if cli.scheme != "https" && strings.Contains(err.Error(), "malformed HTTP response") { return serverResp, fmt.Errorf("%v.\n* Are you trying to connect to a TLS-enabled daemon without TLS?", err) } if cli.scheme == "https" && strings.Contains(err.Error(), "bad certificate") { return serverResp, errors.Wrap(err, "the server probably has client authentication (--tlsverify) enabled; check your TLS client certification settings") } // Don't decorate context sentinel errors; users may be comparing to // them directly. if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { return serverResp, err } if nErr, ok := err.(*url.Error); ok { if nErr, ok := nErr.Err.(*net.OpError); ok { if os.IsPermission(nErr.Err) { return serverResp, errors.Wrapf(err, "permission denied while trying to connect to the modelz agent server socket at %v", cli.host) } } } if err, ok := err.(net.Error); ok { if err.Timeout() { return serverResp, ErrorConnectionFailed(cli.host) } if strings.Contains(err.Error(), "connection refused") || strings.Contains(err.Error(), "dial unix") { return serverResp, ErrorConnectionFailed(cli.host) } } return serverResp, errors.Wrap(err, "error during connect") } if resp != nil { serverResp.statusCode = resp.StatusCode serverResp.body = resp.Body serverResp.header = resp.Header } return serverResp, nil } func (cli *Client) checkResponseErr(serverResp serverResponse) error { if serverResp.statusCode >= 200 && serverResp.statusCode < 400 { return nil } var body []byte var err error if serverResp.body != nil { bodyMax := 1 * 1024 * 1024 // 1 MiB bodyR := &io.LimitedReader{ R: serverResp.body, N: int64(bodyMax), } body, err = io.ReadAll(bodyR) if err != nil { return err } if bodyR.N == 0 { return fmt.Errorf("request returned %s with a message (> %d bytes) for API route and version %s, check if the server supports the requested API version", http.StatusText(serverResp.statusCode), bodyMax, serverResp.reqURL) } } if len(body) == 0 { return fmt.Errorf("request returned %s for API route and version %s, check if the server supports the requested API version", http.StatusText(serverResp.statusCode), serverResp.reqURL) } errorMessage := strings.TrimSpace(string(body)) return errors.Wrap(errors.New(errorMessage), "Error response from gateway") } func (cli *Client) addHeaders(req *http.Request, headers headers) *http.Request { // Add CLI Config's HTTP Headers BEFORE we set the Docker headers // then the user can't change OUR headers for k, v := range cli.customHTTPHeaders { req.Header.Set(k, v) } for k, v := range headers { req.Header[http.CanonicalHeaderKey(k)] = v } return req } func encodeData(data interface{}) (*bytes.Buffer, error) { params := bytes.NewBuffer(nil) if data != nil { if err := json.NewEncoder(params).Encode(data); err != nil { return nil, err } } return params, nil } func ensureReaderClosed(response serverResponse) { if response.body != nil { // Drain up to 512 bytes and close the body to let the Transport reuse the connection io.CopyN(io.Discard, response.body, 512) response.body.Close() } } ================================================ FILE: agent/client/server_label_create.go ================================================ // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. package client import ( "context" "fmt" "net/url" "github.com/tensorchord/openmodelz/agent/api/types" ) // ServerLabelCreate create the labels for the servers. func (cli *Client) ServerLabelCreate(ctx context.Context, name string, labels map[string]string) error { req := types.ServerSpec{ Name: name, Labels: labels, } urlValues := url.Values{} resp, err := cli.post(ctx, fmt.Sprintf(gatewayServerLabelCreateControlPlanePath, name), urlValues, req, nil) defer ensureReaderClosed(resp) if err != nil { return wrapResponseError(err, resp, "server", name) } return wrapResponseError(err, resp, "server", name) } ================================================ FILE: agent/client/server_list.go ================================================ // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. package client import ( "context" "encoding/json" "github.com/tensorchord/openmodelz/agent/api/types" ) // ServerList lists the servers. func (cli *Client) ServerList(ctx context.Context) ([]types.Server, error) { resp, err := cli.get(ctx, gatewayServerControlPlanePath, nil, nil) defer ensureReaderClosed(resp) if err != nil { return nil, wrapResponseError(err, resp, "servers", "") } var servers []types.Server err = json.NewDecoder(resp.body).Decode(&servers) return servers, wrapResponseError(err, resp, "servers", "") } ================================================ FILE: agent/client/server_node_delete.go ================================================ // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. package client import ( "context" "fmt" "net/url" ) // ServerLabelCreate create the labels for the servers. func (cli *Client) ServerNodeDelete(ctx context.Context, name string) error { urlValues := url.Values{} resp, err := cli.delete(ctx, fmt.Sprintf(gatewayServerNodeDeleteControlPlanePath, name), urlValues, nil, nil) defer ensureReaderClosed(resp) if err != nil { return wrapResponseError(err, resp, "server-delete", name) } return wrapResponseError(err, resp, "server-delete", name) } ================================================ FILE: agent/client/transport.go ================================================ // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. package client // import "github.com/docker/docker/client" import ( "crypto/tls" "net/http" ) // resolveTLSConfig attempts to resolve the TLS configuration from the // RoundTripper. func resolveTLSConfig(transport http.RoundTripper) *tls.Config { switch tr := transport.(type) { case *http.Transport: return tr.TLSClientConfig default: return nil } } ================================================ FILE: agent/client/utils.go ================================================ package client import ( "fmt" "strings" "github.com/cockroachdb/errors" "github.com/tensorchord/openmodelz/agent/api/types" ) const ( DefaultPrefix = "modelz-" ) func ParseAgentToken(token string) (types.AgentToken, error) { agentToken := types.AgentToken{} if token == "" { return agentToken, errors.New("agent token is empty") } strings := strings.Split(token, ":") if len(strings) != 3 { return agentToken, errors.New("invalid agent token") } agentToken.ClusterName = strings[0] agentToken.UID = strings[1] agentToken.Token = strings[2] return agentToken, nil } func GetNamespaceByUserID(uid string) string { return fmt.Sprintf("%s%s", DefaultPrefix, uid) } func GetUserIDFromNamespace(ns string) (string, error) { if len(ns) < 8 { return "", fmt.Errorf("namespace too short") } if ns[:len(DefaultPrefix)] != DefaultPrefix { return "", fmt.Errorf("namespace does not start with ") } return ns[7:], nil } ================================================ FILE: agent/cmd/agent/main.go ================================================ package main import ( "fmt" "os" "github.com/sirupsen/logrus" "github.com/urfave/cli/v2" "github.com/tensorchord/openmodelz/agent/pkg/app" "github.com/tensorchord/openmodelz/agent/pkg/version" ) func run(args []string) error { cli.VersionPrinter = func(c *cli.Context) { fmt.Println(c.App.Name, version.Package, c.App.Version, version.Revision) } app := app.New() return app.Run(args) } func handleErr(err error) { if err == nil { return } logrus.Error(err) os.Exit(1) } // @title modelz cluster agent // @version v0.0.23 // @description modelz kubernetes cluster agent // @contact.name modelz support // @contact.url https://github.com/tensorchord/openmodelz // @contact.email modelz-support@tensorchord.ai // @host localhost:8081 // @BasePath / // @schemes http func main() { err := run(os.Args) handleErr(err) } ================================================ FILE: agent/errdefs/defs.go ================================================ // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. package errdefs // import "github.com/docker/docker/errdefs" // ErrNotFound signals that the requested object doesn't exist type ErrNotFound interface { NotFound() } // ErrInvalidParameter signals that the user input is invalid type ErrInvalidParameter interface { InvalidParameter() } // ErrConflict signals that some internal state conflicts with the requested action and can't be performed. // A change in state should be able to clear this error. type ErrConflict interface { Conflict() } // ErrUnauthorized is used to signify that the user is not authorized to perform a specific action type ErrUnauthorized interface { Unauthorized() } // ErrUnavailable signals that the requested action/subsystem is not available. type ErrUnavailable interface { Unavailable() } // ErrForbidden signals that the requested action cannot be performed under any circumstances. // When a ErrForbidden is returned, the caller should never retry the action. type ErrForbidden interface { Forbidden() } // ErrSystem signals that some internal error occurred. // An example of this would be a failed mount request. type ErrSystem interface { System() } // ErrNotModified signals that an action can't be performed because it's already in the desired state type ErrNotModified interface { NotModified() } // ErrNotImplemented signals that the requested action/feature is not implemented on the system as configured. type ErrNotImplemented interface { NotImplemented() } // ErrUnknown signals that the kind of error that occurred is not known. type ErrUnknown interface { Unknown() } // ErrCancelled signals that the action was cancelled. type ErrCancelled interface { Cancelled() } // ErrDeadline signals that the deadline was reached before the action completed. type ErrDeadline interface { DeadlineExceeded() } // ErrDataLoss indicates that data was lost or there is data corruption. type ErrDataLoss interface { DataLoss() } ================================================ FILE: agent/errdefs/doc.go ================================================ // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. // Package errdefs defines a set of error interfaces that packages should use for communicating classes of errors. // Errors that cross the package boundary should implement one (and only one) of these interfaces. // // Packages should not reference these interfaces directly, only implement them. // To check if a particular error implements one of these interfaces, there are helper // functions provided (e.g. `Is`) which can be used rather than asserting the interfaces directly. // If you must assert on these interfaces, be sure to check the causal chain (`err.Cause()`). package errdefs // import "github.com/docker/docker/errdefs" ================================================ FILE: agent/errdefs/helpers.go ================================================ // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. package errdefs // import "github.com/docker/docker/errdefs" import "context" type errNotFound struct{ error } func (errNotFound) NotFound() {} func (e errNotFound) Cause() error { return e.error } func (e errNotFound) Unwrap() error { return e.error } // NotFound is a helper to create an error of the class with the same name from any error type func NotFound(err error) error { if err == nil || IsNotFound(err) { return err } return errNotFound{err} } type errInvalidParameter struct{ error } func (errInvalidParameter) InvalidParameter() {} func (e errInvalidParameter) Cause() error { return e.error } func (e errInvalidParameter) Unwrap() error { return e.error } // InvalidParameter is a helper to create an error of the class with the same name from any error type func InvalidParameter(err error) error { if err == nil || IsInvalidParameter(err) { return err } return errInvalidParameter{err} } type errConflict struct{ error } func (errConflict) Conflict() {} func (e errConflict) Cause() error { return e.error } func (e errConflict) Unwrap() error { return e.error } // Conflict is a helper to create an error of the class with the same name from any error type func Conflict(err error) error { if err == nil || IsConflict(err) { return err } return errConflict{err} } type errUnauthorized struct{ error } func (errUnauthorized) Unauthorized() {} func (e errUnauthorized) Cause() error { return e.error } func (e errUnauthorized) Unwrap() error { return e.error } // Unauthorized is a helper to create an error of the class with the same name from any error type func Unauthorized(err error) error { if err == nil || IsUnauthorized(err) { return err } return errUnauthorized{err} } type errUnavailable struct{ error } func (errUnavailable) Unavailable() {} func (e errUnavailable) Cause() error { return e.error } func (e errUnavailable) Unwrap() error { return e.error } // Unavailable is a helper to create an error of the class with the same name from any error type func Unavailable(err error) error { if err == nil || IsUnavailable(err) { return err } return errUnavailable{err} } type errForbidden struct{ error } func (errForbidden) Forbidden() {} func (e errForbidden) Cause() error { return e.error } func (e errForbidden) Unwrap() error { return e.error } // Forbidden is a helper to create an error of the class with the same name from any error type func Forbidden(err error) error { if err == nil || IsForbidden(err) { return err } return errForbidden{err} } type errSystem struct{ error } func (errSystem) System() {} func (e errSystem) Cause() error { return e.error } func (e errSystem) Unwrap() error { return e.error } // System is a helper to create an error of the class with the same name from any error type func System(err error) error { if err == nil || IsSystem(err) { return err } return errSystem{err} } type errNotModified struct{ error } func (errNotModified) NotModified() {} func (e errNotModified) Cause() error { return e.error } func (e errNotModified) Unwrap() error { return e.error } // NotModified is a helper to create an error of the class with the same name from any error type func NotModified(err error) error { if err == nil || IsNotModified(err) { return err } return errNotModified{err} } type errNotImplemented struct{ error } func (errNotImplemented) NotImplemented() {} func (e errNotImplemented) Cause() error { return e.error } func (e errNotImplemented) Unwrap() error { return e.error } // NotImplemented is a helper to create an error of the class with the same name from any error type func NotImplemented(err error) error { if err == nil || IsNotImplemented(err) { return err } return errNotImplemented{err} } type errUnknown struct{ error } func (errUnknown) Unknown() {} func (e errUnknown) Cause() error { return e.error } func (e errUnknown) Unwrap() error { return e.error } // Unknown is a helper to create an error of the class with the same name from any error type func Unknown(err error) error { if err == nil || IsUnknown(err) { return err } return errUnknown{err} } type errCancelled struct{ error } func (errCancelled) Cancelled() {} func (e errCancelled) Cause() error { return e.error } func (e errCancelled) Unwrap() error { return e.error } // Cancelled is a helper to create an error of the class with the same name from any error type func Cancelled(err error) error { if err == nil || IsCancelled(err) { return err } return errCancelled{err} } type errDeadline struct{ error } func (errDeadline) DeadlineExceeded() {} func (e errDeadline) Cause() error { return e.error } func (e errDeadline) Unwrap() error { return e.error } // Deadline is a helper to create an error of the class with the same name from any error type func Deadline(err error) error { if err == nil || IsDeadline(err) { return err } return errDeadline{err} } type errDataLoss struct{ error } func (errDataLoss) DataLoss() {} func (e errDataLoss) Cause() error { return e.error } func (e errDataLoss) Unwrap() error { return e.error } // DataLoss is a helper to create an error of the class with the same name from any error type func DataLoss(err error) error { if err == nil || IsDataLoss(err) { return err } return errDataLoss{err} } // FromContext returns the error class from the passed in context func FromContext(ctx context.Context) error { e := ctx.Err() if e == nil { return nil } if e == context.Canceled { return Cancelled(e) } if e == context.DeadlineExceeded { return Deadline(e) } return Unknown(e) } ================================================ FILE: agent/errdefs/http_helpers.go ================================================ // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. package errdefs // import "github.com/docker/docker/errdefs" import ( "net/http" "github.com/sirupsen/logrus" ) // FromStatusCode creates an errdef error, based on the provided HTTP status-code func FromStatusCode(err error, statusCode int) error { if err == nil { return err } switch statusCode { case http.StatusNotFound: err = NotFound(err) case http.StatusBadRequest: err = InvalidParameter(err) case http.StatusConflict: err = Conflict(err) case http.StatusUnauthorized: err = Unauthorized(err) case http.StatusServiceUnavailable: err = Unavailable(err) case http.StatusForbidden: err = Forbidden(err) case http.StatusNotModified: err = NotModified(err) case http.StatusNotImplemented: err = NotImplemented(err) case http.StatusInternalServerError: if !IsSystem(err) && !IsUnknown(err) && !IsDataLoss(err) && !IsDeadline(err) && !IsCancelled(err) { err = System(err) } default: logrus.WithError(err).WithFields(logrus.Fields{ "module": "api", "status_code": statusCode, }).Debug("FIXME: Got an status-code for which error does not match any expected type!!!") switch { case statusCode >= 200 && statusCode < 400: // it's a client error case statusCode >= 400 && statusCode < 500: err = InvalidParameter(err) case statusCode >= 500 && statusCode < 600: err = System(err) default: err = Unknown(err) } } return err } ================================================ FILE: agent/errdefs/is.go ================================================ // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. package errdefs // import "github.com/docker/docker/errdefs" type causer interface { Cause() error } func getImplementer(err error) error { switch e := err.(type) { case ErrNotFound, ErrInvalidParameter, ErrConflict, ErrUnauthorized, ErrUnavailable, ErrForbidden, ErrSystem, ErrNotModified, ErrNotImplemented, ErrCancelled, ErrDeadline, ErrDataLoss, ErrUnknown: return err case causer: return getImplementer(e.Cause()) default: return err } } // IsNotFound returns if the passed in error is an ErrNotFound func IsNotFound(err error) bool { _, ok := getImplementer(err).(ErrNotFound) return ok } // IsInvalidParameter returns if the passed in error is an ErrInvalidParameter func IsInvalidParameter(err error) bool { _, ok := getImplementer(err).(ErrInvalidParameter) return ok } // IsConflict returns if the passed in error is an ErrConflict func IsConflict(err error) bool { _, ok := getImplementer(err).(ErrConflict) return ok } // IsUnauthorized returns if the passed in error is an ErrUnauthorized func IsUnauthorized(err error) bool { _, ok := getImplementer(err).(ErrUnauthorized) return ok } // IsUnavailable returns if the passed in error is an ErrUnavailable func IsUnavailable(err error) bool { _, ok := getImplementer(err).(ErrUnavailable) return ok } // IsForbidden returns if the passed in error is an ErrForbidden func IsForbidden(err error) bool { _, ok := getImplementer(err).(ErrForbidden) return ok } // IsSystem returns if the passed in error is an ErrSystem func IsSystem(err error) bool { _, ok := getImplementer(err).(ErrSystem) return ok } // IsNotModified returns if the passed in error is a NotModified error func IsNotModified(err error) bool { _, ok := getImplementer(err).(ErrNotModified) return ok } // IsNotImplemented returns if the passed in error is an ErrNotImplemented func IsNotImplemented(err error) bool { _, ok := getImplementer(err).(ErrNotImplemented) return ok } // IsUnknown returns if the passed in error is an ErrUnknown func IsUnknown(err error) bool { _, ok := getImplementer(err).(ErrUnknown) return ok } // IsCancelled returns if the passed in error is an ErrCancelled func IsCancelled(err error) bool { _, ok := getImplementer(err).(ErrCancelled) return ok } // IsDeadline returns if the passed in error is an ErrDeadline func IsDeadline(err error) bool { _, ok := getImplementer(err).(ErrDeadline) return ok } // IsDataLoss returns if the passed in error is an ErrDataLoss func IsDataLoss(err error) bool { _, ok := getImplementer(err).(ErrDataLoss) return ok } ================================================ FILE: agent/pkg/app/config.go ================================================ package app import ( "github.com/urfave/cli/v2" "github.com/tensorchord/openmodelz/agent/pkg/config" ) func configFromCLI(c *cli.Context) config.Config { cfg := config.New() // server cfg.Server.Dev = c.Bool(flagDev) cfg.Server.ServerPort = c.Int(flagServerPort) cfg.Server.ReadTimeout = c.Duration(flagServerReadTimeout) cfg.Server.WriteTimeout = c.Duration(flagServerWriteTimeout) // kubernetes cfg.KubeConfig.Kubeconfig = c.String(flagKubeConfig) cfg.KubeConfig.MasterURL = c.String(flagMasterURL) cfg.KubeConfig.QPS = c.Int(flagQPS) cfg.KubeConfig.Burst = c.Int(flagBurst) cfg.KubeConfig.ResyncPeriod = c.Duration(flagResyncPeriod) // inference ingress cfg.Ingress.IngressEnabled = c.Bool(flagIngressEnabled) cfg.Ingress.Domain = c.String(flagIngressDomain) cfg.Ingress.AnyIPToDomain = c.Bool(flagIngressAnyIPToDomain) cfg.Ingress.Namespace = c.String(flagIngressNamespace) cfg.Ingress.TLSEnabled = c.Bool(flagIngressTLSEnabled) // inference cfg.Inference.LogTimeout = c.Duration(flagInferenceLogTimeout) cfg.Inference.CacheTTL = c.Duration(flagInferenceCacheTTL) // build cfg.Build.BuildEnabled = c.Bool(flagBuildEnabled) cfg.Build.BuilderImage = c.String(flagBuilderImage) cfg.Build.BuildkitdAddress = c.String(flagBuildkitdAddress) cfg.Build.BuildCtlBin = c.String(flagBuildCtlBin) cfg.Build.BuildRegistry = c.String(flagBuildRegistry) cfg.Build.BuildRegistryToken = c.String(flagBuildRegistryToken) cfg.Build.BuildImagePullSecret = c.String(flagBuildImagePullSecret) // loki cfg.Logs.Timeout = c.Duration(flagLogsTimeout) cfg.Logs.LokiURL = c.String(flagLogsLokiURL) cfg.Logs.LokiUser = c.String(flaglogsLokiUser) cfg.Logs.LokiToken = c.String(flagLogsLokiToken) // metrics cfg.Metrics.PollingInterval = c.Duration(flagMetricsPollingInterval) cfg.Metrics.ServerPort = c.Int(flagMetricsPort) cfg.Metrics.PrometheusHost = c.String(flagMetricsPrometheusHost) cfg.Metrics.PrometheusPort = c.Int(flagMetricsPrometheusPort) // modelz cloud cfg.ModelZCloud.Enabled = c.Bool(flagModelZCloudEnabled) cfg.ModelZCloud.URL = c.String(flagModelZCloudURL) cfg.ModelZCloud.AgentToken = c.String(flagModelZCloudAgentToken) cfg.ModelZCloud.HeartbeatInterval = c.Duration(flagModelZCloudAgentHeartbeatInterval) cfg.ModelZCloud.Region = c.String(flagModelZCloudRegion) cfg.ModelZCloud.UnifiedAPIKey = c.String(flagModelZCloudUnifiedAPIKey) cfg.ModelZCloud.UpstreamTimeout = c.Duration(flagModelZCloudUpstreamTimeout) cfg.ModelZCloud.MaxIdleConnections = c.Int(flagModelZCloudMaxIdleConnections) cfg.ModelZCloud.MaxIdleConnectionsPerHost = c.Int(flagModelZCloudMaxIdleConnectionsPerHost) cfg.ModelZCloud.EventEnabled = c.Bool(flagModelZCloudEventEnabled) return cfg } ================================================ FILE: agent/pkg/app/root.go ================================================ // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. package app import ( "time" "github.com/cockroachdb/errors" "github.com/gin-gonic/gin" "github.com/sirupsen/logrus" cli "github.com/urfave/cli/v2" "github.com/tensorchord/openmodelz/agent/pkg/server" "github.com/tensorchord/openmodelz/agent/pkg/version" ) const ( flagDebug = "debug" flagDev = "dev" // server flagServerPort = "server-port" flagServerReadTimeout = "server-read-timeout" flagServerWriteTimeout = "server-write-timeout" // kubernetes flagMasterURL = "master-url" flagKubeConfig = "kube-config" flagQPS = "kube-qps" flagBurst = "kube-burst" flagResyncPeriod = "kube-resync-period" // inference ingress flagIngressEnabled = "ingress-enabled" flagIngressDomain = "ingress-domain" flagIngressNamespace = "ingress-namespace" flagIngressAnyIPToDomain = "ingress-any-ip-to-domain" flagIngressTLSEnabled = "ingress-tls-enabled" // inference flagInferenceLogTimeout = "inference-log-timeout" flagInferenceCacheTTL = "inference-cache-ttl" // build flagBuildEnabled = "build-enabled" flagBuilderImage = "builder-image" flagBuildkitdAddress = "buildkitd-address" flagBuildCtlBin = "buildctl-bin" flagBuildRegistry = "build-registry" flagBuildRegistryToken = "build-registry-token" flagBuildImagePullSecret = "build-image-pull-secret" // metrics flagMetricsPollingInterval = "metrics-polling-interval" flagMetricsPort = "metrics-port" flagMetricsPrometheusHost = "metrics-prometheus-host" flagMetricsPrometheusPort = "metrics-prometheus-port" // logs flagLogsTimeout = "logs-timeout" flagLogsLokiURL = "logs-loki-url" flaglogsLokiUser = "logs-loki-user" flagLogsLokiToken = "logs-loki-token" // modelz cloud flagModelZCloudEnabled = "modelz-cloud-enabled" flagModelZCloudURL = "modelz-cloud-url" flagModelZCloudAgentToken = "modelz-cloud-agent-token" flagModelZCloudAgentHeartbeatInterval = "modelz-cloud-agent-heartbeat-interval" flagModelZCloudRegion = "modelz-cloud-region" flagModelZCloudUnifiedAPIKey = "modelz-cloud-unified-api-key" flagModelZCloudUpstreamTimeout = "modelz-cloud-upstream-timeout" flagModelZCloudMaxIdleConnections = "modelz-cloud-max-idle-connections" flagModelZCloudMaxIdleConnectionsPerHost = "modelz-cloud-max-idle-connections-per-host" flagModelZCloudEventEnabled = "modelz-cloud-event-enabled" ) type App struct { *cli.App } func New() App { internalApp := cli.NewApp() internalApp.EnableBashCompletion = true internalApp.Name = "modelz-agent" internalApp.Usage = "Cluster agent for modelz" internalApp.HideHelpCommand = true internalApp.HideVersion = false internalApp.Version = version.GetVersion().String() internalApp.Flags = []cli.Flag{ &cli.BoolFlag{ Name: flagDebug, Usage: "enable debug output in logs", }, &cli.BoolFlag{ Name: flagDev, Usage: "enable development mode", }, &cli.IntFlag{ Name: flagServerPort, Value: 8080, Usage: "port to listen on", EnvVars: []string{"MODELZ_AGENT_SERVER_PORT"}, Aliases: []string{"p"}, }, &cli.DurationFlag{ Name: flagServerReadTimeout, Usage: "maximum duration before timing out read of the request, " + "including the body", Value: 305 * time.Second, EnvVars: []string{"MODELZ_AGENT_SERVER_READ_TIMEOUT"}, Aliases: []string{"srt"}, }, &cli.DurationFlag{ Name: flagServerWriteTimeout, Usage: "maximum duration before timing out write of the response, " + "including the body", Value: 305 * time.Second, EnvVars: []string{"MODELZ_AGENT_SERVER_WRITE_TIMEOUT"}, Aliases: []string{"swt"}, }, &cli.StringFlag{ Name: flagMasterURL, Usage: "URL to master for kubernetes cluster", EnvVars: []string{"MODELZ_AGENT_MASTER_URL"}, Aliases: []string{"mu"}, }, &cli.StringFlag{ Name: flagKubeConfig, Usage: "Path to kubeconfig file. If not provided, will use in-cluster config", EnvVars: []string{"MODELZ_AGENT_KUBE_CONFIG"}, Aliases: []string{"kc"}, }, &cli.IntFlag{ Name: flagQPS, Usage: "QPS for kubernetes client", Value: 100, EnvVars: []string{"MODELZ_AGENT_KUBE_QPS"}, Aliases: []string{"kq"}, }, &cli.IntFlag{ Name: flagBurst, Value: 250, Usage: "Burst for kubernetes client", EnvVars: []string{"MODELZ_AGENT_KUBE_BURST"}, Aliases: []string{"kb"}, }, &cli.DurationFlag{ Name: flagResyncPeriod, Value: time.Hour, Usage: "Resync period for kubernetes client", EnvVars: []string{"MODELZ_AGENT_KUBE_RESYNC_PERIOD"}, Aliases: []string{"kr"}, }, &cli.BoolFlag{ Name: flagIngressEnabled, Usage: "Enable inference ingress. " + "If enabled, the agent will create ingress for each inference", Value: false, EnvVars: []string{"MODELZ_AGENT_INGRESS_ENABLED"}, Aliases: []string{"ie"}, }, &cli.StringFlag{ Name: flagIngressDomain, Usage: "Domain for inference ingress", Value: "cloud.modelz.dev", EnvVars: []string{"MODELZ_AGENT_INGRESS_DOMAIN"}, Aliases: []string{"id"}, }, &cli.StringFlag{ Name: flagIngressNamespace, Usage: "Namespace for inference ingress", Value: "default", EnvVars: []string{"MODELZ_AGENT_INGRESS_NAMESPACE"}, Aliases: []string{"in"}, }, &cli.BoolFlag{ Name: flagIngressAnyIPToDomain, Usage: "Enable any ip to domain. " + "If enabled, the agent will create ingress for each inference", Value: false, EnvVars: []string{"MODELZ_AGENT_INGRESS_ANY_IP_TO_DOMAIN"}, Aliases: []string{"iad"}, }, &cli.BoolFlag{ Name: flagIngressTLSEnabled, Usage: "Enable TLS for inference ingress. ", Value: true, EnvVars: []string{"MODELZ_AGENT_INGRESS_TLS_ENABLED"}, Aliases: []string{"it"}, }, &cli.DurationFlag{ Name: flagInferenceLogTimeout, Usage: "Timeout for inference log streaming. " + "If the inference log has not been updated in this time, " + "the connection will be closed.", Value: time.Minute, EnvVars: []string{"MODELZ_AGENT_INFERENCE_LOG_TIMEOUT"}, Aliases: []string{"ilt"}, }, &cli.DurationFlag{ Name: flagInferenceCacheTTL, Usage: "Time to live for inference cache. ", Value: time.Millisecond * 500, EnvVars: []string{"MODELZ_AGENT_INFERENCE_CACHE_TTL"}, Aliases: []string{"ict"}, }, &cli.BoolFlag{ Name: flagBuildEnabled, Hidden: true, Usage: "Enable model build. " + "If enabled, the agent will build inference server image", Value: false, EnvVars: []string{"MODELZ_AGENT_BUILD_ENABLED"}, Aliases: []string{"be"}, }, &cli.StringFlag{ Name: flagBuilderImage, Hidden: true, Usage: "Image to use for building models. " + "Must be a valid docker image reference.", EnvVars: []string{"MODELZ_AGENT_BUILDER_IMAGE"}, Aliases: []string{"bi"}, }, &cli.StringFlag{ Name: flagBuildkitdAddress, Hidden: true, Usage: "Address of buildkitd server. " + "Must be a valid tcp address.", EnvVars: []string{"MODELZ_AGENT_BUILDKITD_ADDRESS"}, Aliases: []string{"ba"}, }, &cli.StringFlag{ Name: flagBuildCtlBin, Hidden: true, Usage: "Path to buildctl binary. " + "Must be a valid path to a binary.", EnvVars: []string{"MODELZ_AGENT_BUILDCTL_BIN"}, Aliases: []string{"bb"}, }, &cli.StringFlag{ Name: flagBuildRegistry, Hidden: true, Usage: "Registry to use for building models. ", EnvVars: []string{"MODELZ_AGENT_BUILD_REGISTRY"}, Aliases: []string{"br"}, }, &cli.StringFlag{ Name: flagBuildRegistryToken, Hidden: true, Usage: "Token to use for building models. ", EnvVars: []string{"MODELZ_AGENT_BUILD_REGISTRY_TOKEN"}, Aliases: []string{"bt"}, }, &cli.StringFlag{ Name: flagBuildImagePullSecret, Hidden: true, Usage: "Image pull secret to use for building models.", EnvVars: []string{"MODELZ_AGENT_BUILD_IMAGE_PULL_SECRET"}, Aliases: []string{"bp"}, Value: "dockerhub-secret", }, &cli.DurationFlag{ Name: flagMetricsPollingInterval, Usage: "Interval to poll metrics from kubernetes", Value: time.Second * 5, EnvVars: []string{"MODELZ_AGENT_METRICS_POLLING_INTERVAL"}, Aliases: []string{"mpi"}, }, &cli.IntFlag{ Name: flagMetricsPort, Usage: "Port to expose metrics on. ", Value: 8082, EnvVars: []string{"MODELZ_AGENT_METRICS_PORT"}, Aliases: []string{"mp"}, }, &cli.StringFlag{ Name: flagMetricsPrometheusHost, Value: "localhost", Usage: "Host to expose prometheus metrics on. ", EnvVars: []string{"MODELZ_AGENT_METRICS_PROMETHEUS_HOST"}, Aliases: []string{"mph"}, }, &cli.IntFlag{ Name: flagMetricsPrometheusPort, Usage: "Port to expose prometheus metrics on. ", Value: 9090, EnvVars: []string{"MODELZ_AGENT_METRICS_PROMETHEUS_PORT"}, Aliases: []string{"mpp"}, }, &cli.DurationFlag{ Name: flagLogsTimeout, Usage: "request timeout to query the logs", Value: time.Second * 5, EnvVars: []string{"MODELZ_AGENT_LOGS_TIMEOUT"}, }, &cli.StringFlag{ Name: flagLogsLokiURL, Hidden: true, Usage: "Loki service URL", EnvVars: []string{"MODELZ_AGENT_LOGS_LOKI_URL"}, }, &cli.StringFlag{ Name: flaglogsLokiUser, Hidden: true, Usage: "Loki service auth user", EnvVars: []string{"MODELZ_AGENT_LOGS_LOKI_USER"}, }, &cli.StringFlag{ Name: flagLogsLokiToken, Hidden: true, Usage: "Loki service auth token", EnvVars: []string{"MODELZ_AGENT_LOGS_LOKI_TOKEN"}, }, &cli.BoolFlag{ Name: flagModelZCloudEnabled, Usage: "Enable modelz cloud, agent as modelz cloud agent", Value: false, EnvVars: []string{"MODELZ_AGENT_MODELZ_CLOUD_ENABLED"}, Aliases: []string{"mzc"}, }, &cli.StringFlag{ Name: flagModelZCloudURL, Usage: "Modelz cloud URL", EnvVars: []string{"MODELZ_AGENT_MODELZ_CLOUD_URL"}, Aliases: []string{"mzu"}, Value: "https://cloud.modelz.ai", }, &cli.StringFlag{ Name: flagModelZCloudAgentToken, Usage: "Modelz cloud agent token", EnvVars: []string{"MODELZ_CLOUD_AGENT_TOKEN"}, Aliases: []string{"mzt"}, }, &cli.DurationFlag{ Name: flagModelZCloudAgentHeartbeatInterval, Usage: "Modelz cloud agent heartbeat interval", EnvVars: []string{"MODELZ_CLOUD_AGENT_HEARTBEAT_INTERVAL"}, Aliases: []string{"mzh"}, Value: time.Minute * 1, }, &cli.StringFlag{ Name: flagModelZCloudRegion, Usage: "Modelz cloud agent region", EnvVars: []string{"MODELZ_CLOUD_AGENT_REGION"}, Aliases: []string{"mzr"}, Value: "us-central1", }, &cli.StringFlag{ Name: flagModelZCloudUnifiedAPIKey, Usage: "Modelz cloud agent unified api key", EnvVars: []string{"MODELZ_CLOUD_AGENT_UNIFIED_API_KEY"}, Aliases: []string{"mzua"}, }, &cli.DurationFlag{ Name: flagModelZCloudUpstreamTimeout, Usage: "upstream timeout", EnvVars: []string{"MODELZ_UPSTREAM_TIMEOUT"}, Aliases: []string{"ut"}, Value: 300 * time.Second, }, &cli.IntFlag{ Name: flagModelZCloudMaxIdleConnections, Usage: "max idle connections", EnvVars: []string{"MODELZ_MAX_IDLE_CONNECTIONS"}, Aliases: []string{"mic"}, Value: 1024, }, &cli.IntFlag{ Name: flagModelZCloudMaxIdleConnectionsPerHost, Usage: "max idle connections per host", EnvVars: []string{"MODELZ_MAX_IDLE_CONNECTIONS_PER_HOST"}, Aliases: []string{"mich"}, Value: 1024, }, &cli.BoolFlag{ Name: flagModelZCloudEventEnabled, Usage: "Enable event logging for modelz cloud.", Value: false, EnvVars: []string{"MODELZ_AGENT_MODELZ_CLOUD_EVENT_ENABLED"}, Aliases: []string{"mze"}, }, } internalApp.Action = runServer // Deal with debug flag. var debugEnabled bool internalApp.Before = func(context *cli.Context) error { debugEnabled = context.Bool(flagDebug) if debugEnabled { logrus.SetReportCaller(true) logrus.SetFormatter(&logrus.TextFormatter{FullTimestamp: true}) logrus.SetLevel(logrus.DebugLevel) gin.SetMode(gin.DebugMode) } else { logrus.SetFormatter(&logrus.JSONFormatter{}) } return nil } return App{ App: internalApp, } } func runServer(clicontext *cli.Context) error { c := configFromCLI(clicontext) if clicontext.Bool(flagDebug) { logrus.Debug("debug mode enabled") cfgString, _ := c.GetString() logrus.WithField("config", cfgString).Debug("config") } if err := c.Validate(); err != nil { if clicontext.Bool(flagDebug) { logrus.WithError(err).Error("invalid config") } else { return errors.Wrap(err, "invalid config") } } s, err := server.New(c) if err != nil { return errors.Wrap(err, "failed to create server") } return s.Run() } ================================================ FILE: agent/pkg/config/config.go ================================================ package config import ( "encoding/json" "errors" "time" ) type Config struct { Server ServerConfig `json:"server,omitempty"` KubeConfig KubeConfig `json:"kube_config,omitempty"` Ingress IngressConfig `json:"ingress,omitempty"` Inference InferenceConfig `json:"inference,omitempty"` Build BuildConfig `json:"build,omitempty"` Metrics MetricsConfig `json:"metrics,omitempty"` Logs LogsConfig `json:"logs,omitempty"` ModelZCloud ModelZCloudConfig `json:"modelz_cloud,omitempty"` } type ModelZCloudConfig struct { Enabled bool `json:"enabled,omitempty"` // URL of apiserver URL string `json:"url,omitempty"` AgentToken string `json:"agent_token,omitempty"` HeartbeatInterval time.Duration `json:"heartbeat_interval,omitempty"` ID string `json:"id,omitempty"` Name string `json:"name,omitempty"` TokenID string `json:"token_id,omitempty"` Region string `json:"region,omitempty"` APIKeys map[string]string `json:"api_keys,omitempty"` UserNamespaces []string `json:"user_namespaces,omitempty"` UnifiedAPIKey string `json:"unified_api_key,omitempty"` UpstreamTimeout time.Duration `json:"upstream_timeout,omitempty"` MaxIdleConnections int `json:"max_idle_connections,omitempty"` MaxIdleConnectionsPerHost int `json:"max_idle_connections_per_host,omitempty"` EventEnabled bool `json:"event_enabled,omitempty"` } type LogsConfig struct { Timeout time.Duration `json:"timeout,omitempty"` LokiURL string `json:"loki_url,omitempty"` LokiUser string `json:"loki_user,omitempty"` LokiToken string `json:"loki_token,omitempty"` } type ServerConfig struct { Dev bool `json:"dev,omitempty"` ServerPort int `json:"server_port,omitempty"` ReadTimeout time.Duration `json:"read_timeout,omitempty"` WriteTimeout time.Duration `json:"write_timeout,omitempty"` } type MetricsConfig struct { PollingInterval time.Duration `json:"polling_interval,omitempty"` ServerPort int `json:"server_port,omitempty"` PrometheusPort int `json:"prometheus_port,omitempty"` PrometheusHost string `json:"prometheus_host,omitempty"` } type BuildConfig struct { BuildEnabled bool `json:"build_enabled,omitempty"` BuilderImage string `json:"builder_image,omitempty"` BuildkitdAddress string `json:"buildkitd_address,omitempty"` BuildCtlBin string `json:"build_ctl_bin,omitempty"` BuildRegistry string `json:"build_registry,omitempty"` BuildRegistryToken string `json:"build_registry_token,omitempty"` BuildImagePullSecret string `json:"build_image_pull_secret,omitempty"` } type InferenceConfig struct { LogTimeout time.Duration `json:"log_timeout,omitempty"` CacheTTL time.Duration `json:"cache_ttl,omitempty"` } type IngressConfig struct { IngressEnabled bool `json:"ingress_enabled,omitempty"` Domain string `json:"domain,omitempty"` Namespace string `json:"namespace,omitempty"` AnyIPToDomain bool `json:"any_ip_to_domain,omitempty"` TLSEnabled bool `json:"tls_enabled,omitempty"` } type KubeConfig struct { Kubeconfig string `json:"kubeconfig,omitempty"` MasterURL string `json:"master_url,omitempty"` QPS int `json:"qps,omitempty"` Burst int `json:"burst,omitempty"` ResyncPeriod time.Duration `json:"resync_period,omitempty"` } func New() Config { return Config{ KubeConfig: KubeConfig{}, Ingress: IngressConfig{}, Inference: InferenceConfig{}, Build: BuildConfig{}, Metrics: MetricsConfig{}, Logs: LogsConfig{}, } } func (c Config) GetString() (string, error) { bytes, err := json.Marshal(c) return string(bytes), err } func (c Config) Validate() error { if c.Server.ServerPort == 0 || c.Server.ReadTimeout == 0 || c.Server.WriteTimeout == 0 { return errors.New("server config is required") } if c.Inference.LogTimeout == 0 { return errors.New("inference log timeout is required") } if c.Build.BuildEnabled { if c.Build.BuildkitdAddress == "" || c.Build.BuilderImage == "" || c.Build.BuildRegistryToken == "" || c.Build.BuildRegistry == "" || c.Build.BuildCtlBin == "" || c.Build.BuildImagePullSecret == "" { return errors.New("build config is required") } } if c.Metrics.ServerPort == 0 || c.Metrics.PollingInterval == 0 || c.Metrics.PrometheusHost == "" || c.Metrics.PrometheusPort == 0 { return errors.New("metrics config is required") } if c.Ingress.IngressEnabled { if c.Ingress.Namespace == "" { return errors.New("ingress namespace is required") } if !c.Ingress.AnyIPToDomain && c.Ingress.Domain == "" { return errors.New("ingress domain is required") } } if c.ModelZCloud.Enabled { if c.ModelZCloud.URL == "" || c.ModelZCloud.AgentToken == "" || c.ModelZCloud.HeartbeatInterval == 0 { return errors.New("modelz cloud config is required") } } return nil } ================================================ FILE: agent/pkg/consts/consts.go ================================================ package consts import "time" const ( Domain = "modelz.live" DefaultPrefix = "modelz-" APIKEY_PREFIX = "mzi-" ) const DefaultAPIServerReadyTimeout = 15 * time.Minute ================================================ FILE: agent/pkg/docs/docs.go ================================================ // Package docs GENERATED BY SWAG; DO NOT EDIT // This file was generated by swaggo/swag package docs import "github.com/swaggo/swag" const docTemplate = `{ "schemes": {{ marshal .Schemes }}, "swagger": "2.0", "info": { "description": "{{escape .Description}}", "title": "{{.Title}}", "contact": { "name": "modelz support", "url": "https://github.com/tensorchord/openmodelz", "email": "modelz-support@tensorchord.ai" }, "version": "{{.Version}}" }, "host": "{{.Host}}", "basePath": "{{.BasePath}}", "paths": { "/gradio/{id}": { "get": { "description": "Reverse proxy to the backend gradio.", "consumes": [ "*/*" ], "produces": [ "application/json" ], "tags": [ "inference" ], "summary": "Reverse proxy to the backend gradio.", "parameters": [ { "type": "string", "description": "Deployment ID", "name": "id", "in": "path", "required": true } ], "responses": { "201": { "description": "Created" } } }, "post": { "description": "Reverse proxy to the backend gradio.", "consumes": [ "*/*" ], "produces": [ "application/json" ], "tags": [ "inference" ], "summary": "Reverse proxy to the backend gradio.", "parameters": [ { "type": "string", "description": "Deployment ID", "name": "id", "in": "path", "required": true } ], "responses": { "201": { "description": "Created" } } } }, "/healthz": { "get": { "description": "Healthz", "consumes": [ "application/json" ], "produces": [ "application/json" ], "tags": [ "system" ], "summary": "Healthz", "responses": { "200": { "description": "OK" } } } }, "/inference/{name}": { "get": { "description": "Inference proxy.", "consumes": [ "application/json" ], "produces": [ "application/json" ], "tags": [ "inference-proxy" ], "summary": "Inference.", "parameters": [ { "type": "string", "description": "inference id", "name": "name", "in": "path", "required": true } ], "responses": { "200": { "description": "OK" }, "303": { "description": "See Other" }, "400": { "description": "Bad Request" }, "404": { "description": "Not Found" }, "500": { "description": "Internal Server Error" } } }, "put": { "description": "Inference proxy.", "consumes": [ "application/json" ], "produces": [ "application/json" ], "tags": [ "inference-proxy" ], "summary": "Inference.", "parameters": [ { "type": "string", "description": "inference id", "name": "name", "in": "path", "required": true } ], "responses": { "200": { "description": "OK" }, "303": { "description": "See Other" }, "400": { "description": "Bad Request" }, "404": { "description": "Not Found" }, "500": { "description": "Internal Server Error" } } }, "post": { "description": "Inference proxy.", "consumes": [ "application/json" ], "produces": [ "application/json" ], "tags": [ "inference-proxy" ], "summary": "Inference.", "parameters": [ { "type": "string", "description": "inference id", "name": "name", "in": "path", "required": true } ], "responses": { "200": { "description": "OK" }, "303": { "description": "See Other" }, "400": { "description": "Bad Request" }, "404": { "description": "Not Found" }, "500": { "description": "Internal Server Error" } } }, "delete": { "description": "Inference proxy.", "consumes": [ "application/json" ], "produces": [ "application/json" ], "tags": [ "inference-proxy" ], "summary": "Inference.", "parameters": [ { "type": "string", "description": "inference id", "name": "name", "in": "path", "required": true } ], "responses": { "200": { "description": "OK" }, "303": { "description": "See Other" }, "400": { "description": "Bad Request" }, "404": { "description": "Not Found" }, "500": { "description": "Internal Server Error" } } } }, "/mosec/{id}": { "get": { "description": "Proxy to the backend mosec.", "consumes": [ "*/*" ], "produces": [ "application/json" ], "tags": [ "inference" ], "summary": "Proxy to the backend mosec.", "parameters": [ { "type": "string", "description": "Deployment ID", "name": "id", "in": "path", "required": true } ], "responses": { "201": { "description": "Created" } } } }, "/mosec/{id}/inference": { "post": { "description": "Proxy to the backend mosec.", "consumes": [ "*/*" ], "produces": [ "application/json" ], "tags": [ "inference" ], "summary": "Proxy to the backend mosec.", "parameters": [ { "type": "string", "description": "Deployment ID", "name": "id", "in": "path", "required": true } ], "responses": { "201": { "description": "Created" } } } }, "/mosec/{id}/metrics": { "get": { "description": "Proxy to the backend mosec.", "consumes": [ "*/*" ], "produces": [ "application/json" ], "tags": [ "inference" ], "summary": "Proxy to the backend mosec.", "parameters": [ { "type": "string", "description": "Deployment ID", "name": "id", "in": "path", "required": true } ], "responses": { "201": { "description": "Created" } } } }, "/other/{id}": { "get": { "description": "Reverse proxy to the backend other.", "consumes": [ "*/*" ], "produces": [ "application/json" ], "tags": [ "inference" ], "summary": "Reverse proxy to the backend other.", "parameters": [ { "type": "string", "description": "Deployment ID", "name": "id", "in": "path", "required": true } ], "responses": { "201": { "description": "Created" } } }, "post": { "description": "Reverse proxy to the backend other.", "consumes": [ "*/*" ], "produces": [ "application/json" ], "tags": [ "inference" ], "summary": "Reverse proxy to the backend other.", "parameters": [ { "type": "string", "description": "Deployment ID", "name": "id", "in": "path", "required": true } ], "responses": { "201": { "description": "Created" } } } }, "/streamlit/{id}": { "get": { "description": "Reverse proxy to streamlit.", "consumes": [ "*/*" ], "produces": [ "application/json" ], "tags": [ "inference" ], "summary": "Reverse proxy to streamlit.", "parameters": [ { "type": "string", "description": "Deployment ID", "name": "id", "in": "path", "required": true } ], "responses": { "201": { "description": "Created" } } }, "post": { "description": "Reverse proxy to streamlit.", "consumes": [ "*/*" ], "produces": [ "application/json" ], "tags": [ "inference" ], "summary": "Reverse proxy to streamlit.", "parameters": [ { "type": "string", "description": "Deployment ID", "name": "id", "in": "path", "required": true } ], "responses": { "201": { "description": "Created" } } } }, "/system/build": { "get": { "description": "List the builds.", "consumes": [ "application/json" ], "produces": [ "application/json" ], "tags": [ "build" ], "summary": "List the builds.", "parameters": [ { "type": "string", "description": "Namespace", "name": "namespace", "in": "query", "required": true } ], "responses": { "200": { "description": "OK", "schema": { "type": "array", "items": { "$ref": "#/definitions/types.Build" } } } } }, "post": { "description": "Create the build.", "consumes": [ "application/json" ], "produces": [ "application/json" ], "tags": [ "build" ], "summary": "Create the build.", "parameters": [ { "description": "build", "name": "body", "in": "body", "required": true, "schema": { "$ref": "#/definitions/types.Build" } } ], "responses": { "200": { "description": "OK", "schema": { "$ref": "#/definitions/types.Build" } } } } }, "/system/build/{name}": { "get": { "description": "Get the build by name.", "consumes": [ "application/json" ], "produces": [ "application/json" ], "tags": [ "build" ], "summary": "Get the build by name.", "parameters": [ { "type": "string", "description": "Namespace", "name": "namespace", "in": "query", "required": true }, { "type": "string", "description": "inference id", "name": "name", "in": "path", "required": true } ], "responses": { "200": { "description": "OK", "schema": { "$ref": "#/definitions/types.Build" } } } } }, "/system/image-cache": { "post": { "description": "Create the image cache.", "consumes": [ "application/json" ], "produces": [ "application/json" ], "tags": [ "image-cache" ], "summary": "Create the image cache.", "parameters": [ { "description": "image-cache", "name": "body", "in": "body", "required": true, "schema": { "$ref": "#/definitions/types.ImageCache" } } ], "responses": { "201": { "description": "Created", "schema": { "$ref": "#/definitions/types.ImageCache" } } } } }, "/system/inference/{name}": { "get": { "description": "Get the inference by name.", "consumes": [ "application/json" ], "produces": [ "application/json" ], "tags": [ "inference" ], "summary": "Get the inference by name.", "parameters": [ { "type": "string", "description": "Namespace", "name": "namespace", "in": "query", "required": true }, { "type": "string", "description": "inference id", "name": "name", "in": "path", "required": true } ], "responses": { "200": { "description": "OK", "schema": { "$ref": "#/definitions/types.InferenceDeployment" } } } } }, "/system/inference/{name}/instance/{instance}": { "post": { "description": "Attach to the inference instance.", "consumes": [ "application/json" ], "produces": [ "application/json" ], "tags": [ "inference" ], "summary": "Attach to the inference instance.", "parameters": [ { "type": "string", "description": "Namespace", "name": "namespace", "in": "query", "required": true }, { "type": "string", "description": "Name", "name": "name", "in": "path", "required": true }, { "type": "string", "description": "Instance name", "name": "instance", "in": "path", "required": true } ], "responses": { "200": { "description": "OK", "schema": { "type": "array", "items": { "$ref": "#/definitions/types.InferenceDeployment" } } } } } }, "/system/inference/{name}/instances": { "get": { "description": "List the inference instances.", "consumes": [ "application/json" ], "produces": [ "application/json" ], "tags": [ "inference" ], "summary": "List the inference instances.", "parameters": [ { "type": "string", "description": "Namespace", "name": "namespace", "in": "query", "required": true }, { "type": "string", "description": "Name", "name": "name", "in": "path", "required": true } ], "responses": { "200": { "description": "OK", "schema": { "type": "array", "items": { "$ref": "#/definitions/types.InferenceDeployment" } } } } } }, "/system/inferences": { "get": { "description": "List the inferences.", "consumes": [ "application/json" ], "produces": [ "application/json" ], "tags": [ "inference" ], "summary": "List the inferences.", "parameters": [ { "type": "string", "description": "Namespace", "name": "namespace", "in": "query", "required": true } ], "responses": { "200": { "description": "OK", "schema": { "type": "array", "items": { "$ref": "#/definitions/types.InferenceDeployment" } } } } }, "put": { "description": "Update the inferences.", "consumes": [ "application/json" ], "produces": [ "application/json" ], "tags": [ "inference" ], "summary": "Update the inferences.", "parameters": [ { "description": "query params", "name": "request", "in": "body", "required": true, "schema": { "$ref": "#/definitions/types.InferenceDeployment" } }, { "type": "string", "description": "Namespace", "name": "namespace", "in": "query", "required": true } ], "responses": { "202": { "description": "Accepted", "schema": { "$ref": "#/definitions/types.InferenceDeployment" } } } }, "post": { "description": "Create the inferences.", "consumes": [ "application/json" ], "produces": [ "application/json" ], "tags": [ "inference" ], "summary": "Create the inferences.", "parameters": [ { "description": "query params", "name": "request", "in": "body", "required": true, "schema": { "$ref": "#/definitions/types.InferenceDeployment" } } ], "responses": { "201": { "description": "Created", "schema": { "$ref": "#/definitions/types.InferenceDeployment" } } } }, "delete": { "description": "Delete the inferences.", "consumes": [ "application/json" ], "produces": [ "application/json" ], "tags": [ "inference" ], "summary": "Delete the inferences.", "parameters": [ { "description": "query params", "name": "request", "in": "body", "required": true, "schema": { "$ref": "#/definitions/types.DeleteFunctionRequest" } }, { "type": "string", "description": "Namespace", "name": "namespace", "in": "query", "required": true } ], "responses": { "202": { "description": "Accepted", "schema": { "$ref": "#/definitions/types.DeleteFunctionRequest" } } } } }, "/system/info": { "get": { "description": "Get system info.", "consumes": [ "application/json" ], "produces": [ "application/json" ], "tags": [ "system" ], "summary": "Get system info.", "responses": { "200": { "description": "OK", "schema": { "$ref": "#/definitions/types.ProviderInfo" } } } } }, "/system/logs/build": { "get": { "description": "Get the build logs.", "consumes": [ "application/json" ], "produces": [ "application/json" ], "tags": [ "log" ], "summary": "Get the build logs.", "parameters": [ { "type": "string", "description": "Namespace", "name": "namespace", "in": "query", "required": true }, { "type": "string", "description": "Build Name", "name": "name", "in": "query", "required": true }, { "type": "string", "description": "Instance", "name": "instance", "in": "query" }, { "type": "integer", "description": "Tail", "name": "tail", "in": "query" }, { "type": "boolean", "description": "Follow", "name": "follow", "in": "query" }, { "type": "string", "description": "Since", "name": "since", "in": "query" } ], "responses": { "200": { "description": "OK", "schema": { "type": "array", "items": { "$ref": "#/definitions/types.Message" } } } } } }, "/system/logs/inference": { "get": { "description": "Get the inference logs.", "consumes": [ "application/json" ], "produces": [ "application/json" ], "tags": [ "log" ], "summary": "Get the inference logs.", "parameters": [ { "type": "string", "description": "Namespace", "name": "namespace", "in": "query", "required": true }, { "type": "string", "description": "Name", "name": "name", "in": "query", "required": true }, { "type": "string", "description": "Instance", "name": "instance", "in": "query" }, { "type": "integer", "description": "Tail", "name": "tail", "in": "query" }, { "type": "boolean", "description": "Follow", "name": "follow", "in": "query" }, { "type": "string", "description": "Since", "name": "since", "in": "query" }, { "type": "string", "description": "End", "name": "end", "in": "query" } ], "responses": { "200": { "description": "OK", "schema": { "type": "array", "items": { "$ref": "#/definitions/types.Message" } } } } } }, "/system/namespaces": { "get": { "description": "List the namespaces.", "consumes": [ "application/json" ], "produces": [ "application/json" ], "tags": [ "namespace" ], "summary": "List the namespaces.", "responses": { "200": { "description": "OK", "schema": { "type": "array", "items": { "type": "string" } } } } }, "post": { "description": "Create the namespace.", "consumes": [ "application/json" ], "produces": [ "application/json" ], "tags": [ "namespace" ], "summary": "Create the namespace.", "parameters": [ { "description": "Namespace name", "name": "body", "in": "body", "required": true, "schema": { "$ref": "#/definitions/types.NamespaceRequest" } } ], "responses": { "200": { "description": "OK", "schema": { "$ref": "#/definitions/types.NamespaceRequest" } } } }, "delete": { "description": "Delete the namespace.", "consumes": [ "application/json" ], "produces": [ "application/json" ], "tags": [ "namespace" ], "summary": "Delete the namespace.", "parameters": [ { "description": "Namespace name", "name": "body", "in": "body", "required": true, "schema": { "$ref": "#/definitions/types.NamespaceRequest" } } ], "responses": { "200": { "description": "OK", "schema": { "$ref": "#/definitions/types.NamespaceRequest" } } } } }, "/system/scale-inference": { "post": { "description": "Scale the inferences.", "consumes": [ "application/json" ], "produces": [ "application/json" ], "tags": [ "inference" ], "summary": "Scale the inferences.", "parameters": [ { "type": "string", "description": "Namespace", "name": "namespace", "in": "query", "required": true }, { "description": "query params", "name": "request", "in": "body", "required": true, "schema": { "$ref": "#/definitions/types.ScaleServiceRequest" } } ], "responses": { "202": { "description": "Accepted", "schema": { "type": "array", "items": { "$ref": "#/definitions/types.ScaleServiceRequest" } } }, "400": { "description": "Bad Request" } } } }, "/system/server/{name}/delete": { "delete": { "description": "Delete a node.", "consumes": [ "application/json" ], "produces": [ "application/json" ], "tags": [ "namespace" ], "summary": "Delete a node from the cluster.", "parameters": [ { "type": "string", "description": "Server Name", "name": "name", "in": "path", "required": true } ], "responses": { "200": { "description": "OK" } } } }, "/system/server/{name}/labels": { "post": { "description": "List the servers.", "consumes": [ "application/json" ], "produces": [ "application/json" ], "tags": [ "namespace" ], "summary": "List the servers.", "parameters": [ { "type": "string", "description": "Server Name", "name": "name", "in": "path", "required": true }, { "description": "query params", "name": "request", "in": "body", "required": true, "schema": { "$ref": "#/definitions/types.ServerSpec" } } ], "responses": { "200": { "description": "OK", "schema": { "type": "array", "items": { "type": "string" } } } } } }, "/system/servers": { "get": { "description": "List the servers.", "consumes": [ "application/json" ], "produces": [ "application/json" ], "tags": [ "namespace" ], "summary": "List the servers.", "responses": { "200": { "description": "OK", "schema": { "type": "array", "items": { "$ref": "#/definitions/types.Server" } } } } } } }, "definitions": { "types.AuthN": { "type": "object", "properties": { "password": { "type": "string" }, "token": { "type": "string" }, "username": { "type": "string" } } }, "types.Build": { "type": "object", "properties": { "spec": { "$ref": "#/definitions/types.BuildSpec" }, "status": { "$ref": "#/definitions/types.BuildStatus" } } }, "types.BuildSpec": { "type": "object", "properties": { "authn": { "$ref": "#/definitions/types.AuthN" }, "branch": { "type": "string" }, "buildTarget": { "$ref": "#/definitions/types.BuildTarget" }, "image": { "type": "string" }, "image_tag": { "type": "string" }, "name": { "type": "string" }, "namespace": { "type": "string" }, "repository": { "description": "repository is the URL", "type": "string" }, "revision": { "description": "revision is the commit hash for the specified revision.\n+optional", "type": "string" }, "secret_id": { "type": "string" } } }, "types.BuildStatus": { "type": "object", "properties": { "phase": { "type": "string" } } }, "types.BuildTarget": { "type": "object", "properties": { "builder": { "type": "string" }, "digest": { "type": "string" }, "directory": { "description": "directory is the target directory name.\nMust not contain or start with '..'. If '.' is supplied, the volume directory will be the\ngit repository. Otherwise, if specified, the volume will contain the git repository in\nthe subdirectory with the given name.\n+optional", "type": "string" }, "duration": { "type": "string" }, "image": { "type": "string" }, "image_tag": { "type": "string" }, "registry": { "type": "string" }, "registry_token": { "type": "string" } } }, "types.DeleteFunctionRequest": { "type": "object", "properties": { "functionName": { "type": "string" } } }, "types.ImageCache": { "type": "object", "properties": { "force_full_cache": { "type": "boolean" }, "image": { "type": "string" }, "name": { "description": "Name is the name of the inference.", "type": "string" }, "namespace": { "type": "string" }, "node_selector": { "type": "string" } } }, "types.InferenceDeployment": { "type": "object", "properties": { "spec": { "$ref": "#/definitions/types.InferenceDeploymentSpec" }, "status": { "$ref": "#/definitions/types.InferenceDeploymentStatus" } } }, "types.InferenceDeploymentSpec": { "type": "object", "properties": { "annotations": { "description": "Annotations are key-value pairs that may be attached to the inference.", "type": "object", "additionalProperties": { "type": "string" } }, "command": { "description": "Command to run when starting the", "type": "string" }, "constraints": { "description": "Constraints are the constraints for the inference.", "type": "array", "items": { "type": "string" } }, "envVars": { "description": "EnvVars can be provided to set environment variables for the inference runtime.", "type": "object", "additionalProperties": { "type": "string" } }, "framework": { "description": "Framework is the inference framework.", "type": "string" }, "http_probe_path": { "description": "HTTPProbePath is the path of the http probe.", "type": "string" }, "image": { "description": "Image is a fully-qualified container image", "type": "string" }, "labels": { "description": "Labels are key-value pairs that may be attached to the inference.", "type": "object", "additionalProperties": { "type": "string" } }, "name": { "description": "Name is the name of the inference.", "type": "string" }, "namespace": { "description": "Namespace for the inference.", "type": "string" }, "port": { "description": "Port is the port exposed by the inference.", "type": "integer" }, "resources": { "description": "Resources are the compute resource requirements.", "$ref": "#/definitions/types.ResourceRequirements" }, "scaling": { "description": "Scaling is the scaling configuration for the inference.", "$ref": "#/definitions/types.ScalingConfig" }, "secrets": { "description": "Secrets list of secrets to be made available to inference.", "type": "array", "items": { "type": "string" } } } }, "types.InferenceDeploymentStatus": { "type": "object", "properties": { "availableReplicas": { "description": "AvailableReplicas is the count of replicas ready to receive\ninvocations as reported by the faas-provider", "type": "integer" }, "createdAt": { "description": "CreatedAt is the time read back from the faas backend's\ndata store for when the function or its container was created.", "type": "string" }, "eventMessage": { "description": "EventMessage record human readable message indicating details about the event of deployment.", "type": "string" }, "invocationCount": { "description": "InvocationCount count of invocations", "type": "integer" }, "phase": { "type": "string" }, "replicas": { "description": "Replicas desired within the cluster", "type": "integer" }, "usage": { "description": "Usage represents CPU and RAM used by all of the\nfunctions' replicas. Divide by AvailableReplicas for an\naverage value per replica.", "$ref": "#/definitions/types.InferenceUsage" } } }, "types.InferenceUsage": { "type": "object", "properties": { "cpu": { "description": "CPU is the increase in CPU usage since the last measurement\nequivalent to Kubernetes' concept of millicores.", "type": "number" }, "gpu": { "type": "number" }, "totalMemoryBytes": { "description": "TotalMemoryBytes is the total memory usage in bytes.", "type": "number" } } }, "types.Message": { "type": "object", "properties": { "instance": { "description": "instance is the name/id of the specific function instance", "type": "string" }, "name": { "description": "Name is the function name", "type": "string" }, "namespace": { "type": "string" }, "text": { "description": "Text is the raw log message content", "type": "string" }, "timestamp": { "description": "Timestamp is the timestamp of when the log message was recorded", "type": "string" } } }, "types.NamespaceRequest": { "type": "object", "properties": { "name": { "type": "string" } } }, "types.NodeSystemInfo": { "type": "object", "properties": { "architecture": { "description": "The Architecture reported by the node", "type": "string" }, "kernelVersion": { "description": "Kernel Version reported by the node from 'uname -r' (e.g. 3.16.0-0.bpo.4-amd64).", "type": "string" }, "machineID": { "description": "MachineID reported by the node. For unique machine identification\nin the cluster this field is preferred. Learn more from man(5)\nmachine-id: http://man7.org/linux/man-pages/man5/machine-id.5.html", "type": "string" }, "operatingSystem": { "description": "The Operating System reported by the node", "type": "string" }, "osImage": { "description": "OS Image reported by the node from /etc/os-release (e.g. Debian GNU/Linux 7 (wheezy)).", "type": "string" } } }, "types.ProviderInfo": { "type": "object", "properties": { "orchestration": { "type": "string" }, "provider": { "type": "string" }, "version": { "$ref": "#/definitions/types.VersionInfo" } } }, "types.ResourceList": { "type": "object", "additionalProperties": { "type": "string" } }, "types.ResourceRequirements": { "type": "object", "properties": { "limits": { "description": "Limits describes the maximum amount of compute resources allowed.\nMore info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/\n+optional", "$ref": "#/definitions/types.ResourceList" }, "requests": { "description": "Requests describes the minimum amount of compute resources required.\nIf Requests is omitted for a container, it defaults to Limits if that is explicitly specified,\notherwise to an implementation-defined value.\nMore info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/\n+optional", "$ref": "#/definitions/types.ResourceList" } } }, "types.ScaleServiceRequest": { "type": "object", "properties": { "attempt": { "type": "integer" }, "eventMessage": { "type": "string" }, "replicas": { "type": "integer" }, "serviceName": { "type": "string" } } }, "types.ScalingConfig": { "type": "object", "properties": { "max_replicas": { "description": "MaxReplicas is the upper limit for the number of replicas to which the\nautoscaler can scale up. It cannot be less that minReplicas. It defaults\nto 1.", "type": "integer" }, "min_replicas": { "description": "MinReplicas is the lower limit for the number of replicas to which the\nautoscaler can scale down. It defaults to 0.", "type": "integer" }, "startup_duration": { "description": "StartupDuration is the duration (in seconds) of startup time.", "type": "integer" }, "target_load": { "description": "TargetLoad is the target load. In capacity mode, it is the expected number of the inflight requests per replica.", "type": "integer" }, "type": { "description": "Type is the scaling type. It can be either \"capacity\" or \"rps\". Default is \"capacity\".", "type": "string" }, "zero_duration": { "description": "ZeroDuration is the duration (in seconds) of zero load before scaling down to zero. Default is 5 minutes.", "type": "integer" } } }, "types.Server": { "type": "object", "properties": { "spec": { "$ref": "#/definitions/types.ServerSpec" }, "status": { "$ref": "#/definitions/types.ServerStatus" } } }, "types.ServerSpec": { "type": "object", "properties": { "labels": { "type": "object", "additionalProperties": { "type": "string" } }, "name": { "type": "string" } } }, "types.ServerStatus": { "type": "object", "properties": { "allocatable": { "$ref": "#/definitions/types.ResourceList" }, "capacity": { "$ref": "#/definitions/types.ResourceList" }, "phase": { "type": "string" }, "system": { "$ref": "#/definitions/types.NodeSystemInfo" } } }, "types.VersionInfo": { "type": "object", "properties": { "build_date": { "type": "string" }, "compiler": { "type": "string" }, "git_commit": { "type": "string" }, "git_tag": { "type": "string" }, "git_tree_state": { "type": "string" }, "go_version": { "type": "string" }, "platform": { "type": "string" }, "version": { "type": "string" } } } } }` // SwaggerInfo holds exported Swagger Info so clients can modify it var SwaggerInfo = &swag.Spec{ Version: "v0.0.23", Host: "localhost:8081", BasePath: "/", Schemes: []string{"http"}, Title: "modelz cluster agent", Description: "modelz kubernetes cluster agent", InfoInstanceName: "swagger", SwaggerTemplate: docTemplate, } func init() { swag.Register(SwaggerInfo.InstanceName(), SwaggerInfo) } ================================================ FILE: agent/pkg/event/event.go ================================================ package event import ( "context" "fmt" "github.com/sirupsen/logrus" "github.com/tensorchord/openmodelz/agent/api/types" "github.com/tensorchord/openmodelz/agent/client" ) type Interface interface { CreateDeploymentEvent(namespace, deployment, event, message string) error } type EventRecorder struct { Client *client.Client AgentToken string } func NewEventRecorder(client *client.Client, token string) Interface { return &EventRecorder{ Client: client, AgentToken: token, } } func (e *EventRecorder) CreateDeploymentEvent(namespace, deployment, event, message string) error { user, err := client.GetUserIDFromNamespace(namespace) if err != nil { return err } else if user == "" { return fmt.Errorf("user id is empty") } deploymentEvent := types.DeploymentEvent{ UserID: user, DeploymentID: deployment, EventType: event, Message: message, } err = e.Client.CreateDeploymentEvent(context.TODO(), e.AgentToken, deploymentEvent) if err != nil { logrus.Errorf("failed to create deployment event: %v", err) return err } return nil } ================================================ FILE: agent/pkg/event/fake.go ================================================ package event type Fake struct { } func NewFake() Interface { return &Fake{} } func (f *Fake) CreateDeploymentEvent(namespace, deployment, event, message string) error { return nil } ================================================ FILE: agent/pkg/event/suite_test.go ================================================ package event import ( "testing" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) func TestBuilder(t *testing.T) { RegisterFailHandler(Fail) RunSpecs(t, "event") } ================================================ FILE: agent/pkg/event/username.go ================================================ package event import "fmt" const ( DefaultPrefix = "modelz-" ) func getUserIDFromNamespace(ns string) (string, error) { if len(ns) < 8 { return "", fmt.Errorf("namespace too short") } if ns[:len(DefaultPrefix)] != DefaultPrefix { return "", fmt.Errorf("namespace does not start with %s", DefaultPrefix) } return ns[len(DefaultPrefix):], nil } ================================================ FILE: agent/pkg/event/util.go ================================================ package event import ( "database/sql" ) func NullStringBuilder(String string, Valid bool) sql.NullString { return sql.NullString{String: String, Valid: Valid} } ================================================ FILE: agent/pkg/k8s/convert_inference.go ================================================ package k8s import ( "github.com/tensorchord/openmodelz/agent/api/types" "github.com/tensorchord/openmodelz/modelzetes/pkg/apis/modelzetes/v2alpha1" "github.com/tensorchord/openmodelz/modelzetes/pkg/consts" appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" ) func AsInferenceDeployment(inf *v2alpha1.Inference, item *appsv1.Deployment) *types.InferenceDeployment { if inf == nil { return nil } res := &types.InferenceDeployment{ Spec: types.InferenceDeploymentSpec{ Name: inf.Name, Framework: types.Framework(inf.Spec.Framework), Image: inf.Spec.Image, Namespace: inf.Namespace, EnvVars: inf.Spec.EnvVars, Secrets: inf.Spec.Secrets, Constraints: inf.Spec.Constraints, Labels: inf.Spec.Labels, Annotations: inf.Spec.Annotations, }, Status: types.InferenceDeploymentStatus{ Phase: types.PhaseNoReplicas, }, } if inf.Spec.Scaling != nil { res.Spec.Scaling = &types.ScalingConfig{ MinReplicas: inf.Spec.Scaling.MinReplicas, MaxReplicas: inf.Spec.Scaling.MaxReplicas, TargetLoad: inf.Spec.Scaling.TargetLoad, ZeroDuration: inf.Spec.Scaling.ZeroDuration, StartupDuration: inf.Spec.Scaling.StartupDuration, } if inf.Spec.Scaling.Type != nil { typ := types.ScalingType(*inf.Spec.Scaling.Type) res.Spec.Scaling.Type = &typ } } if inf.Spec.Port != nil { res.Spec.Port = inf.Spec.Port } var replicas int32 = 0 // Get status according to the deployment. if item != nil { if item.Spec.Replicas != nil { replicas = *item.Spec.Replicas } res.Status.Replicas = replicas res.Status.CreatedAt = &item.CreationTimestamp.Time res.Status.InvocationCount = 0 res.Status.AvailableReplicas = item.Status.AvailableReplicas res.Status.Phase = AsStatusPhase(item) } return res } func AsResourceList(resources v1.ResourceList) types.ResourceList { res := types.ResourceList{} gpuResource := resources[consts.ResourceNvidiaGPU] gpuPtr := &gpuResource if !resources.Cpu().IsZero() { res[types.ResourceCPU] = types.Quantity( resources.Cpu().String()) } if !resources.Memory().IsZero() { res[types.ResourceMemory] = types.Quantity( resources.Memory().String()) } if !gpuPtr.IsZero() { res[types.ResourceGPU] = types.Quantity( gpuPtr.String()) } return res } func AsStatusPhase(item *appsv1.Deployment) types.Phase { phase := types.PhaseNotReady for _, c := range item.Status.Conditions { if c.Type == appsv1.DeploymentAvailable && c.Status == v1.ConditionTrue { phase = types.PhaseReady } else if c.Type == appsv1.DeploymentProgressing && c.Status == v1.ConditionFalse { phase = types.PhaseScaling } } if item.Spec.Replicas != nil && *item.Spec.Replicas == 0 { phase = types.PhaseNoReplicas } if item.DeletionTimestamp != nil { phase = types.PhaseTerminating } return phase } ================================================ FILE: agent/pkg/k8s/convert_inference_test.go ================================================ package k8s import ( "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/tensorchord/openmodelz/agent/api/types" "github.com/tensorchord/openmodelz/modelzetes/pkg/apis/modelzetes/v2alpha1" "github.com/tensorchord/openmodelz/modelzetes/pkg/consts" . "github.com/tensorchord/openmodelz/modelzetes/pkg/pointer" appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" v1types "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) var _ = Describe("agent/pkg/k8s/convert_inference", func() { It("function AsResourceList", func() { tcs := []struct { resource v1.ResourceList expect types.ResourceList }{ { resource: map[v1types.ResourceName]resource.Quantity{ v1types.ResourceCPU: resource.MustParse("0"), v1types.ResourceMemory: resource.MustParse("0"), consts.ResourceNvidiaGPU: resource.MustParse("0"), }, expect: types.ResourceList{}, }, { resource: map[v1types.ResourceName]resource.Quantity{ v1types.ResourceCPU: resource.MustParse("0"), v1types.ResourceMemory: resource.MustParse("500m"), consts.ResourceNvidiaGPU: resource.MustParse("0"), }, expect: types.ResourceList{ types.ResourceMemory: types.Quantity("500m"), }, }, { resource: map[v1types.ResourceName]resource.Quantity{ v1types.ResourceCPU: resource.MustParse("0"), v1types.ResourceMemory: resource.MustParse("0"), consts.ResourceNvidiaGPU: resource.MustParse("0.5"), }, expect: types.ResourceList{ types.ResourceGPU: types.Quantity("500m"), }, }, { resource: map[v1types.ResourceName]resource.Quantity{ v1types.ResourceCPU: resource.MustParse("0.1"), v1types.ResourceMemory: resource.MustParse("0"), consts.ResourceNvidiaGPU: resource.MustParse("0"), }, expect: types.ResourceList{ types.ResourceCPU: types.Quantity("100m"), }, }, } for _, tc := range tcs { value := AsResourceList(tc.resource) Expect(value).To(Equal(tc.expect)) } }) It("function AsInferenceDeployment", func() { mockTime, _ := time.Parse("2006-01-02", "2023-09-07") tcs := []struct { inf *v2alpha1.Inference deployment *appsv1.Deployment expect *types.InferenceDeployment }{ { inf: nil, deployment: nil, expect: nil, }, { inf: Ptr(v2alpha1.Inference{}), deployment: Ptr(appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ CreationTimestamp: metav1.Time{ Time: mockTime, }, }, }), expect: Ptr(types.InferenceDeployment{ Status: types.InferenceDeploymentStatus{ Phase: types.PhaseNotReady, CreatedAt: Ptr(mockTime), }, }), }, { inf: Ptr(v2alpha1.Inference{ Spec: v2alpha1.InferenceSpec{ Scaling: Ptr(v2alpha1.ScalingConfig{ Type: Ptr(v2alpha1.ScalingTypeCapacity), }), }, }), deployment: Ptr(appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ CreationTimestamp: metav1.Time{ Time: mockTime, }, }, }), expect: Ptr(types.InferenceDeployment{ Spec: types.InferenceDeploymentSpec{ Scaling: Ptr(types.ScalingConfig{ Type: Ptr(types.ScalingTypeCapacity), }), }, Status: types.InferenceDeploymentStatus{ Phase: types.PhaseNotReady, CreatedAt: Ptr(mockTime), }, }), }, } for _, tc := range tcs { value := AsInferenceDeployment(tc.inf, tc.deployment) Expect(value).To(Equal(tc.expect)) } }) }) ================================================ FILE: agent/pkg/k8s/convert_job.go ================================================ package k8s import ( v1 "k8s.io/api/batch/v1" "github.com/tensorchord/openmodelz/agent/api/types" ) func AsBuild(job v1.Job) (types.Build, error) { build := types.Build{ Spec: types.BuildSpec{ Name: job.Name, Namespace: job.Namespace, }, } if job.Status.Succeeded > 0 { build.Status.Phase = types.BuildPhaseSucceeded } else if job.Status.Failed > 0 { build.Status.Phase = types.BuildPhaseFailed } else if job.Status.Active > 0 { build.Status.Phase = types.BuildPhaseRunning } else { build.Status.Phase = types.BuildPhasePending } return build, nil } ================================================ FILE: agent/pkg/k8s/convert_pod.go ================================================ package k8s import ( "github.com/tensorchord/openmodelz/modelzetes/pkg/consts" v1 "k8s.io/api/core/v1" "github.com/tensorchord/openmodelz/agent/api/types" ) func MakeLabelSelector(name string) map[string]string { return map[string]string{ "app": name, } } func InstanceFromPod(pod v1.Pod) *types.InferenceDeploymentInstance { i := &types.InferenceDeploymentInstance{ Spec: types.InferenceDeploymentInstanceSpec{ Namespace: pod.Namespace, Name: pod.Name, OwnerReference: pod.Labels[consts.LabelInferenceName], }, Status: types.InferenceDeploymentInstanceStatus{ Reason: pod.Status.Reason, Message: pod.Status.Message, }, } if pod.Status.StartTime != nil { i.Status.StartTime = pod.Status.StartTime.Time } switch pod.Status.Phase { case v1.PodRunning: i.Status.Phase = types.InstancePhaseRunning case v1.PodPending: i.Status.Phase = types.InstancePhasePending case v1.PodFailed: i.Status.Phase = types.InstancePhaseFailed case v1.PodSucceeded: i.Status.Phase = types.InstancePhaseSucceeded case v1.PodUnknown: i.Status.Phase = types.InstancePhaseUnknown } if pod.Status.Conditions != nil { for _, c := range pod.Status.Conditions { if c.Type == v1.PodScheduled && c.Status == v1.ConditionFalse { i.Status.Phase = types.InstancePhaseScheduling i.Status.Reason = c.Reason i.Status.Message = c.Message break } } } if len(pod.Status.ContainerStatuses) != 0 { if pod.Status.ContainerStatuses[0].Started != nil && !*pod.Status.ContainerStatuses[0].Started { i.Status.Phase = types.InstancePhaseCreating if pod.Status.ContainerStatuses[0].State.Waiting != nil { i.Status.Reason = pod.Status.ContainerStatuses[0].State.Waiting.Reason i.Status.Message = pod.Status.ContainerStatuses[0].State.Waiting.Message i.Status.Phase = types.InstancePhase( pod.Status.ContainerStatuses[0].State.Waiting.Reason) } else if pod.Status.ContainerStatuses[0].State.Running != nil { i.Status.Phase = types.InstancePhaseInitializing } else if pod.Status.ContainerStatuses[0].State.Terminated != nil { i.Status.Phase = types.InstancePhaseFailed i.Status.Reason = pod.Status.ContainerStatuses[0].State.Terminated.Reason i.Status.Message = pod.Status.ContainerStatuses[0].State.Terminated.Message } } } return i } ================================================ FILE: agent/pkg/k8s/convert_pod_test.go ================================================ package k8s import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/sirupsen/logrus" . "github.com/tensorchord/openmodelz/modelzetes/pkg/pointer" "github.com/tensorchord/openmodelz/agent/api/types" v1 "k8s.io/api/core/v1" ) var _ = Describe("agent/pkg/k8s/convert_pod", func() { It("function InstanceFromPod", func() { tcs := []struct { desc string pod v1.Pod expect *types.InferenceDeploymentInstance }{ { desc: "empty pod", pod: v1.Pod{}, expect: Ptr( types.InferenceDeploymentInstance{}, ), }, { desc: "running pod", pod: v1.Pod{ Status: v1.PodStatus{ Phase: v1.PodRunning, }, }, expect: Ptr( types.InferenceDeploymentInstance{ Status: types.InferenceDeploymentInstanceStatus{ Phase: types.InstancePhaseRunning, }, }, ), }, { desc: "pending pod", pod: v1.Pod{ Status: v1.PodStatus{ Phase: v1.PodPending, }, }, expect: Ptr( types.InferenceDeploymentInstance{ Status: types.InferenceDeploymentInstanceStatus{ Phase: types.InstancePhasePending, }, }, ), }, { desc: "scheduling pod", pod: v1.Pod{ Status: v1.PodStatus{ Phase: v1.PodPending, Conditions: []v1.PodCondition{ { Type: v1.PodScheduled, Status: v1.ConditionFalse, }, }, }, }, expect: Ptr( types.InferenceDeploymentInstance{ Status: types.InferenceDeploymentInstanceStatus{ Phase: types.InstancePhaseScheduling, }, }, ), }, { desc: "failed pod", pod: v1.Pod{ Status: v1.PodStatus{ Phase: v1.PodFailed, }, }, expect: Ptr( types.InferenceDeploymentInstance{ Status: types.InferenceDeploymentInstanceStatus{ Phase: types.InstancePhaseFailed, }, }, ), }, { desc: "succeed pod", pod: v1.Pod{ Status: v1.PodStatus{ Phase: v1.PodSucceeded, }, }, expect: Ptr( types.InferenceDeploymentInstance{ Status: types.InferenceDeploymentInstanceStatus{ Phase: types.InstancePhaseSucceeded, }, }, ), }, { desc: "unknown pod", pod: v1.Pod{ Status: v1.PodStatus{ Phase: v1.PodUnknown, }, }, expect: Ptr( types.InferenceDeploymentInstance{ Status: types.InferenceDeploymentInstanceStatus{ Phase: types.InstancePhaseUnknown, }, }, ), }, { desc: "creating pod", pod: v1.Pod{ Status: v1.PodStatus{ ContainerStatuses: []v1.ContainerStatus{ { Started: Ptr(false), }, }, }, }, expect: Ptr( types.InferenceDeploymentInstance{ Status: types.InferenceDeploymentInstanceStatus{ Phase: types.InstancePhaseCreating, }, }, ), }, { desc: "waiting pod", pod: v1.Pod{ Status: v1.PodStatus{ ContainerStatuses: []v1.ContainerStatus{ { Started: Ptr(false), State: v1.ContainerState{ Waiting: Ptr(v1.ContainerStateWaiting{ Reason: "mock-status", }), }, }, }, }, }, expect: Ptr( types.InferenceDeploymentInstance{ Status: types.InferenceDeploymentInstanceStatus{ Phase: types.InstancePhase("mock-status"), Reason: "mock-status", }, }, ), }, { desc: "initializing pod", pod: v1.Pod{ Status: v1.PodStatus{ ContainerStatuses: []v1.ContainerStatus{ { Started: Ptr(false), State: v1.ContainerState{ Running: Ptr(v1.ContainerStateRunning{}), }, }, }, }, }, expect: Ptr( types.InferenceDeploymentInstance{ Status: types.InferenceDeploymentInstanceStatus{ Phase: types.InstancePhaseInitializing, }, }, ), }, { desc: "terminated pod", pod: v1.Pod{ Status: v1.PodStatus{ ContainerStatuses: []v1.ContainerStatus{ { Started: Ptr(false), State: v1.ContainerState{ Terminated: Ptr(v1.ContainerStateTerminated{}), }, }, }, }, }, expect: Ptr( types.InferenceDeploymentInstance{ Status: types.InferenceDeploymentInstanceStatus{ Phase: types.InstancePhaseFailed, }, }, ), }, } for _, tc := range tcs { logrus.Info(tc.desc) value := InstanceFromPod(tc.pod) Expect(value).To(Equal(tc.expect)) } }) }) ================================================ FILE: agent/pkg/k8s/generate_image_cache.go ================================================ package k8s import ( "time" kubefledged "github.com/senthilrch/kube-fledged/pkg/apis/kubefledged/v1alpha3" "github.com/tensorchord/openmodelz/agent/api/types" modelzetes "github.com/tensorchord/openmodelz/modelzetes/pkg/apis/modelzetes/v2alpha1" "github.com/tensorchord/openmodelz/modelzetes/pkg/consts" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" ) func MakeImageCache(req types.ImageCache, inference *modelzetes.Inference) *kubefledged.ImageCache { nodeSlector := map[string]string{ consts.LabelServerResource: string(req.NodeSelector), } cache := &kubefledged.ImageCache{ ObjectMeta: v1.ObjectMeta{ Name: req.Name, Namespace: req.Namespace, OwnerReferences: []metav1.OwnerReference{ *metav1.NewControllerRef(inference, schema.GroupVersionKind{ Group: modelzetes.SchemeGroupVersion.Group, Version: modelzetes.SchemeGroupVersion.Version, Kind: modelzetes.Kind, }), }, }, Spec: kubefledged.ImageCacheSpec{ CacheSpec: []kubefledged.CacheSpecImages{ { Images: []kubefledged.Image{ { Name: req.Image, ForceFullCache: req.ForceFullCache, }, }, NodeSelector: nodeSlector, }, }, }, Status: kubefledged.ImageCacheStatus{ StartTime: &metav1.Time{Time: time.Now()}, }, } return cache } ================================================ FILE: agent/pkg/k8s/generate_job.go ================================================ package k8s import ( "time" "github.com/cockroachdb/errors" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "github.com/tensorchord/openmodelz/agent/api/types" "github.com/tensorchord/openmodelz/modelzetes/pkg/apis/modelzetes/v2alpha1" "github.com/tensorchord/openmodelz/modelzetes/pkg/consts" ) func MakeBuild(req types.Build, inference *v2alpha1.Inference, builderImage, buildkitdAddr, buildctlBin, secret string) (*batchv1.Job, error) { job := &batchv1.Job{} duration, err := time.ParseDuration(req.Spec.BuildTarget.Duration) if err != nil { return nil, errors.Wrap(err, "failed to parse duration") } seconds := int64(duration.Seconds()) defaultBackoffLimit := int32(0) defaultTTLSecondsAfterFinished := int32(60 * 60 * 24 * 7) // 7 days envs := []corev1.EnvVar{ { Name: "MODELZ_BUILD_NAME", Value: req.Spec.Name, }, { Name: "MODELZ_BUILDER", Value: string(req.Spec.BuildTarget.Builder), }, { Name: "MODELZ_BUILD_ARTIFACT_IMAGE", Value: req.Spec.BuildTarget.ArtifactImage, }, { Name: "MODELZ_BUILD_ARTIFACT_IMAGE_TAG", Value: req.Spec.BuildTarget.ArtifactImageTag, }, { Name: "MODELZ_REGISTRY", Value: req.Spec.BuildTarget.Registry, }, { Name: "MODELZ_REGISTRY_TOKEN", Value: req.Spec.BuildTarget.RegistryToken, }, } if req.Spec.BuildTarget.Builder != types.BuilderTypeImage { envs = append(envs, buildEnvsForDockerfileOrEnvd(req, buildkitdAddr, buildctlBin)...) } else { envs = append(envs, buildEnvsForImage(req)...) } ownerReference := []metav1.OwnerReference{ *metav1.NewControllerRef(inference, schema.GroupVersionKind{ Group: v2alpha1.SchemeGroupVersion.Group, Version: v2alpha1.SchemeGroupVersion.Version, Kind: v2alpha1.Kind, }), } job = &batchv1.Job{ ObjectMeta: metav1.ObjectMeta{ Name: req.Spec.Name, Namespace: req.Spec.Namespace, OwnerReferences: ownerReference, Labels: map[string]string{ consts.LabelBuildName: req.Spec.Name, consts.AnnotationBuilding: "true", }, }, Spec: batchv1.JobSpec{ ActiveDeadlineSeconds: &seconds, BackoffLimit: &defaultBackoffLimit, TTLSecondsAfterFinished: &defaultTTLSecondsAfterFinished, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{ consts.LabelBuildName: req.Spec.Name, }, }, Spec: corev1.PodSpec{ ImagePullSecrets: []corev1.LocalObjectReference{ {Name: secret}, }, Volumes: []corev1.Volume{ { Name: "workspace", VolumeSource: corev1.VolumeSource{ EmptyDir: &corev1.EmptyDirVolumeSource{}, }, }, }, RestartPolicy: corev1.RestartPolicyNever, Containers: []corev1.Container{ { Name: req.Spec.Name, Image: builderImage, ImagePullPolicy: corev1.PullAlways, VolumeMounts: []corev1.VolumeMount{ { Name: "workspace", MountPath: "/workspace", }, }, Env: envs, }, }, }, }, }, } return job, nil } func buildEnvsForImage(req types.Build) []corev1.EnvVar { envs := []corev1.EnvVar{} if req.Spec.DockerSource.AuthN.Username != "" { envs = append(envs, corev1.EnvVar{ Name: "MODELZ_SOURCE_REGISTRY_USERNAME", Value: req.Spec.DockerSource.AuthN.Username, }) } if req.Spec.DockerSource.AuthN.Password != "" { envs = append(envs, corev1.EnvVar{ Name: "MODELZ_SOURCE_REGISTRY_PASSWORD", Value: req.Spec.DockerSource.AuthN.Password, }) } if req.Spec.DockerSource.AuthN.Token != "" { envs = append(envs, corev1.EnvVar{ Name: "MODELZ_SOURCE_REGISTRY_TOKEN", Value: req.Spec.DockerSource.AuthN.Token, }) } if req.Spec.DockerSource.ArtifactImage != "" { envs = append(envs, corev1.EnvVar{ Name: "MODELZ_SOURCE_REGISTRY_IMAGE", Value: req.Spec.DockerSource.ArtifactImage, }) } if req.Spec.DockerSource.ArtifactImageTag != "" { envs = append(envs, corev1.EnvVar{ Name: "MODELZ_SOURCE_REGISTRY_IMAGE_TAG", Value: req.Spec.DockerSource.ArtifactImageTag, }) } return envs } func buildEnvsForDockerfileOrEnvd(req types.Build, buildkitdAddr, buildctlBin string) []corev1.EnvVar { return []corev1.EnvVar{ { Name: "MODELZ_BUILD_GIT_URL", Value: req.Spec.Repository, }, { Name: "MODELZ_BUILD_GIT_BRANCH", Value: req.Spec.Branch, }, { Name: "MODELZ_BUILD_GIT_COMMIT", Value: req.Spec.Revision, }, { Name: "MODELZ_BUILD_BASE_DIR", Value: req.Spec.BuildTarget.Directory, }, { Name: "MODELZ_WORKSPACE", Value: "/workspace", }, { Name: "MODELZ_BUILDKITD_ADDRESS", Value: buildkitdAddr, }, { Name: "MODELZ_BUILDER_BIN", Value: buildctlBin, }, } } ================================================ FILE: agent/pkg/k8s/managed_cluster.go ================================================ package k8s import ( "k8s.io/apimachinery/pkg/version" "k8s.io/client-go/kubernetes" ) func GetKubernetesVersion(client kubernetes.Interface) (*version.Info, error) { return client.Discovery().ServerVersion() } ================================================ FILE: agent/pkg/k8s/resolver.go ================================================ package k8s import ( "context" "fmt" "math/rand" "net/url" "strconv" "github.com/anthhub/forwarder" "github.com/phayes/freeport" "github.com/sirupsen/logrus" "github.com/tensorchord/openmodelz/modelzetes/pkg/consts" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" corelister "k8s.io/client-go/listers/core/v1" "k8s.io/client-go/rest" "github.com/tensorchord/openmodelz/agent/errdefs" ) type Resolver interface { Resolve(namespace, name string) (url.URL, error) Close(url url.URL) } func NewPortForwardingResolver(cfg *rest.Config, cli kubernetes.Interface) Resolver { return &PortForwardingResolver{ config: cfg, cli: cli, results: make(map[int]*forwarder.Result), } } func NewEndpointResolver(lister corelister.EndpointsLister) Resolver { return &EndpointResolver{ EndpointLister: lister, } } type PortForwardingResolver struct { config *rest.Config cli kubernetes.Interface results map[int]*forwarder.Result } func (e *PortForwardingResolver) Resolve(namespace, name string) (url.URL, error) { port, err := freeport.GetFreePort() if err != nil { return url.URL{}, err } svc, err := e.cli.CoreV1().Services(namespace).Get(context.Background(), "mdz-"+name, metav1.GetOptions{}) if err != nil { return url.URL{}, err } if svc.Spec.Ports == nil || len(svc.Spec.Ports) == 0 { return url.URL{}, errdefs.System(fmt.Errorf("no ports found in service %s", svc.Name)) } options := []*forwarder.Option{ { // the local port for forwarding LocalPort: port, // the k8s pod port RemotePort: svc.Spec.Ports[0].TargetPort.IntValue(), // the forwarding service name ServiceName: "mdz-" + name, // namespace default is "default" Namespace: namespace, }, } ret, err := forwarder.WithRestConfig(context.Background(), options, e.config) if err != nil { return url.URL{}, err } e.results[port] = ret // wait forwarding ready // the remote and local ports are listed _, err = ret.Ready() if err != nil { return url.URL{}, err } // the ports are ready res, err := url.Parse("http://localhost:" + strconv.Itoa(port)) return *res, err } func (e *PortForwardingResolver) Close(url url.URL) { port, err := strconv.Atoi(url.Port()) if err != nil { panic(err) } logrus.Infof("close port forwarding %d\n", port) if e.results[port] == nil { logrus.Infof("port forwarding %d not found\n", port) return } logrus.Infof("pointer: %v", e.results[port]) e.results[port].Close() } type EndpointResolver struct { EndpointLister corelister.EndpointsLister } func (e EndpointResolver) Resolve(namespace, name string) (url.URL, error) { svcName := consts.DefaultServicePrefix + name svc, err := e.EndpointLister.Endpoints(namespace).Get(svcName) if err != nil { if k8serrors.IsNotFound(err) { return url.URL{}, errdefs.NotFound(err) } return url.URL{}, errdefs.System(err) } if len(svc.Subsets) == 0 { return url.URL{}, errdefs.NotFound( fmt.Errorf("no subsets for \"%s.%s\"", svcName, namespace)) } all := len(svc.Subsets[0].Addresses) if len(svc.Subsets[0].Addresses) == 0 { return url.URL{}, errdefs.NotFound( fmt.Errorf("no addresses for \"%s.%s\"", svcName, namespace)) } target := rand.Intn(all) serviceIP := svc.Subsets[0].Addresses[target].IP servicePort := svc.Subsets[0].Ports[0].Port urlStr := fmt.Sprintf("http://%s:%d", serviceIP, servicePort) urlRes, err := url.Parse(urlStr) if err != nil { return url.URL{}, errdefs.System(err) } return *urlRes, nil } func (e EndpointResolver) Close(url.URL) { // do nothing } ================================================ FILE: agent/pkg/k8s/suite_test.go ================================================ package k8s import ( "testing" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) func TestBuilder(t *testing.T) { RegisterFailHandler(Fail) RunSpecs(t, "k8s") } ================================================ FILE: agent/pkg/log/factory.go ================================================ package log import ( "context" "github.com/tensorchord/openmodelz/agent/api/types" ) // Requester submits queries the logging system. type Requester interface { // Query submits a log request to the actual logging system. Query(ctx context.Context, req types.LogRequest) (<-chan types.Message, error) } ================================================ FILE: agent/pkg/log/k8s.go ================================================ package log import ( "bufio" "context" "fmt" "io" "strings" "time" "github.com/sirupsen/logrus" "github.com/tensorchord/openmodelz/modelzetes/pkg/consts" corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/informers" "k8s.io/client-go/informers/internalinterfaces" "k8s.io/client-go/kubernetes" v1 "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/client-go/tools/cache" "github.com/tensorchord/openmodelz/agent/api/types" "github.com/tensorchord/openmodelz/agent/errdefs" ) const ( // podInformerResync is the period between cache syncs in the pod informer podInformerResync = 5 * time.Second // defaultLogSince is the fallback log stream history defaultLogSince = 5 * time.Minute // LogBufferSize number of log messages that may be buffered LogBufferSize = 500 * 2 ) // K8sAPIRequestor implements the Requestor interface for k8s type K8sAPIRequestor struct { client kubernetes.Interface } func NewK8sAPIRequestor(client kubernetes.Interface) Requester { return &K8sAPIRequestor{ client: client, } } func (k *K8sAPIRequestor) Query(ctx context.Context, r types.LogRequest) (<-chan types.Message, error) { var sinceTime, endTime time.Time if r.Since != "" { var err error sinceTime, err = time.Parse(time.RFC3339, r.Since) if err != nil { return nil, errdefs.InvalidParameter(err) } } if r.End != "" { var err error endTime, err = time.Parse(time.RFC3339, r.End) if err != nil { return nil, errdefs.InvalidParameter(err) } } else if r.Follow { // avoid truncate endTime = time.Now().Add(time.Hour) } else { endTime = time.Now() } logStream, err := getLogs(ctx, k.client, r.Name, r.Namespace, int64(r.Tail), &sinceTime, r.Follow) if err != nil { return nil, err } msgStream := make(chan types.Message, LogBufferSize) go func() { defer close(msgStream) // here we depend on the fact that logStream will close when the context is cancelled, // this ensures that the go routine will resolve for msg := range logStream { // if we have an end time, we should stop streaming logs after that time if endTime.After(msg.Timestamp) { msgStream <- types.Message{ Timestamp: msg.Timestamp, Text: msg.Text, Name: msg.Name, Instance: msg.Instance, Namespace: msg.Namespace, } } } }() return msgStream, nil } // getLogs returns a channel of logs for the given function func getLogs(ctx context.Context, client kubernetes.Interface, functionName, namespace string, tail int64, since *time.Time, follow bool) ( <-chan types.Message, error) { added, err := startFunctionPodInformer(ctx, client, functionName, namespace) if err != nil { return nil, err } logs := make(chan types.Message, LogBufferSize) go func() { var watching uint defer close(logs) finished := make(chan error) for { select { case <-ctx.Done(): return case <-finished: watching-- if watching == 0 && !follow { return } case p := <-added: watching++ go func() { finished <- podLogs(ctx, client.CoreV1().Pods(namespace), p, functionName, namespace, tail, since, follow, logs) }() } } }() return logs, nil } // podLogs returns a stream of logs lines from the specified pod func podLogs(ctx context.Context, i v1.PodInterface, pod, container, namespace string, tail int64, since *time.Time, follow bool, dst chan<- types.Message) error { opts := &corev1.PodLogOptions{ Follow: follow, Timestamps: true, Container: container, } if tail > 0 { opts.TailLines = &tail } if opts.TailLines == nil || since != nil { opts.SinceSeconds = parseSince(since) } stream, err := i.GetLogs(pod, opts).Stream(ctx) if err != nil { return err } defer stream.Close() done := make(chan error) go func() { scanner := bufio.NewScanner(stream) for scanner.Scan() { msg, ts := extractTimestampAndMsg(scanner.Text()) dst <- types.Message{ Timestamp: ts, Text: msg, Instance: pod, Name: container, Namespace: namespace, } } if err := scanner.Err(); err != nil { done <- err return } }() select { case <-ctx.Done(): logrus.Debug("get-log context cancelled") return ctx.Err() case err := <-done: if err != io.EOF { logrus.Debugf("failed to read from pod log: %v", err) return err } return nil } } // startFunctionPodInformer will gather the list of existing Pods for the function, it will // watch for newly added or deleted function instances. func startFunctionPodInformer(ctx context.Context, client kubernetes.Interface, functionName, namespace string) (<-chan string, error) { functionSelector := &metav1.LabelSelector{ MatchLabels: map[string]string{consts.LabelInferenceName: functionName}, } selector, err := metav1.LabelSelectorAsSelector(functionSelector) if err != nil { return nil, errdefs.InvalidParameter(err) } logrus.WithFields(logrus.Fields{ "selector": selector.String(), "namespace": namespace, }).Debugf("starting log pod informer") factory := informers.NewFilteredSharedInformerFactory( client, podInformerResync, namespace, withLabels(selector.String()), ) podInformer := factory.Core().V1().Pods() podsResp, err := client.CoreV1().Pods(namespace).List(context.TODO(), metav1.ListOptions{LabelSelector: selector.String()}) if err != nil { if k8serrors.IsNotFound(err) { return nil, errdefs.NotFound(err) } else { return nil, errdefs.System(err) } } pods := podsResp.Items if len(pods) == 0 { return nil, errdefs.NotFound( fmt.Errorf("no pods found for inference: %s", functionName)) } // prepare channel with enough space for the current instance set added := make(chan string, len(pods)) podInformer.Informer().AddEventHandler(&podLoggerEventHandler{ added: added, }) // will add existing pods to the chan and then listen for any new pods go podInformer.Informer().Run(ctx.Done()) go func() { <-ctx.Done() close(added) }() return added, nil } // parseSince returns the time.Duration of the requested Since value _or_ 5 minutes func parseSince(r *time.Time) *int64 { var since int64 if r == nil || r.IsZero() { since = int64(defaultLogSince.Seconds()) return &since } since = int64(time.Since(*r).Seconds()) return &since } func extractTimestampAndMsg(logText string) (string, time.Time) { // first 32 characters is the k8s timestamp parts := strings.SplitN(logText, " ", 2) ts, err := time.Parse(time.RFC3339Nano, parts[0]) if err != nil { logrus.WithField("logText", logText). Errorf("error parsing timestamp: %s", err) return "", time.Time{} } if len(parts) == 2 { return parts[1], ts } return "", ts } func withLabels(selector string) internalinterfaces.TweakListOptionsFunc { return func(opts *metav1.ListOptions) { opts.LabelSelector = selector } } type podLoggerEventHandler struct { cache.ResourceEventHandler added chan<- string deleted chan<- string } func (h *podLoggerEventHandler) OnAdd(obj interface{}, isInitialList bool) { pod := obj.(*corev1.Pod) logrus.WithField("pod", pod.Name).Debugf("log pod informer added a pod") h.added <- pod.Name } func (h *podLoggerEventHandler) OnUpdate(oldObj, newObj interface{}) { // purposefully empty, we don't need to do anything for logs on update } func (h *podLoggerEventHandler) OnDelete(obj interface{}) { // this may not be needed, the log stream Reader _should_ close on its own without // us needing to watch and close it // pod := obj.(*corev1.Pod) // h.deleted <- pod.Name } ================================================ FILE: agent/pkg/log/loki.go ================================================ package log import ( "context" "encoding/json" "fmt" "net/http" "net/url" "strconv" "time" "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "github.com/tensorchord/openmodelz/agent/api/types" "github.com/tensorchord/openmodelz/agent/errdefs" ) const ( // refer to https://grafana.com/docs/loki/latest/api/#query-loki-over-a-range-of-time lokiQueryRangePath = "/loki/api/v1/query_range" ) type RangeQueryResponse struct { Data struct { Result []struct { Stream struct { Cluster string `json:"cluster,omitempty"` Container string `json:"container,omitempty"` Namespace string `json:"namespace,omitempty"` Pod string `json:"pod,omitempty"` Job string `json:"job,omitempty"` } Values [][]string `json:"values,omitempty"` } ResultType string `json:"resultType,omitempty"` } Status string `json:"status,omitempty"` } type LokiAPIRequestor struct { client http.Client url string user string token string } func NewLokiAPIRequestor(url, user, token string) Requester { loki := LokiAPIRequestor{ url: url, user: user, token: token, client: http.Client{}, } return &loki } func (l *LokiAPIRequestor) Query(ctx context.Context, r types.LogRequest) (<-chan types.Message, error) { var sinceTime time.Time if r.Since != "" { var err error sinceTime, err = time.Parse(time.RFC3339, r.Since) if err != nil { return nil, errdefs.InvalidParameter(err) } } logs, err := l.getLogs(ctx, &sinceTime, r.Namespace, r.Name) return logs, err } func (l *LokiAPIRequestor) getLogs(ctx context.Context, since *time.Time, namespace, name string) (<-chan types.Message, error) { endpoint, err := url.JoinPath(l.url, lokiQueryRangePath) if err != nil { return nil, errors.Wrap(err, "failed to construct the query URL") } req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) if err != nil { return nil, errors.Wrap(err, "failed to construct the Loki request") } req.SetBasicAuth(l.user, l.token) query := url.Values{} if since != nil { query.Add("start", since.String()) if time.Since(*since) > time.Hour*24*30 { // max query range is 30 days query.Add("end", strconv.Itoa(int(since.Add(time.Hour*24*30).UnixNano()))) } } query.Add("query", fmt.Sprintf(`{namespace="%s",pod="%s"}`, namespace, name)) req.URL.RawQuery = query.Encode() logrus.Debugf("get log from %s", req.URL.String()) resp, err := l.client.Do(req) if err != nil { return nil, errors.Wrap(err, "failed to request the Loki service") } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return nil, errors.Newf("failed to request the Loki, err[%s]", resp.Status) } var queryResp RangeQueryResponse err = json.NewDecoder(resp.Body).Decode(&queryResp) if err != nil { return nil, errors.Wrap(err, "failed to unmarshal json") } if len(queryResp.Data.Result) == 0 { return nil, errors.New("result contains ") } msgStream := make(chan types.Message, LogBufferSize) go func() { defer close(msgStream) for _, value := range queryResp.Data.Result[0].Values { timestamp, err := time.Parse(time.RFC3339, value[0]) if err != nil { logrus.Infof("failed to parse timestamp %s during parse log from %s:%s\n", value[0], namespace, name) continue } msgStream <- types.Message{ Timestamp: timestamp, Text: value[1], Name: name, Namespace: namespace, Instance: name, } } }() return msgStream, nil } ================================================ FILE: agent/pkg/metrics/exporter.go ================================================ // Copyright (c) Alex Ellis 2017 // Copyright (c) 2018 OpenFaaS Author(s) // Licensed under the MIT license. See LICENSE file in the project root for full license information. package metrics import ( "context" "fmt" "time" "github.com/prometheus/client_golang/prometheus" "github.com/sirupsen/logrus" "github.com/tensorchord/openmodelz/agent/api/types" "github.com/tensorchord/openmodelz/agent/pkg/runtime" ) // Exporter is a prometheus metrics collector. // It is an implementation of https://pkg.go.dev/github.com/prometheus/client_golang/prometheus#Collector. type Exporter struct { metricOptions MetricOptions runtime runtime.Runtime services []types.InferenceDeployment logger *logrus.Entry } // NewExporter creates a new exporter for the OpenFaaS gateway metrics func NewExporter(options MetricOptions, r runtime.Runtime) *Exporter { return &Exporter{ metricOptions: options, runtime: r, services: []types.InferenceDeployment{}, logger: logrus.WithField("component", "exporter"), } } // Describe is to describe the metrics for Prometheus func (e *Exporter) Describe(ch chan<- *prometheus.Desc) { e.metricOptions.GatewayInferenceInvocation.Describe(ch) e.metricOptions.GatewayInferencesHistogram.Describe(ch) e.metricOptions.ServiceReplicasGauge.Describe(ch) e.metricOptions.ServiceAvailableReplicasGauge.Describe(ch) e.metricOptions.ServiceTargetLoad.Describe(ch) e.metricOptions.GatewayInferenceInvocationStarted.Describe(ch) e.metricOptions.GatewayInferenceInvocationInflight.Describe(ch) } // Collect collects data to be consumed by prometheus func (e *Exporter) Collect(ch chan<- prometheus.Metric) { e.metricOptions.GatewayInferenceInvocation.Collect(ch) e.metricOptions.GatewayInferencesHistogram.Collect(ch) e.metricOptions.ServiceReplicasGauge.Reset() e.metricOptions.ServiceAvailableReplicasGauge.Reset() e.metricOptions.ServiceTargetLoad.Reset() e.metricOptions.PodStartHistogram.Collect(ch) for _, service := range e.services { var serviceName string if len(service.Spec.Namespace) > 0 { serviceName = fmt.Sprintf("%s.%s", service.Spec.Name, service.Spec.Namespace) } else { serviceName = service.Spec.Name } // Initial services information if nil after recent deployment e.metricOptions.GatewayInferenceInvocationStarted.WithLabelValues(serviceName) e.metricOptions.GatewayInferenceInvocationInflight.WithLabelValues(serviceName) // Set current replica count e.metricOptions.ServiceReplicasGauge.WithLabelValues(serviceName). Set(float64(service.Status.Replicas)) // Set available replica count e.metricOptions.ServiceAvailableReplicasGauge.WithLabelValues(serviceName). Set(float64(service.Status.AvailableReplicas)) // Set target load if service.Spec.Scaling != nil { e.metricOptions.ServiceTargetLoad.WithLabelValues( serviceName, string(*service.Spec.Scaling.Type)). Set(float64(*service.Spec.Scaling.TargetLoad)) } } e.metricOptions.GatewayInferenceInvocationStarted.Collect(ch) e.metricOptions.GatewayInferenceInvocationInflight.Collect(ch) e.metricOptions.ServiceReplicasGauge.Collect(ch) e.metricOptions.ServiceAvailableReplicasGauge.Collect(ch) e.metricOptions.ServiceTargetLoad.Collect(ch) } // StartServiceWatcher starts a ticker and collects service replica counts to expose to prometheus func (e *Exporter) StartServiceWatcher( ctx context.Context, interval time.Duration) { ticker := time.NewTicker(interval) quit := make(chan struct{}) go func() { for { select { case <-ticker.C: namespaces, err := e.runtime.NamespaceList(ctx) if err != nil { e.logger.Debug("unable to list namespaces: ", err) } services := []types.InferenceDeployment{} for _, namespace := range namespaces { nsServices, err := e.runtime.InferenceList(namespace) if err != nil { e.logger.Debug("unable to list services: ", err) continue } services = append(services, nsServices...) } e.services = services break case <-quit: return } } }() } ================================================ FILE: agent/pkg/metrics/metrics.go ================================================ // Copyright (c) Alex Ellis 2017. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. package metrics import ( "net/http" "sync" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" ) // MetricOptions to be used by web handlers type MetricOptions struct { GatewayInferenceInvocation *prometheus.CounterVec GatewayInferencesHistogram *prometheus.HistogramVec GatewayInferenceInvocationStarted *prometheus.CounterVec GatewayInferenceInvocationInflight *prometheus.GaugeVec ServiceReplicasGauge *prometheus.GaugeVec ServiceAvailableReplicasGauge *prometheus.GaugeVec ServiceTargetLoad *prometheus.GaugeVec PodStartHistogram *prometheus.HistogramVec } // ServiceMetricOptions provides RED metrics type ServiceMetricOptions struct { Histogram *prometheus.HistogramVec Counter *prometheus.CounterVec } // Synchronize to make sure MustRegister only called once var once = sync.Once{} // RegisterExporter registers with Prometheus for tracking func RegisterExporter(exporter *Exporter) { once.Do(func() { prometheus.MustRegister(exporter) }) } // PrometheusHandler Bootstraps prometheus for metrics collection func PrometheusHandler() http.Handler { return promhttp.Handler() } // BuildMetricsOptions builds metrics for tracking inferences in the API gateway func BuildMetricsOptions() MetricOptions { gatewayInferencesHistogram := prometheus.NewHistogramVec(prometheus.HistogramOpts{ Name: "gateway_inferences_seconds", Help: "Inference time taken", }, []string{"inference_name", "code"}) gatewayInferenceInvocation := prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "gateway", Subsystem: "inference", Name: "invocation_total", Help: "Inference metrics", }, []string{"inference_name", "code"}, ) serviceReplicas := prometheus.NewGaugeVec( prometheus.GaugeOpts{ Namespace: "gateway", Name: "service_count", Help: "Current count of replicas for inference", }, []string{"inference_name"}, ) serviceAvailableReplicas := prometheus.NewGaugeVec( prometheus.GaugeOpts{ Namespace: "gateway", Name: "service_available_count", Help: "Current count of available replicas for inference", }, []string{"inference_name"}, ) serviceTargetLoad := prometheus.NewGaugeVec( prometheus.GaugeOpts{ Namespace: "gateway", Name: "service_target_load", Help: "Target load for inference", }, []string{"inference_name", "scaling_type"}, ) gatewayInferenceInvocationStarted := prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "gateway", Subsystem: "inference", Name: "invocation_started", Help: "The total number of inference HTTP requests started.", }, []string{"inference_name"}, ) gatewayInferenceInvocationInflight := prometheus.NewGaugeVec( prometheus.GaugeOpts{ Namespace: "gateway", Subsystem: "inference", Name: "invocation_inflight", Help: "The number of inference HTTP inflight requests.", }, []string{"inference_name"}, ) podStartHistogram := prometheus.NewHistogramVec(prometheus.HistogramOpts{ Name: "pod_start_seconds", Help: "Pod start time taken", Buckets: prometheus.ExponentialBuckets(8, 1.5, 10), }, []string{"inference_name", "source_image"}) metricsOptions := MetricOptions{ GatewayInferencesHistogram: gatewayInferencesHistogram, GatewayInferenceInvocation: gatewayInferenceInvocation, ServiceReplicasGauge: serviceReplicas, ServiceAvailableReplicasGauge: serviceAvailableReplicas, ServiceTargetLoad: serviceTargetLoad, GatewayInferenceInvocationStarted: gatewayInferenceInvocationStarted, GatewayInferenceInvocationInflight: gatewayInferenceInvocationInflight, PodStartHistogram: podStartHistogram, } return metricsOptions } ================================================ FILE: agent/pkg/prom/prometheus_query.go ================================================ package prom import ( "encoding/json" "fmt" "io/ioutil" "net/http" "net/url" "strconv" "github.com/sirupsen/logrus" "github.com/tensorchord/openmodelz/agent/api/types" ) // PrometheusQuery represents parameters for querying Prometheus type PrometheusQuery struct { Port int Host string Client *http.Client } type PrometheusQueryFetcher interface { Fetch(query string) (*VectorQueryResponse, error) } // NewPrometheusQuery create a NewPrometheusQuery func NewPrometheusQuery(host string, port int, client *http.Client) PrometheusQuery { return PrometheusQuery{ Client: client, Host: host, Port: port, } } func (p PrometheusQuery) AddMetrics(inferences []types.InferenceDeployment) { if len(inferences) > 0 { ns := inferences[0].Spec.Namespace q := fmt.Sprintf(`sum(gateway_inference_invocation_total{inference_name=~".*.%s"}) by (inference_name)`, ns) // Restrict query results to only inference names matching namespace suffix. results, err := p.Fetch(url.QueryEscape(q)) if err != nil { // log the error but continue, the mixIn will correctly handle the empty results. logrus.Debugf("Error querying Prometheus: %s\n", err.Error()) } mixIn(inferences, results) } } // Fetch queries aggregated stats func (q PrometheusQuery) Fetch(query string) (*VectorQueryResponse, error) { req, reqErr := http.NewRequest(http.MethodGet, fmt.Sprintf("http://%s:%d/api/v1/query?query=%s", q.Host, q.Port, query), nil) if reqErr != nil { return nil, reqErr } res, getErr := q.Client.Do(req) if getErr != nil { return nil, getErr } if res.Body != nil { defer res.Body.Close() } bytesOut, readErr := ioutil.ReadAll(res.Body) if readErr != nil { return nil, readErr } if res.StatusCode != http.StatusOK { return nil, fmt.Errorf("unexpected status code from Prometheus want: %d, got: %d, body: %s", http.StatusOK, res.StatusCode, string(bytesOut)) } var values VectorQueryResponse unmarshalErr := json.Unmarshal(bytesOut, &values) if unmarshalErr != nil { return nil, fmt.Errorf("error unmarshalling result: %s, '%s'", unmarshalErr, string(bytesOut)) } return &values, nil } type VectorQueryResponse struct { Data struct { Result []struct { Metric struct { Code string `json:"code"` ScalingType string `json:"scaling_type"` InferenceName string `json:"inference_name"` } Value []interface{} `json:"value"` } } } func mixIn(inferences []types.InferenceDeployment, metrics *VectorQueryResponse) { if inferences == nil || metrics == nil { return } for i, inference := range inferences { for _, v := range metrics.Data.Result { if v.Metric.InferenceName == fmt.Sprintf("%s.%s", inference.Spec.Name, inference.Spec.Namespace) { metricValue := v.Value[1] switch value := metricValue.(type) { case string: f, err := strconv.ParseFloat(value, 64) if err != nil { logrus.Debugf("add_metrics: unable to convert value %q for metric: %s", value, err) continue } inferences[i].Status.InvocationCount += int32(f) } } } } } ================================================ FILE: agent/pkg/runtime/build.go ================================================ package runtime import ( "context" "fmt" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/tensorchord/openmodelz/agent/api/types" "github.com/tensorchord/openmodelz/agent/errdefs" "github.com/tensorchord/openmodelz/agent/pkg/k8s" "github.com/tensorchord/openmodelz/modelzetes/pkg/apis/modelzetes/v2alpha1" "github.com/tensorchord/openmodelz/modelzetes/pkg/consts" ) func (r generalRuntime) BuildList(ctx context.Context, namespace string) ( []types.Build, error) { res := []types.Build{} jobs, err := r.kubeClient.BatchV1().Jobs(namespace). List(ctx, metav1.ListOptions{ LabelSelector: fmt.Sprintf("%s=true", consts.AnnotationBuilding), }) if err != nil { if !k8serrors.IsNotFound(err) { return nil, errdefs.System(err) } } if jobs != nil { for _, job := range jobs.Items { build, err := k8s.AsBuild(job) if err != nil { return nil, errdefs.System(err) } res = append(res, build) } } return res, nil } func (r generalRuntime) BuildCreate(ctx context.Context, req types.Build, inference *v2alpha1.Inference, builderImage, buildkitdAddress, buildCtlBin, secret string) error { buildJob, err := k8s.MakeBuild(req, inference, builderImage, buildkitdAddress, buildCtlBin, secret) if err != nil { return errdefs.System(err) } if _, err := r.kubeClient.BatchV1().Jobs(req.Spec.Namespace). Create(ctx, buildJob, metav1.CreateOptions{}); err != nil { return errdefs.System(err) } return nil } func (r generalRuntime) BuildGet(ctx context.Context, namespace, buildName string) (types.Build, error) { job, err := r.kubeClient.BatchV1().Jobs(namespace).Get(ctx, buildName, metav1.GetOptions{}) if err != nil { if k8serrors.IsNotFound(err) { return types.Build{}, errdefs.NotFound(err) } return types.Build{}, errdefs.System(err) } res, err := k8s.AsBuild(*job) if err != nil { return types.Build{}, errdefs.System(err) } return res, nil } ================================================ FILE: agent/pkg/runtime/cluster_info_get.go ================================================ package runtime import ( "strings" "github.com/tensorchord/openmodelz/agent/api/types" "github.com/tensorchord/openmodelz/agent/pkg/k8s" "github.com/tensorchord/openmodelz/agent/pkg/version" ) func (r generalRuntime) GetClusterInfo(cluster *types.ManagedCluster) error { info, err := k8s.GetKubernetesVersion(r.kubeClient) if err != nil { return err } cluster.KubernetesVersion = info.GitVersion cluster.Platform = info.Platform v := version.GetVersion() cluster.Version = v.Version resources, err := r.ListServerResource() if err != nil { return err } cluster.ServerResources = strings.Join(resources, ";") return nil } ================================================ FILE: agent/pkg/runtime/image_cache.go ================================================ package runtime import ( "context" "github.com/sirupsen/logrus" "github.com/tensorchord/openmodelz/agent/api/types" "github.com/tensorchord/openmodelz/agent/pkg/k8s" modelzetes "github.com/tensorchord/openmodelz/modelzetes/pkg/apis/modelzetes/v2alpha1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func (r generalRuntime) ImageCacheCreate(ctx context.Context, req types.ImageCache, inference *modelzetes.Inference) error { imageCache := k8s.MakeImageCache(req, inference) logrus.Infof("%v", imageCache) if _, err := r.kubefledgedClient.KubefledgedV1alpha3(). ImageCaches(req.Namespace). Create(ctx, imageCache, metav1.CreateOptions{}); err != nil { return err } return nil } ================================================ FILE: agent/pkg/runtime/inference_create.go ================================================ package runtime import ( "context" "fmt" v1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/tensorchord/openmodelz/agent/api/types" "github.com/tensorchord/openmodelz/agent/errdefs" "github.com/tensorchord/openmodelz/agent/pkg/config" localconsts "github.com/tensorchord/openmodelz/agent/pkg/consts" ingressv1 "github.com/tensorchord/openmodelz/ingress-operator/pkg/apis/modelzetes/v1" v2alpha1 "github.com/tensorchord/openmodelz/modelzetes/pkg/apis/modelzetes/v2alpha1" "github.com/tensorchord/openmodelz/modelzetes/pkg/consts" ) func (r generalRuntime) InferenceCreate(ctx context.Context, req types.InferenceDeployment, cfg config.IngressConfig, event string, serverPort int) error { namespace := req.Spec.Namespace if r.eventEnabled { err := r.eventRecorder.CreateDeploymentEvent(namespace, req.Spec.Name, event, "") if err != nil { return err } } inf, err := makeInference(req) if err != nil { return err } // Create the ingress // TODO(gaocegege): Check if the domain is already used. if r.ingressEnabled { name := req.Spec.Labels[consts.LabelName] if r.ingressAnyIPToDomain { // Get the service with type=loadbalancer. svcs, err := r.kubeClient.CoreV1().Services("").List(ctx, metav1.ListOptions{}) if err != nil { return errdefs.System(fmt.Errorf("failed to list services: %v", err)) } if len(svcs.Items) == 0 { return errdefs.System(fmt.Errorf("no service with type=LoadBalancer")) } var externalIP string for _, s := range svcs.Items { if s.Spec.Type == v1.ServiceTypeLoadBalancer { if len(s.Status.LoadBalancer.Ingress) == 0 { continue } externalIP = s.Status.LoadBalancer.Ingress[0].IP break } } // Set the domain to ingressDomain := fmt.Sprintf("%s.%s", externalIP, localconsts.Domain) cfg.Domain = ingressDomain } domain, err := makeDomain(name, cfg.Domain) if err != nil { return errdefs.InvalidParameter(err) } // Set the domain. // Create the inference with the ingress domain. if inf.Spec.Annotations == nil { inf.Spec.Annotations = make(map[string]string) } if cfg.TLSEnabled { inf.Spec.Annotations[AnnotationDomain] = fmt.Sprintf("https://%s", domain) } else { inf.Spec.Annotations[AnnotationDomain] = fmt.Sprintf("http://%s", domain) } _, err = r.inferenceClient.TensorchordV2alpha1(). Inferences(namespace).Create( ctx, inf, metav1.CreateOptions{}) if err != nil { if k8serrors.IsAlreadyExists(err) { return errdefs.Conflict(err) } else { return errdefs.System(err) } } cfg.Domain = domain ingress, err := makeIngress(req, cfg) if err != nil { return err } _, err = r.ingressClient.TensorchordV1(). InferenceIngresses(cfg.Namespace). Create(ctx, ingress, metav1.CreateOptions{}) if err != nil { if k8serrors.IsAlreadyExists(err) { return errdefs.Conflict(err) } else { return errdefs.System(err) } } } else { // Set the gateway kubernetes service domain. domain := fmt.Sprintf("gateway.default:%d/api/v1/%s/%s/", serverPort, string(req.Spec.Framework), req.Spec.Name) if inf.Spec.Annotations == nil { inf.Spec.Annotations = make(map[string]string) } if cfg.TLSEnabled { inf.Spec.Annotations[AnnotationDomain] = fmt.Sprintf("https://%s", domain) } else { inf.Spec.Annotations[AnnotationDomain] = fmt.Sprintf("http://%s", domain) } _, err = r.inferenceClient.TensorchordV2alpha1(). Inferences(namespace).Create( ctx, inf, metav1.CreateOptions{}) if err != nil { if k8serrors.IsAlreadyExists(err) { return errdefs.Conflict(err) } else { return errdefs.System(err) } } } return nil } func makeInference(request types.InferenceDeployment) (*v2alpha1.Inference, error) { is := &v2alpha1.Inference{ ObjectMeta: metav1.ObjectMeta{ Name: request.Spec.Name, Namespace: request.Spec.Namespace, Labels: map[string]string{ consts.LabelInferenceName: request.Spec.Name, }, }, Spec: v2alpha1.InferenceSpec{ Name: request.Spec.Name, Image: request.Spec.Image, Framework: v2alpha1.Framework(request.Spec.Framework), Port: request.Spec.Port, Command: request.Spec.Command, EnvVars: request.Spec.EnvVars, Secrets: request.Spec.Secrets, Constraints: request.Spec.Constraints, Labels: request.Spec.Labels, Annotations: request.Spec.Annotations, HTTPProbePath: request.Spec.HTTPProbePath, }, } if request.Spec.Scaling != nil { is.Spec.Scaling = &v2alpha1.ScalingConfig{ MinReplicas: request.Spec.Scaling.MinReplicas, MaxReplicas: request.Spec.Scaling.MaxReplicas, TargetLoad: request.Spec.Scaling.TargetLoad, ZeroDuration: request.Spec.Scaling.ZeroDuration, StartupDuration: request.Spec.Scaling.StartupDuration, } if request.Spec.Scaling.Type != nil { buf := v2alpha1.ScalingType(*request.Spec.Scaling.Type) is.Spec.Scaling.Type = &buf } } rr, err := createResources(request) if err != nil { return nil, errdefs.InvalidParameter(err) } is.Spec.Resources = &rr return is, nil } func makeIngress(request types.InferenceDeployment, cfg config.IngressConfig) (*ingressv1.InferenceIngress, error) { labels := map[string]string{ consts.LabelInferenceName: request.Spec.Name, consts.LabelInferenceNamespace: request.Spec.Namespace, } if request.Spec.Labels == nil { return nil, errdefs.InvalidParameter(fmt.Errorf("labels is required")) } ingress := &ingressv1.InferenceIngress{ ObjectMeta: metav1.ObjectMeta{ Name: request.Spec.Name, Namespace: cfg.Namespace, Labels: labels, }, Spec: ingressv1.InferenceIngressSpec{ Domain: cfg.Domain, Framework: string(request.Spec.Framework), IngressType: "nginx", BypassGateway: false, Function: request.Spec.Name, TLS: &ingressv1.InferenceIngressTLS{ Enabled: cfg.TLSEnabled, }, }, } annotation := map[string]string{} if value, exist := request.Spec.Annotations[consts.AnnotationControlPlaneKey]; exist { annotation[consts.AnnotationControlPlaneKey] = value } ingress.Annotations = annotation return ingress, nil } ================================================ FILE: agent/pkg/runtime/inference_delete.go ================================================ package runtime import ( "context" ingressclientset "github.com/tensorchord/openmodelz/ingress-operator/pkg/client/clientset/versioned" inferenceclientset "github.com/tensorchord/openmodelz/modelzetes/pkg/client/clientset/versioned" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/tensorchord/openmodelz/agent/errdefs" ) func (r generalRuntime) InferenceDelete(ctx context.Context, namespace, inferenceName, ingressNamespace, event string) error { if r.eventEnabled { err := r.eventRecorder.CreateDeploymentEvent(namespace, inferenceName, event, "") if err != nil { return err } } getOpts := metav1.GetOptions{} // This makes sure we don't delete non-labelled deployments _, err := r.inferenceClient.TensorchordV2alpha1(). Inferences(namespace). Get(context.TODO(), inferenceName, getOpts) if err != nil { if k8serrors.IsNotFound(err) { return errdefs.NotFound(err) } else { return errdefs.System(err) } } if err := deleteInference(ctx, namespace, r.inferenceClient, r.ingressClient, ingressNamespace, inferenceName, r.ingressEnabled); err != nil { return err } return nil } func deleteInference(ctx context.Context, namespace string, clientset inferenceclientset.Interface, ingressClient ingressclientset.Interface, baseNamespace string, inferenceName string, ingressEnabled bool) error { foregroundPolicy := metav1.DeletePropagationForeground opts := &metav1.DeleteOptions{PropagationPolicy: &foregroundPolicy} if deployErr := clientset.TensorchordV2alpha1().Inferences(namespace). Delete(ctx, inferenceName, *opts); deployErr != nil { if k8serrors.IsNotFound(deployErr) { return errdefs.NotFound(deployErr) } else { return errdefs.System(deployErr) } } if ingressEnabled && ingressClient != nil { if err := ingressClient.TensorchordV1().InferenceIngresses(baseNamespace).Delete(ctx, inferenceName, *opts); err != nil { if k8serrors.IsNotFound(err) { return errdefs.NotFound(err) } else { return errdefs.System(err) } } } return nil } ================================================ FILE: agent/pkg/runtime/inference_exec.go ================================================ package runtime import ( "errors" "fmt" "io" "net/http" "time" "github.com/gin-gonic/gin" "github.com/gorilla/websocket" "github.com/sirupsen/logrus" v1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/rand" clientsetscheme "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/tools/remotecommand" "github.com/tensorchord/openmodelz/agent/errdefs" ) const ( // Time allowed to write a message to the peer. writeWait = 10 * time.Second // ctrl+d to close terminal. endOfTransmission = "\u0004" ) func (r generalRuntime) InferenceExec(ctx *gin.Context, namespace, instance string, commands []string, tty bool) error { pod, err := r.kubeClient.CoreV1().Pods(namespace).Get( ctx.Request.Context(), instance, metav1.GetOptions{}) if err != nil { if k8serrors.IsNotFound(err) { return errdefs.NotFound(errors.New("inference instance not found")) } return errdefs.System(err) } if pod.Status.Phase != v1.PodRunning { return errdefs.Unavailable(errors.New("inference instance is not running")) } req := r.kubeClient.CoreV1().RESTClient().Post(). Resource("pods"). Name(instance). Namespace(namespace). SubResource("exec") req.VersionedParams(&v1.PodExecOptions{ Command: commands, Stdin: tty, Stdout: true, Stderr: true, TTY: tty, }, clientsetscheme.ParameterCodec) exec, err := remotecommand.NewSPDYExecutor(r.clientConfig, http.MethodPost, req.URL()) if err != nil { return errdefs.System(err) } if tty { t, err := newTerminalSession(fmt.Sprintf("exec/%s/%s/%s", namespace, instance, rand.String(5)), ctx.Request, ctx.Writer) if err != nil { return err } defer t.Close() logrus.WithField("exec", exec).Debugf("executing command") if err = exec.StreamWithContext(ctx.Request.Context(), remotecommand.StreamOptions{ Stdin: t, Stdout: t, Stderr: t, TerminalSizeQueue: t, Tty: true, }); err != nil { // The response is already hijacked, so we can't return an error. logrus.Warnf("exec stream failed: %v", err) return nil } } else { logrus.Debugf("running without tty") if err := exec.StreamWithContext(ctx.Request.Context(), remotecommand.StreamOptions{ Stdout: ctx.Writer, Stderr: ctx.Writer, Tty: tty, }); err != nil { return errdefs.System(err) } } return nil } type PtyHandler interface { io.Reader io.Writer remotecommand.TerminalSizeQueue } // TerminalMessage is the messaging protocol between ShellController and TerminalSession. // // OP DIRECTION FIELD(S) USED DESCRIPTION // --------------------------------------------------------------------- // bind fe->be SessionID Id sent back from TerminalResponse // stdin fe->be Data Keystrokes/paste buffer // resize fe->be Rows, Cols New terminal size // stdout be->fe Data Output from the process // toast be->fe Data OOB message to be shown to the user type TerminalMessage struct { ID string `json:"id,omitempty"` Op string `json:"op,omitempty"` Data string `json:"data,omitempty"` Rows uint16 `json:"rows,omitempty"` Cols uint16 `json:"cols,omitempty"` } // TerminalSession type TerminalSession struct { ID string wsConn *websocket.Conn sizeChan chan remotecommand.TerminalSize doneChan chan struct{} } // TerminalSize handles pty->process resize events // Called in a loop from remotecommand as long as the process is running func (t *TerminalSession) Next() *remotecommand.TerminalSize { select { case size := <-t.sizeChan: return &size case <-t.doneChan: return nil } } // Read handles pty->process messages (stdin, resize) // Called in a loop from remotecommand as long as the process is running func (t *TerminalSession) Read(p []byte) (int, error) { var msg TerminalMessage if err := t.wsConn.ReadJSON(&msg); err != nil { logrus.Debugf("%s: read json failed: %v", t.ID, err) return copy(p, endOfTransmission), err } logrus.Debugf("%s: read json: %v", t.ID, msg) switch msg.Op { case "stdin": logrus.WithField("remote", t.wsConn.RemoteAddr()).Debugf("%s: read %d bytes: %s", t.ID, len(msg.Data), msg.Data) size := copy(p, msg.Data) logrus.WithField("remote", t.wsConn.RemoteAddr()).Debugf("%s: copied %d bytes: %s", t.ID, size, p) return size, nil case "resize": t.sizeChan <- remotecommand.TerminalSize{Width: msg.Cols, Height: msg.Rows} return 0, nil default: logrus.WithField("remote", t.wsConn.RemoteAddr()).Debugf("%s: unknown message type '%s'", t.ID, msg.Op) return copy(p, endOfTransmission), fmt.Errorf("unknown message type '%s'", msg.Op) } } // Write handles process->pty stdout // Called from remotecommand whenever there is any output func (t *TerminalSession) Write(p []byte) (int, error) { msg := TerminalMessage{ Op: "stdout", Data: string(p), } logrus.WithField("remote", t.wsConn.RemoteAddr()).Debugf("%s: write %d bytes: %s", t.ID, len(p), string(p)) if err := t.wsConn.WriteJSON(msg); err != nil { logrus.WithField("remote", t.wsConn.RemoteAddr()).Debugf("write message failed: %v", err) return 0, err } return len(p), nil } func (t *TerminalSession) Close() error { close(t.doneChan) return t.wsConn.Close() } func newTerminalSession(id string, r *http.Request, w http.ResponseWriter) (*TerminalSession, error) { upgrader := websocket.Upgrader{ ReadBufferSize: 1024, WriteBufferSize: 1024, CheckOrigin: func(r *http.Request) bool { return true }, } conn, err := upgrader.Upgrade(w, r, nil) if err != nil { return nil, err } return &TerminalSession{ ID: id, wsConn: conn, sizeChan: make(chan remotecommand.TerminalSize), doneChan: make(chan struct{}), }, nil } ================================================ FILE: agent/pkg/runtime/inference_get.go ================================================ package runtime import ( "fmt" k8serrors "k8s.io/apimachinery/pkg/api/errors" v1 "k8s.io/client-go/listers/apps/v1" "github.com/tensorchord/openmodelz/agent/api/types" "github.com/tensorchord/openmodelz/agent/errdefs" "github.com/tensorchord/openmodelz/agent/pkg/k8s" apis "github.com/tensorchord/openmodelz/modelzetes/pkg/apis/modelzetes/v2alpha1" "github.com/tensorchord/openmodelz/modelzetes/pkg/client/listers/modelzetes/v2alpha1" ) func (r generalRuntime) InferenceGet(namespace, inferenceName string) ( *types.InferenceDeployment, error) { return inferenceGet(namespace, inferenceName, r.inferenceInformer.Lister(), r.deploymentInformer.Lister()) } func (r generalRuntime) InferenceGetCRD(namespace, name string) (*apis.Inference, error) { inference, err := r.inferenceInformer.Lister().Inferences(namespace).Get(name) if err != nil { if k8serrors.IsNotFound(err) { return nil, errdefs.NotFound(err) } return nil, err } return inference, nil } // inferenceGet returns a inference or nil if not found func inferenceGet(namespace string, inferenceName string, infLister v2alpha1.InferenceLister, lister v1.DeploymentLister) (*types.InferenceDeployment, error) { inference, err := infLister.Inferences(namespace).Get(inferenceName) if err != nil { if k8serrors.IsNotFound(err) { return nil, errdefs.NotFound(err) } return nil, err } item, err := lister.Deployments(namespace). Get(inferenceName) if err != nil { if !k8serrors.IsNotFound(err) { return nil, err } } inf := k8s.AsInferenceDeployment(inference, item) if inf != nil { return inf, nil } return nil, fmt.Errorf("inference: %s not found", inferenceName) } ================================================ FILE: agent/pkg/runtime/inference_instance.go ================================================ package runtime import ( k8serrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/labels" v1 "k8s.io/client-go/listers/core/v1" "github.com/tensorchord/openmodelz/agent/api/types" "github.com/tensorchord/openmodelz/agent/errdefs" "github.com/tensorchord/openmodelz/agent/pkg/k8s" ) func (r generalRuntime) InferenceInstanceList(namespace, inferenceName string) ( []types.InferenceDeploymentInstance, error) { return getInstances(namespace, inferenceName, r.podInformer.Lister()) } func getInstances(functionNamespace string, functionName string, lister v1.PodLister) ([]types.InferenceDeploymentInstance, error) { instances := make([]types.InferenceDeploymentInstance, 0) items, err := lister.List( labels.SelectorFromSet(k8s.MakeLabelSelector(functionName))) if err != nil { if k8serrors.IsNotFound(err) { return nil, nil } return nil, errdefs.System(err) } for _, item := range items { if item != nil { instance := k8s.InstanceFromPod(*item) if instance != nil { instances = append(instances, *instance) } } } return instances, nil } ================================================ FILE: agent/pkg/runtime/inference_list.go ================================================ package runtime import ( "sort" mv2alpha1 "github.com/tensorchord/openmodelz/modelzetes/pkg/apis/modelzetes/v2alpha1" modelzetesv2alpha1 "github.com/tensorchord/openmodelz/modelzetes/pkg/client/listers/modelzetes/v2alpha1" "github.com/tensorchord/openmodelz/modelzetes/pkg/consts" appsv1 "k8s.io/api/apps/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/selection" v1 "k8s.io/client-go/listers/apps/v1" "github.com/tensorchord/openmodelz/agent/api/types" "github.com/tensorchord/openmodelz/agent/errdefs" "github.com/tensorchord/openmodelz/agent/pkg/k8s" ) func (r generalRuntime) InferenceList(namespace string) ([]types.InferenceDeployment, error) { infLister := r.inferenceInformer.Lister() deploymentLister := r.deploymentInformer.Lister() functions, err := inferenceList(namespace, infLister, deploymentLister) if err != nil { return nil, err } return functions, nil } func inferenceList(functionNamespace string, infLister modelzetesv2alpha1.InferenceLister, deploymentLister v1.DeploymentLister) ([]types.InferenceDeployment, error) { functions := []types.InferenceDeployment{} sel := labels.NewSelector() req, err := labels.NewRequirement(consts.LabelInferenceName, selection.Exists, []string{}) if err != nil { return functions, errdefs.NotFound(err) } onlyFunctions := sel.Add(*req) inferences, err := infLister.Inferences(functionNamespace). List(labels.Everything()) if err != nil { if k8serrors.IsNotFound(err) { return functions, nil } else { return functions, errdefs.System(err) } } deploys, err := deploymentLister.Deployments(functionNamespace).List(onlyFunctions) if err != nil { if k8serrors.IsNotFound(err) { return getInferences(inferences, deploys) } else { return functions, errdefs.System(err) } } return getInferences(inferences, deploys) } func getInferences(inferences []*mv2alpha1.Inference, deploys []*appsv1.Deployment) ([]types.InferenceDeployment, error) { sort.Slice(inferences, func(i, j int) bool { return (*inferences[i]).Name < (*inferences[j]).Name }) sort.Slice(deploys, func(i, j int) bool { return (*deploys[i]).Name < (*deploys[j]).Name }) res := []types.InferenceDeployment{} j := 0 for i := range inferences { if j >= len(deploys) { res = append(res, *k8s.AsInferenceDeployment(inferences[i], nil)) } else if inferences[i].Name != deploys[j].Name { res = append(res, *k8s.AsInferenceDeployment(inferences[i], nil)) } else { res = append(res, *k8s.AsInferenceDeployment(inferences[i], deploys[j])) j++ } } return res, nil } ================================================ FILE: agent/pkg/runtime/inference_replicas.go ================================================ package runtime import ( "context" "github.com/tensorchord/openmodelz/modelzetes/pkg/consts" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/tensorchord/openmodelz/agent/api/types" "github.com/tensorchord/openmodelz/agent/errdefs" ) func (r generalRuntime) InferenceScale(ctx context.Context, namespace string, req types.ScaleServiceRequest, inf *types.InferenceDeployment) (err error) { options := metav1.GetOptions{ TypeMeta: metav1.TypeMeta{ Kind: "Deployment", APIVersion: "apps/v1", }, } deployment, err := r.kubeClient.AppsV1().Deployments(namespace). Get(ctx, req.ServiceName, options) if err != nil { return errdefs.InvalidParameter(err) } oldReplicas := *deployment.Spec.Replicas replicas := int32(req.Replicas) if inf.Spec.Scaling != nil { minReplicas := *inf.Spec.Scaling.MinReplicas if replicas < minReplicas { replicas = minReplicas } maxReplicas := *inf.Spec.Scaling.MaxReplicas if replicas > maxReplicas { replicas = maxReplicas } } if replicas >= consts.MaxReplicas { replicas = consts.MaxReplicas } if oldReplicas == replicas { return nil } event := types.DeploymentScaleDownEvent if oldReplicas < replicas { event = types.DeploymentScaleUpEvent } var building bool if r.buildEnabled { _, building = deployment.Annotations[consts.AnnotationBuilding] } if building { event = types.DeploymentScaleBlockEvent req.EventMessage = "Deployment is building image, scale is blocked" replicas = 0 } if r.eventEnabled { // Only create event when the first time scale up/down if req.Attempt == 0 { err = r.eventRecorder.CreateDeploymentEvent(namespace, deployment.Name, event, req.EventMessage) if err != nil { return err } } } deployment.Spec.Replicas = &replicas r.logger.WithField("deployment", deployment.Name). WithField("namespace", namespace). WithField("replicas", replicas).Debug("scaling deployment") if _, err = r.kubeClient.AppsV1().Deployments(namespace). Update(ctx, deployment, metav1.UpdateOptions{}); err != nil { return errdefs.System(err) } return nil } ================================================ FILE: agent/pkg/runtime/inference_update.go ================================================ package runtime import ( "context" "github.com/tensorchord/openmodelz/modelzetes/pkg/apis/modelzetes/v2alpha1" inferenceclientset "github.com/tensorchord/openmodelz/modelzetes/pkg/client/clientset/versioned" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/tensorchord/openmodelz/agent/api/types" "github.com/tensorchord/openmodelz/agent/errdefs" ) func (r generalRuntime) InferenceUpdate(ctx context.Context, namespace string, req types.InferenceDeployment, event string) (err error) { if r.eventEnabled { err := r.eventRecorder.CreateDeploymentEvent(namespace, req.Spec.Name, event, req.Status.EventMessage) if err != nil { return err } } if err = updateInference(ctx, namespace, r.inferenceClient, req); err != nil { return err } return nil } func updateInference( ctx context.Context, functionNamespace string, inferenceClient inferenceclientset.Interface, request types.InferenceDeployment) (err error) { actual, err := inferenceClient.TensorchordV2alpha1(). Inferences(functionNamespace).Get( ctx, request.Spec.Name, metav1.GetOptions{}) if err != nil { if k8serrors.IsNotFound(err) { return errdefs.NotFound(err) } else { return errdefs.System(err) } } expected := actual.DeepCopy() if request.Spec.Image != "" { expected.Spec.Image = request.Spec.Image } if request.Spec.Scaling != nil { expected.Spec.Scaling = &v2alpha1.ScalingConfig{ MinReplicas: request.Spec.Scaling.MinReplicas, MaxReplicas: request.Spec.Scaling.MaxReplicas, TargetLoad: request.Spec.Scaling.TargetLoad, ZeroDuration: request.Spec.Scaling.ZeroDuration, StartupDuration: request.Spec.Scaling.StartupDuration, } if request.Spec.Scaling.Type != nil { expected.Spec.Scaling.Type = new(v2alpha1.ScalingType) *expected.Spec.Scaling.Type = v2alpha1.ScalingType(*request.Spec.Scaling.Type) } } if request.Spec.EnvVars != nil { expected.Spec.EnvVars = request.Spec.EnvVars } if request.Spec.Secrets != nil { expected.Spec.Secrets = request.Spec.Secrets } if request.Spec.Constraints != nil { expected.Spec.Constraints = request.Spec.Constraints } if request.Spec.Labels != nil { expected.Spec.Labels = request.Spec.Labels } if request.Spec.Annotations != nil { expected.Spec.Annotations = request.Spec.Annotations } if request.Spec.Resources != nil { rr, err := createResources(request) if err != nil { return errdefs.InvalidParameter(err) } expected.Spec.Resources = &rr } if _, err := inferenceClient.TensorchordV2alpha1(). Inferences(functionNamespace).Update( ctx, expected, metav1.UpdateOptions{}); err != nil { if k8serrors.IsNotFound(err) { return errdefs.NotFound(err) } else { return errdefs.System(err) } } return nil } ================================================ FILE: agent/pkg/runtime/mock/mock.go ================================================ // Code generated by MockGen. DO NOT EDIT. // Source: pkg/runtime/runtime.go // Package mock is a generated GoMock package. package mock import ( context "context" reflect "reflect" gin "github.com/gin-gonic/gin" gomock "github.com/golang/mock/gomock" types "github.com/tensorchord/openmodelz/agent/api/types" config "github.com/tensorchord/openmodelz/agent/pkg/config" v2alpha1 "github.com/tensorchord/openmodelz/modelzetes/pkg/apis/modelzetes/v2alpha1" ) // MockRuntime is a mock of Runtime interface. type MockRuntime struct { ctrl *gomock.Controller recorder *MockRuntimeMockRecorder } // MockRuntimeMockRecorder is the mock recorder for MockRuntime. type MockRuntimeMockRecorder struct { mock *MockRuntime } // NewMockRuntime creates a new mock instance. func NewMockRuntime(ctrl *gomock.Controller) *MockRuntime { mock := &MockRuntime{ctrl: ctrl} mock.recorder = &MockRuntimeMockRecorder{mock} return mock } // EXPECT returns an object that allows the caller to indicate expected use. func (m *MockRuntime) EXPECT() *MockRuntimeMockRecorder { return m.recorder } // BuildCreate mocks base method. func (m *MockRuntime) BuildCreate(ctx context.Context, req types.Build, inference *v2alpha1.Inference, builderImage, buildkitdAddress, buildCtlBin, secret string) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "BuildCreate", ctx, req, inference, builderImage, buildkitdAddress, buildCtlBin, secret) ret0, _ := ret[0].(error) return ret0 } // BuildCreate indicates an expected call of BuildCreate. func (mr *MockRuntimeMockRecorder) BuildCreate(ctx, req, inference, builderImage, buildkitdAddress, buildCtlBin, secret interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BuildCreate", reflect.TypeOf((*MockRuntime)(nil).BuildCreate), ctx, req, inference, builderImage, buildkitdAddress, buildCtlBin, secret) } // BuildGet mocks base method. func (m *MockRuntime) BuildGet(ctx context.Context, namespace, buildName string) (types.Build, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "BuildGet", ctx, namespace, buildName) ret0, _ := ret[0].(types.Build) ret1, _ := ret[1].(error) return ret0, ret1 } // BuildGet indicates an expected call of BuildGet. func (mr *MockRuntimeMockRecorder) BuildGet(ctx, namespace, buildName interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BuildGet", reflect.TypeOf((*MockRuntime)(nil).BuildGet), ctx, namespace, buildName) } // BuildList mocks base method. func (m *MockRuntime) BuildList(ctx context.Context, namespace string) ([]types.Build, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "BuildList", ctx, namespace) ret0, _ := ret[0].([]types.Build) ret1, _ := ret[1].(error) return ret0, ret1 } // BuildList indicates an expected call of BuildList. func (mr *MockRuntimeMockRecorder) BuildList(ctx, namespace interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BuildList", reflect.TypeOf((*MockRuntime)(nil).BuildList), ctx, namespace) } // GetClusterInfo mocks base method. func (m *MockRuntime) GetClusterInfo(cluster *types.ManagedCluster) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetClusterInfo", cluster) ret0, _ := ret[0].(error) return ret0 } // GetClusterInfo indicates an expected call of GetClusterInfo. func (mr *MockRuntimeMockRecorder) GetClusterInfo(cluster interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetClusterInfo", reflect.TypeOf((*MockRuntime)(nil).GetClusterInfo), cluster) } // ImageCacheCreate mocks base method. func (m *MockRuntime) ImageCacheCreate(ctx context.Context, req types.ImageCache, inference *v2alpha1.Inference) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "ImageCacheCreate", ctx, req, inference) ret0, _ := ret[0].(error) return ret0 } // ImageCacheCreate indicates an expected call of ImageCacheCreate. func (mr *MockRuntimeMockRecorder) ImageCacheCreate(ctx, req, inference interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ImageCacheCreate", reflect.TypeOf((*MockRuntime)(nil).ImageCacheCreate), ctx, req, inference) } // InferenceCreate mocks base method. func (m *MockRuntime) InferenceCreate(ctx context.Context, req types.InferenceDeployment, cfg config.IngressConfig, event string, serverPort int) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "InferenceCreate", ctx, req, cfg, event, serverPort) ret0, _ := ret[0].(error) return ret0 } // InferenceCreate indicates an expected call of InferenceCreate. func (mr *MockRuntimeMockRecorder) InferenceCreate(ctx, req, cfg, event, serverPort interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InferenceCreate", reflect.TypeOf((*MockRuntime)(nil).InferenceCreate), ctx, req, cfg, event, serverPort) } // InferenceDelete mocks base method. func (m *MockRuntime) InferenceDelete(ctx context.Context, namespace, inferenceName, ingressNamespace, event string) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "InferenceDelete", ctx, namespace, inferenceName, ingressNamespace, event) ret0, _ := ret[0].(error) return ret0 } // InferenceDelete indicates an expected call of InferenceDelete. func (mr *MockRuntimeMockRecorder) InferenceDelete(ctx, namespace, inferenceName, ingressNamespace, event interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InferenceDelete", reflect.TypeOf((*MockRuntime)(nil).InferenceDelete), ctx, namespace, inferenceName, ingressNamespace, event) } // InferenceExec mocks base method. func (m *MockRuntime) InferenceExec(ctx *gin.Context, namespace, instance string, commands []string, tty bool) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "InferenceExec", ctx, namespace, instance, commands, tty) ret0, _ := ret[0].(error) return ret0 } // InferenceExec indicates an expected call of InferenceExec. func (mr *MockRuntimeMockRecorder) InferenceExec(ctx, namespace, instance, commands, tty interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InferenceExec", reflect.TypeOf((*MockRuntime)(nil).InferenceExec), ctx, namespace, instance, commands, tty) } // InferenceGet mocks base method. func (m *MockRuntime) InferenceGet(namespace, inferenceName string) (*types.InferenceDeployment, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "InferenceGet", namespace, inferenceName) ret0, _ := ret[0].(*types.InferenceDeployment) ret1, _ := ret[1].(error) return ret0, ret1 } // InferenceGet indicates an expected call of InferenceGet. func (mr *MockRuntimeMockRecorder) InferenceGet(namespace, inferenceName interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InferenceGet", reflect.TypeOf((*MockRuntime)(nil).InferenceGet), namespace, inferenceName) } // InferenceGetCRD mocks base method. func (m *MockRuntime) InferenceGetCRD(namespace, name string) (*v2alpha1.Inference, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "InferenceGetCRD", namespace, name) ret0, _ := ret[0].(*v2alpha1.Inference) ret1, _ := ret[1].(error) return ret0, ret1 } // InferenceGetCRD indicates an expected call of InferenceGetCRD. func (mr *MockRuntimeMockRecorder) InferenceGetCRD(namespace, name interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InferenceGetCRD", reflect.TypeOf((*MockRuntime)(nil).InferenceGetCRD), namespace, name) } // InferenceInstanceList mocks base method. func (m *MockRuntime) InferenceInstanceList(namespace, inferenceName string) ([]types.InferenceDeploymentInstance, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "InferenceInstanceList", namespace, inferenceName) ret0, _ := ret[0].([]types.InferenceDeploymentInstance) ret1, _ := ret[1].(error) return ret0, ret1 } // InferenceInstanceList indicates an expected call of InferenceInstanceList. func (mr *MockRuntimeMockRecorder) InferenceInstanceList(namespace, inferenceName interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InferenceInstanceList", reflect.TypeOf((*MockRuntime)(nil).InferenceInstanceList), namespace, inferenceName) } // InferenceList mocks base method. func (m *MockRuntime) InferenceList(namespace string) ([]types.InferenceDeployment, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "InferenceList", namespace) ret0, _ := ret[0].([]types.InferenceDeployment) ret1, _ := ret[1].(error) return ret0, ret1 } // InferenceList indicates an expected call of InferenceList. func (mr *MockRuntimeMockRecorder) InferenceList(namespace interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InferenceList", reflect.TypeOf((*MockRuntime)(nil).InferenceList), namespace) } // InferenceScale mocks base method. func (m *MockRuntime) InferenceScale(ctx context.Context, namespace string, req types.ScaleServiceRequest, inf *types.InferenceDeployment) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "InferenceScale", ctx, namespace, req, inf) ret0, _ := ret[0].(error) return ret0 } // InferenceScale indicates an expected call of InferenceScale. func (mr *MockRuntimeMockRecorder) InferenceScale(ctx, namespace, req, inf interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InferenceScale", reflect.TypeOf((*MockRuntime)(nil).InferenceScale), ctx, namespace, req, inf) } // InferenceUpdate mocks base method. func (m *MockRuntime) InferenceUpdate(ctx context.Context, namespace string, req types.InferenceDeployment, event string) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "InferenceUpdate", ctx, namespace, req, event) ret0, _ := ret[0].(error) return ret0 } // InferenceUpdate indicates an expected call of InferenceUpdate. func (mr *MockRuntimeMockRecorder) InferenceUpdate(ctx, namespace, req, event interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InferenceUpdate", reflect.TypeOf((*MockRuntime)(nil).InferenceUpdate), ctx, namespace, req, event) } // NamespaceCreate mocks base method. func (m *MockRuntime) NamespaceCreate(ctx context.Context, name string) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "NamespaceCreate", ctx, name) ret0, _ := ret[0].(error) return ret0 } // NamespaceCreate indicates an expected call of NamespaceCreate. func (mr *MockRuntimeMockRecorder) NamespaceCreate(ctx, name interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NamespaceCreate", reflect.TypeOf((*MockRuntime)(nil).NamespaceCreate), ctx, name) } // NamespaceDelete mocks base method. func (m *MockRuntime) NamespaceDelete(ctx context.Context, name string) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "NamespaceDelete", ctx, name) ret0, _ := ret[0].(error) return ret0 } // NamespaceDelete indicates an expected call of NamespaceDelete. func (mr *MockRuntimeMockRecorder) NamespaceDelete(ctx, name interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NamespaceDelete", reflect.TypeOf((*MockRuntime)(nil).NamespaceDelete), ctx, name) } // NamespaceGet mocks base method. func (m *MockRuntime) NamespaceGet(ctx context.Context, name string) bool { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "NamespaceGet", ctx, name) ret0, _ := ret[0].(bool) return ret0 } // NamespaceGet indicates an expected call of NamespaceGet. func (mr *MockRuntimeMockRecorder) NamespaceGet(ctx, name interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NamespaceGet", reflect.TypeOf((*MockRuntime)(nil).NamespaceGet), ctx, name) } // NamespaceList mocks base method. func (m *MockRuntime) NamespaceList(ctx context.Context) ([]string, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "NamespaceList", ctx) ret0, _ := ret[0].([]string) ret1, _ := ret[1].(error) return ret0, ret1 } // NamespaceList indicates an expected call of NamespaceList. func (mr *MockRuntimeMockRecorder) NamespaceList(ctx interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NamespaceList", reflect.TypeOf((*MockRuntime)(nil).NamespaceList), ctx) } // ServerDeleteNode mocks base method. func (m *MockRuntime) ServerDeleteNode(ctx context.Context, name string) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "ServerDeleteNode", ctx, name) ret0, _ := ret[0].(error) return ret0 } // ServerDeleteNode indicates an expected call of ServerDeleteNode. func (mr *MockRuntimeMockRecorder) ServerDeleteNode(ctx, name interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ServerDeleteNode", reflect.TypeOf((*MockRuntime)(nil).ServerDeleteNode), ctx, name) } // ServerLabelCreate mocks base method. func (m *MockRuntime) ServerLabelCreate(ctx context.Context, name string, spec types.ServerSpec) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "ServerLabelCreate", ctx, name, spec) ret0, _ := ret[0].(error) return ret0 } // ServerLabelCreate indicates an expected call of ServerLabelCreate. func (mr *MockRuntimeMockRecorder) ServerLabelCreate(ctx, name, spec interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ServerLabelCreate", reflect.TypeOf((*MockRuntime)(nil).ServerLabelCreate), ctx, name, spec) } // ServerList mocks base method. func (m *MockRuntime) ServerList(ctx context.Context) ([]types.Server, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "ServerList", ctx) ret0, _ := ret[0].([]types.Server) ret1, _ := ret[1].(error) return ret0, ret1 } // ServerList indicates an expected call of ServerList. func (mr *MockRuntimeMockRecorder) ServerList(ctx interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ServerList", reflect.TypeOf((*MockRuntime)(nil).ServerList), ctx) } ================================================ FILE: agent/pkg/runtime/namespace.go ================================================ package runtime import ( "context" "fmt" v1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/tensorchord/openmodelz/agent/errdefs" "github.com/tensorchord/openmodelz/modelzetes/pkg/consts" ) func (r generalRuntime) NamespaceList(ctx context.Context) ([]string, error) { ns, err := r.kubeClient.CoreV1().Namespaces().List(ctx, metav1.ListOptions{ LabelSelector: fmt.Sprintf("%s=true", consts.LabelNamespace), }) if err != nil { if k8serrors.IsNotFound(err) { return nil, nil } else { return nil, errdefs.System(err) } } res := make([]string, len(ns.Items)) for i, n := range ns.Items { res[i] = n.Name } return res, nil } func (r generalRuntime) NamespaceCreate(ctx context.Context, name string) error { ns := &v1.Namespace{ ObjectMeta: metav1.ObjectMeta{ Name: name, Labels: map[string]string{ consts.LabelNamespace: "true", }, }, } _, err := r.kubeClient.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{}) if err != nil { if k8serrors.IsAlreadyExists(err) { return errdefs.Conflict(err) } else { return errdefs.System(err) } } return nil } func (r generalRuntime) NamespaceGet(ctx context.Context, name string) bool { _, err := r.kubeClient.CoreV1().Namespaces().Get(ctx, name, metav1.GetOptions{}) if err != nil { return false } return true } func (r generalRuntime) NamespaceDelete(ctx context.Context, name string) error { err := r.kubeClient.CoreV1().Namespaces().Delete(ctx, name, metav1.DeleteOptions{}) return err } ================================================ FILE: agent/pkg/runtime/node.go ================================================ package runtime import ( "context" "github.com/sirupsen/logrus" "github.com/tensorchord/openmodelz/modelzetes/pkg/consts" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func (r generalRuntime) ListServerResource() ([]string, error) { resources := []string{} listOptions := metav1.ListOptions{ LabelSelector: consts.LabelServerResource, } nodes, err := r.kubeClient.CoreV1().Nodes().List(context.Background(), listOptions) if err != nil { logrus.Errorf("failed to list nodes: %v", err) return resources, err } for _, node := range nodes.Items { resources = append(resources, node.Labels[consts.LabelServerResource]) } return resources, nil } ================================================ FILE: agent/pkg/runtime/runtime.go ================================================ package runtime import ( "context" "github.com/gin-gonic/gin" "github.com/sirupsen/logrus" apicorev1 "k8s.io/api/core/v1" appsv1 "k8s.io/client-go/informers/apps/v1" corev1 "k8s.io/client-go/informers/core/v1" "k8s.io/client-go/kubernetes" clientsetscheme "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" kubefledged "github.com/senthilrch/kube-fledged/pkg/client/clientset/versioned" "github.com/tensorchord/openmodelz/agent/api/types" "github.com/tensorchord/openmodelz/agent/pkg/config" "github.com/tensorchord/openmodelz/agent/pkg/event" ingressclient "github.com/tensorchord/openmodelz/ingress-operator/pkg/client/clientset/versioned" "github.com/tensorchord/openmodelz/modelzetes/pkg/apis/modelzetes/v2alpha1" apis "github.com/tensorchord/openmodelz/modelzetes/pkg/apis/modelzetes/v2alpha1" modelzetes "github.com/tensorchord/openmodelz/modelzetes/pkg/apis/modelzetes/v2alpha1" clientset "github.com/tensorchord/openmodelz/modelzetes/pkg/client/clientset/versioned" modelzv2alpha1 "github.com/tensorchord/openmodelz/modelzetes/pkg/client/informers/externalversions/modelzetes/v2alpha1" ) type Runtime interface { // build BuildList(ctx context.Context, namespace string) ([]types.Build, error) BuildCreate(ctx context.Context, req types.Build, inference *v2alpha1.Inference, builderImage, buildkitdAddress, buildCtlBin, secret string) error BuildGet(ctx context.Context, namespace, buildName string) (types.Build, error) // cache ImageCacheCreate(ctx context.Context, req types.ImageCache, inference *modelzetes.Inference) error // inference InferenceCreate(ctx context.Context, req types.InferenceDeployment, cfg config.IngressConfig, event string, serverPort int) error InferenceDelete(ctx context.Context, namespace, inferenceName, ingressNamespace, event string) error InferenceExec(ctx *gin.Context, namespace, instance string, commands []string, tty bool) error InferenceGet(namespace, inferenceName string) (*types.InferenceDeployment, error) InferenceGetCRD(namespace, name string) (*apis.Inference, error) InferenceInstanceList(namespace, inferenceName string) ([]types.InferenceDeploymentInstance, error) InferenceList(namespace string) ([]types.InferenceDeployment, error) InferenceScale(ctx context.Context, namespace string, req types.ScaleServiceRequest, inf *types.InferenceDeployment) error InferenceUpdate(ctx context.Context, namespace string, req types.InferenceDeployment, event string) (err error) // namespace NamespaceList(ctx context.Context) ([]string, error) NamespaceCreate(ctx context.Context, name string) error NamespaceGet(ctx context.Context, name string) bool NamespaceDelete(ctx context.Context, name string) error // server ServerDeleteNode(ctx context.Context, name string) error ServerLabelCreate(ctx context.Context, name string, spec types.ServerSpec) error ServerList(ctx context.Context) ([]types.Server, error) // managed cluster GetClusterInfo(cluster *types.ManagedCluster) error } type generalRuntime struct { endpointsInformer corev1.EndpointsInformer deploymentInformer appsv1.DeploymentInformer inferenceInformer modelzv2alpha1.InferenceInformer podInformer corev1.PodInformer kubeClient kubernetes.Interface clientConfig *rest.Config restClient *rest.RESTClient ingressClient ingressclient.Interface inferenceClient clientset.Interface kubefledgedClient kubefledged.Interface logger *logrus.Entry eventRecorder event.Interface ingressEnabled bool ingressAnyIPToDomain bool eventEnabled bool buildEnabled bool } func New(clientConfig *rest.Config, endpointsInformer corev1.EndpointsInformer, deploymentInformer appsv1.DeploymentInformer, inferenceInformer modelzv2alpha1.InferenceInformer, podInformer corev1.PodInformer, kubeClient kubernetes.Interface, ingressClient ingressclient.Interface, kubefledgedClient kubefledged.Interface, inferenceClient clientset.Interface, eventRecorder event.Interface, ingressEnabled bool, eventEnabled bool, buildEnabled bool, ingressAnyIPToDomain bool, ) (Runtime, error) { r := generalRuntime{ endpointsInformer: endpointsInformer, deploymentInformer: deploymentInformer, inferenceInformer: inferenceInformer, podInformer: podInformer, kubeClient: kubeClient, kubefledgedClient: kubefledgedClient, clientConfig: clientConfig, ingressClient: ingressClient, inferenceClient: inferenceClient, logger: logrus.WithField("component", "runtime"), eventRecorder: eventRecorder, ingressEnabled: ingressEnabled, ingressAnyIPToDomain: ingressAnyIPToDomain, eventEnabled: eventEnabled, buildEnabled: buildEnabled, } // Ref https://github.com/operator-framework/operator-sdk/issues/1570 clientConfig.APIPath = "api" clientConfig.GroupVersion = &apicorev1.SchemeGroupVersion clientConfig.NegotiatedSerializer = clientsetscheme.Codecs r.clientConfig = clientConfig restClient, err := rest.RESTClientFor(clientConfig) if err != nil { return r, err } r.restClient = restClient return r, nil } ================================================ FILE: agent/pkg/runtime/server_delete.go ================================================ package runtime import ( "context" "github.com/tensorchord/openmodelz/agent/errdefs" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func (r generalRuntime) ServerDeleteNode(ctx context.Context, name string) error { err := r.kubeClient.CoreV1().Nodes().Delete(ctx, name, metav1.DeleteOptions{}) if err != nil { if k8serrors.IsNotFound(err) { return errdefs.NotFound(err) } return errdefs.System(err) } return nil } ================================================ FILE: agent/pkg/runtime/server_label_create.go ================================================ package runtime import ( "context" "path/filepath" "github.com/tensorchord/openmodelz/agent/api/types" "github.com/tensorchord/openmodelz/agent/errdefs" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func (r generalRuntime) ServerLabelCreate(ctx context.Context, name string, spec types.ServerSpec) error { node, err := r.kubeClient.CoreV1().Nodes().Get(ctx, name, metav1.GetOptions{}) if err != nil { if k8serrors.IsNotFound(err) { return errdefs.NotFound(err) } else { return errdefs.System(err) } } if len(node.Labels) == 0 { node.Labels = map[string]string{} } for k, v := range spec.Labels { node.Labels[filepath.Join("tensorchord.ai", k)] = v } _, err = r.kubeClient.CoreV1().Nodes().Update(ctx, node, metav1.UpdateOptions{}) if err != nil { if k8serrors.IsNotFound(err) { return errdefs.NotFound(err) } else { return errdefs.System(err) } } return nil } ================================================ FILE: agent/pkg/runtime/server_list.go ================================================ package runtime import ( "context" "strings" "github.com/tensorchord/openmodelz/agent/api/types" "github.com/tensorchord/openmodelz/agent/errdefs" "github.com/tensorchord/openmodelz/agent/pkg/k8s" "github.com/tensorchord/openmodelz/modelzetes/pkg/consts" v1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func (r generalRuntime) ServerList(ctx context.Context) ([]types.Server, error) { nodes, err := r.kubeClient.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) if err != nil { if k8serrors.IsNotFound(err) { return nil, errdefs.NotFound(err) } else { return nil, errdefs.System(err) } } if len(nodes.Items) == 0 { return nil, nil } return getServers(nodes.Items), nil } func getServers(nodes []v1.Node) []types.Server { res := []types.Server{} for _, n := range nodes { res = append(res, getServer(n)) } return res } func getServer(n v1.Node) types.Server { node := types.Server{ Spec: types.ServerSpec{ Name: n.Name, Labels: make(map[string]string), }, Status: types.ServerStatus{ Allocatable: k8s.AsResourceList(n.Status.Allocatable), Capacity: k8s.AsResourceList(n.Status.Capacity), System: types.NodeSystemInfo{ MachineID: n.Status.NodeInfo.MachineID, KernelVersion: n.Status.NodeInfo.KernelVersion, OSImage: n.Status.NodeInfo.OSImage, OperatingSystem: n.Status.NodeInfo.OperatingSystem, Architecture: n.Status.NodeInfo.Architecture, }, }, } for k, v := range n.Labels { if strings.HasPrefix(k, "ai.tensorchord.") { node.Spec.Labels[strings.TrimPrefix(k, "ai.tensorchord.")] = v } if k == consts.LabelServerResource { node.Status.System.ResourceType = v } } phase := "Ready" for _, c := range n.Status.Conditions { if c.Type == v1.NodeReady && c.Status != v1.ConditionTrue { phase = "NotReady" } else if c.Type == v1.NodeDiskPressure && c.Status != v1.ConditionFalse { phase = "DiskPressure" } else if c.Type == v1.NodeMemoryPressure && c.Status != v1.ConditionFalse { phase = "MemoryPressure" } else if c.Type == v1.NodePIDPressure && c.Status != v1.ConditionFalse { phase = "PIDPressure" } else if c.Type == v1.NodeNetworkUnavailable && c.Status != v1.ConditionFalse { phase = "NetworkUnavailable" } } node.Status.Phase = phase return node } ================================================ FILE: agent/pkg/runtime/util_domain.go ================================================ package runtime import ( "fmt" "github.com/dchest/uniuri" ) const ( AnnotationDomain = "ai.tensorchord.domain" ) const ( // stdLen is a standard length of uniuri string to achieve ~95 bits of entropy. stdLen = 16 ) // StdChars is a set of standard characters allowed in uniuri string. var stdChars = []byte("abcdefghijklmnopqrstuvwxyz0123456789") func makeDomain(name, baseDomain string) (string, error) { if baseDomain == "" { return "", fmt.Errorf("base domain is required") } if name == "" { return "", fmt.Errorf("domain name is required") } hash := uniuri.NewLenChars(stdLen, stdChars) return fmt.Sprintf("%s-%s.%s", name, hash, baseDomain), nil } ================================================ FILE: agent/pkg/runtime/util_resource.go ================================================ package runtime import ( "github.com/tensorchord/openmodelz/modelzetes/pkg/consts" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" "github.com/tensorchord/openmodelz/agent/api/types" ) func createResources(request types.InferenceDeployment) (corev1.ResourceRequirements, error) { resources := corev1.ResourceRequirements{ Limits: corev1.ResourceList{}, Requests: corev1.ResourceList{}, } if request.Spec.Resources == nil { return resources, nil } // Set Memory limits if request.Spec.Resources.Limits[types.ResourceMemory] != "" { qty, err := resource.ParseQuantity( string(request.Spec.Resources.Limits[types.ResourceMemory])) if err != nil { return resources, err } resources.Limits[corev1.ResourceMemory] = qty } if request.Spec.Resources.Requests[types.ResourceMemory] != "" { qty, err := resource.ParseQuantity( string(request.Spec.Resources.Requests[types.ResourceMemory])) if err != nil { return resources, err } resources.Requests[corev1.ResourceMemory] = qty } // Set CPU limits if request.Spec.Resources.Limits[types.ResourceCPU] != "" { qty, err := resource.ParseQuantity( string(request.Spec.Resources.Limits[types.ResourceCPU])) if err != nil { return resources, err } resources.Limits[corev1.ResourceCPU] = qty } if request.Spec.Resources.Requests[types.ResourceCPU] != "" { qty, err := resource.ParseQuantity( string(request.Spec.Resources.Requests[types.ResourceCPU])) if err != nil { return resources, err } resources.Requests[corev1.ResourceCPU] = qty } // Set GPU limits if request.Spec.Resources.Limits[types.ResourceGPU] != "" { qty, err := resource.ParseQuantity( string(request.Spec.Resources.Limits[types.ResourceGPU])) if err != nil { return resources, err } resources.Limits[consts.ResourceNvidiaGPU] = qty } if request.Spec.Resources.Requests[types.ResourceGPU] != "" { qty, err := resource.ParseQuantity( string(request.Spec.Resources.Requests[types.ResourceGPU])) if err != nil { return resources, err } resources.Requests[consts.ResourceNvidiaGPU] = qty } return resources, nil } ================================================ FILE: agent/pkg/scaling/function_scaler.go ================================================ package scaling import ( "context" "fmt" "sync" "time" "github.com/dgraph-io/ristretto" "github.com/sirupsen/logrus" "github.com/tensorchord/openmodelz/agent/api/types" "github.com/tensorchord/openmodelz/agent/pkg/runtime" ) const ( maxPollCount = 1000 retries = 20 pollInterval = time.Millisecond * 100 ) // InferenceScaler create a new scaler with the specified // ScalingConfig func NewInferenceScaler(r runtime.Runtime, defaultTTL time.Duration) (*InferenceScaler, error) { cache, err := ristretto.NewCache(&ristretto.Config{ NumCounters: 1e7, MaxCost: 1 << 28, BufferItems: 64, }) if err != nil { return nil, err } return &InferenceScaler{ cache: *cache, runtime: r, defaultTTL: defaultTTL, }, nil } // InferenceScaler scales from zero type InferenceScaler struct { cache ristretto.Cache mu sync.RWMutex runtime runtime.Runtime defaultTTL time.Duration } // FunctionScaleResult holds the result of scaling from zero type FunctionScaleResult struct { Available bool Error error Found bool Duration time.Duration } func (s *InferenceScaler) get( namespace, inferenceName string) (ServiceQueryResponse, error) { key := inferenceName + "." + namespace s.mu.RLock() raw, exit := s.cache.Get(key) s.mu.RUnlock() if exit { return raw.(ServiceQueryResponse), nil } s.mu.Lock() defer s.mu.Unlock() raw, exit = s.cache.Get(key) if exit { return raw.(ServiceQueryResponse), nil } // The wasn't a hit, or there were no available replicas found // so query the live endpoint inf, err := s.runtime.InferenceGet(namespace, inferenceName) if err != nil { return ServiceQueryResponse{}, err } sqr, err := AsServerQueryResponse(inf) if err != nil { return ServiceQueryResponse{}, err } if sqr == nil { return ServiceQueryResponse{}, fmt.Errorf("unable to get service query response") } s.cache.SetWithTTL(key, *sqr, 1, s.defaultTTL) return *sqr, nil } // Scale scales a function from zero replicas to 1 or the value set in // the minimum replicas metadata func (s *InferenceScaler) Scale(ctx context.Context, namespace, inferenceName string) FunctionScaleResult { start := time.Now() resp, err := s.get(namespace, inferenceName) if err != nil { return FunctionScaleResult{ Error: err, Available: false, Found: false, Duration: time.Since(start), } } // Check if there are available replicas in the live data if resp.AvailableReplicas > 0 { return FunctionScaleResult{ Error: nil, Available: true, Found: true, Duration: time.Since(start), } } // If the desired replica count is 0, then a scale up event // is required. if resp.Replicas == 0 { // If the max replicas is 0, then the function is not // scalable if resp.MaxReplicas == 0 { return FunctionScaleResult{ Error: fmt.Errorf("unable to scale up %s, max replicas is 0", inferenceName), Available: false, Found: true, Duration: time.Since(start), } } minReplicas := uint64(1) if resp.MinReplicas > 0 { minReplicas = resp.MinReplicas } // In a retry-loop, first query desired replicas, then // set them if the value is still at 0. scaleResult := Retry(func(attempt int) error { inf, err := s.runtime.InferenceGet(namespace, inferenceName) if err != nil { return err } // The scale up is complete because the desired replica count // has been set to 1 or more. if inf.Status.Replicas > 0 { return nil } // Request a scale up to the minimum amount of replicas if err := s.runtime.InferenceScale(ctx, namespace, types.ScaleServiceRequest{ ServiceName: inferenceName, Replicas: minReplicas, EventMessage: fmt.Sprintf("scale up to replicas %d", minReplicas), Attempt: attempt, }, inf); err != nil { return err } logrus.WithField("inference", inferenceName). WithField("replicas", minReplicas). Debug("scaling up inference") return nil }, "Scale", retries, pollInterval) if scaleResult != nil { return FunctionScaleResult{ Error: scaleResult, Available: false, Found: true, Duration: time.Since(start), } } } switch resp.Framework { // Return early for prototype frameworks. case "gradio", "streamlit": return FunctionScaleResult{ Error: nil, Available: false, Found: true, Duration: time.Since(start), } } // Holding pattern for at least one function replica to be available for i := 0; i < maxPollCount; i++ { inf, err := s.runtime.InferenceGet(namespace, inferenceName) if err != nil { return FunctionScaleResult{ Error: err, Available: false, Found: true, Duration: time.Since(start), } } totalTime := time.Since(start) if inf.Status.AvailableReplicas > 0 { logrus.Debugf("[Ready] function=%s waited for - %.4fs", inferenceName, totalTime.Seconds()) return FunctionScaleResult{ Error: nil, Available: true, Found: true, Duration: totalTime, } } time.Sleep(pollInterval) } return FunctionScaleResult{ Error: nil, Available: true, Found: true, Duration: time.Since(start), } } ================================================ FILE: agent/pkg/scaling/ranges.go ================================================ package scaling import "time" type ScaleType string const ( // DefaultMinReplicas is the minimal amount of replicas for a service. DefaultMinReplicas = 1 // DefaultMaxReplicas is the amount of replicas a service will auto-scale up to. DefaultMaxReplicas = 5 DefaultZeroDuration = 3 * time.Minute // DefaultScalingFactor is the defining proportion for the scaling increments. DefaultScalingFactor = 10 ScaleTypeRPS ScaleType = "rps" ScaleTypeCapacity ScaleType = "capacity" // MinScaleLabel label indicating min scale for a Inference MinScaleLabel = "ai.tensorchord.scale.min" // MaxScaleLabel label indicating max scale for a Inference MaxScaleLabel = "ai.tensorchord.scale.max" // ScalingFactorLabel label indicates the scaling factor for a Inference ScalingFactorLabel = "ai.tensorchord.scale.factor" // TargetLoadLabel label indicates the target load for a Inference TargetLoadLabel = "ai.tensorchord.scale.target" // ZeroDurationLabel label indicates the zero duration for a Inference ZeroDurationLabel = "ai.tensorchord.scale.zero-duration" // ScaleTypeLabel label indicates the scale type for a Inference ScaleTypeLabel = "ai.tensorchord.scale.type" FrameworkLabel = "ai.tensorchord.framework" ) ================================================ FILE: agent/pkg/scaling/retry.go ================================================ package scaling import ( "log" "time" ) type routine func(attempt int) error func Retry(r routine, label string, attempts int, interval time.Duration) error { var err error for i := 0; i < attempts; i++ { res := r(i) if res != nil { err = res log.Printf("[%s]: %d/%d, error: %s\n", label, i, attempts, res) } else { err = nil break } time.Sleep(interval) } return err } ================================================ FILE: agent/pkg/scaling/service_query.go ================================================ // Copyright (c) OpenFaaS Author(s). All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. package scaling import "time" // ServiceQuery provides interface for replica querying/setting type ServiceQuery interface { GetReplicas(service, namespace string) (response ServiceQueryResponse, err error) SetReplicas(service, namespace string, count uint64) error } // ServiceQueryResponse response from querying a function status type ServiceQueryResponse struct { Framework string TargetLoad uint64 ZeroDuration time.Duration Replicas uint64 MaxReplicas uint64 MinReplicas uint64 ScalingFactor uint64 AvailableReplicas uint64 Annotations map[string]string } ================================================ FILE: agent/pkg/scaling/util.go ================================================ package scaling import ( "time" "github.com/tensorchord/openmodelz/agent/api/types" ) func AsServerQueryResponse(inf *types.InferenceDeployment) (*ServiceQueryResponse, error) { if inf == nil { return nil, nil } res := ServiceQueryResponse{} res.Replicas = uint64(inf.Status.Replicas) res.Annotations = inf.Spec.Annotations res.AvailableReplicas = uint64(inf.Status.AvailableReplicas) res.Framework = string(inf.Spec.Framework) res.MinReplicas = uint64(*inf.Spec.Scaling.MinReplicas) res.MaxReplicas = uint64(*inf.Spec.Scaling.MaxReplicas) res.TargetLoad = uint64(*inf.Spec.Scaling.TargetLoad) res.ZeroDuration = time.Duration(*inf.Spec.Scaling.ZeroDuration) * time.Second return &res, nil } ================================================ FILE: agent/pkg/server/error.go ================================================ package server import ( "bytes" "fmt" "net/http" "github.com/tensorchord/openmodelz/agent/errdefs" ) // Error defines a standard application error. type Error struct { // Machine-readable error code. HTTPStatusCode int `json:"http_status_code,omitempty"` // Human-readable message. Message string `json:"message,omitempty"` Request string `json:"request,omitempty"` // Logical operation and nested error. Op string `json:"op,omitempty"` Err error `json:"error,omitempty"` } // Error returns the string representation of the error message. func (e *Error) Error() string { var buf bytes.Buffer // Print the current operation in our stack, if any. if e.Op != "" { fmt.Fprintf(&buf, "%s: ", e.Op) } // If wrapping an error, print its Error() message. // Otherwise print the error code & message. if e.Err != nil { buf.WriteString(e.Err.Error()) } else { if e.HTTPStatusCode != 0 { fmt.Fprintf(&buf, "<%s> ", http.StatusText(e.HTTPStatusCode)) } buf.WriteString(e.Message) } return buf.String() } func NewError(code int, err error, op string) error { return &Error{ HTTPStatusCode: code, Err: err, Message: err.Error(), Op: op, } } func errFromErrDefs(err error, op string) error { if errdefs.IsCancelled(err) { return NewError(http.StatusRequestTimeout, err, op) } else if errdefs.IsConflict(err) { return NewError(http.StatusConflict, err, op) } else if errdefs.IsDataLoss(err) { return NewError(http.StatusInternalServerError, err, op) } else if errdefs.IsDeadline(err) { return NewError(http.StatusRequestTimeout, err, op) } else if errdefs.IsForbidden(err) { return NewError(http.StatusForbidden, err, op) } else if errdefs.IsInvalidParameter(err) { return NewError(http.StatusBadRequest, err, op) } else if errdefs.IsNotFound(err) { return NewError(http.StatusNotFound, err, op) } else if errdefs.IsNotImplemented(err) { return NewError(http.StatusNotImplemented, err, op) } else if errdefs.IsNotModified(err) { return NewError(http.StatusNotModified, err, op) } else if errdefs.IsSystem(err) { return NewError(http.StatusInternalServerError, err, op) } else if errdefs.IsUnauthorized(err) { return NewError(http.StatusUnauthorized, err, op) } else if errdefs.IsUnavailable(err) { return NewError(http.StatusServiceUnavailable, err, op) } else if errdefs.IsUnknown(err) { return NewError(http.StatusInternalServerError, err, op) } return NewError(http.StatusInternalServerError, err, op) } ================================================ FILE: agent/pkg/server/handler_build_create.go ================================================ package server import ( "net/http" "github.com/gin-gonic/gin" "github.com/sirupsen/logrus" "github.com/tensorchord/openmodelz/agent/api/types" ) // @Summary Create the build. // @Description Create the build. // @Tags build // @Accept json // @Produce json // @Param body body types.Build true "build" // @Success 200 {object} types.Build // @Router /system/build [post] func (s *Server) handleBuildCreate(c *gin.Context) error { var req types.Build if err := c.ShouldBindJSON(&req); err != nil { return NewError( http.StatusBadRequest, err, "build-create") } if err := s.validator.ValidateBuildRequest(&req); err != nil { return NewError( http.StatusBadRequest, err, "build-create") } s.validator.DefaultBuildRequest(&req) inference, err := s.runtime.InferenceGetCRD(req.Spec.Namespace, req.Spec.Name) if err != nil { return errFromErrDefs(err, "inference-instance-list") } if err := s.runtime.BuildCreate(c.Request.Context(), req, inference, s.config.Build.BuilderImage, s.config.Build.BuildkitdAddress, s.config.Build.BuildCtlBin, s.config.Build.BuildImagePullSecret); err != nil { logrus.Errorf("failed to create build: %v", err) return errFromErrDefs(err, "build-create") } c.JSON(http.StatusOK, req) return nil } ================================================ FILE: agent/pkg/server/handler_build_get.go ================================================ package server import ( "errors" "net/http" "github.com/gin-gonic/gin" _ "github.com/tensorchord/openmodelz/agent/api/types" ) // @Summary Get the build by name. // @Description Get the build by name. // @Tags build // @Accept json // @Produce json // @Param namespace query string true "Namespace" // @Param name path string true "inference id" // @Success 200 {object} types.Build // @Router /system/build/{name} [get] func (s *Server) handleBuildGet(c *gin.Context) error { namespace := c.Query("namespace") if namespace == "" { return NewError( http.StatusBadRequest, errors.New("namespace is required"), "inference-list") } name := c.Param("name") if name == "" { return NewError( http.StatusBadRequest, errors.New("name is required"), "build-get") } build, err := s.runtime.BuildGet(c.Request.Context(), namespace, name) if err != nil { return errFromErrDefs(err, "build-get") } c.JSON(http.StatusOK, build) return nil } ================================================ FILE: agent/pkg/server/handler_build_list.go ================================================ package server import ( "errors" "net/http" "github.com/gin-gonic/gin" _ "github.com/tensorchord/openmodelz/agent/api/types" ) // @Summary List the builds. // @Description List the builds. // @Tags build // @Accept json // @Produce json // @Param namespace query string true "Namespace" // @Success 200 {object} []types.Build // @Router /system/build [get] func (s *Server) handleBuildList(c *gin.Context) error { namespace := c.Query("namespace") if namespace == "" { return NewError( http.StatusBadRequest, errors.New("namespace is required"), "inference-list") } builds, err := s.runtime.BuildList(c.Request.Context(), namespace) if err != nil { return errFromErrDefs(err, "build-list") } c.JSON(http.StatusOK, builds) return nil } ================================================ FILE: agent/pkg/server/handler_build_logs.go ================================================ package server import ( "github.com/gin-gonic/gin" _ "github.com/tensorchord/openmodelz/agent/api/types" ) // @Summary Get the build logs. // @Description Get the build logs. // @Tags log // @Accept json // @Produce json // @Param namespace query string true "Namespace" // @Param name query string true "Build Name" // @Param instance query string false "Instance" // @Param tail query int false "Tail" // @Param follow query bool false "Follow" // @Param since query string false "Since" // @Success 200 {object} []types.Message // @Router /system/logs/build [get] func (s *Server) handleBuildLogs(c *gin.Context) error { return s.getLogsFromRequester(c, s.buildLogRequester) } ================================================ FILE: agent/pkg/server/handler_gradio_proxy.go ================================================ package server import ( "fmt" "io" "net" "net/http" "net/http/httputil" "net/url" "path" "strconv" "time" "github.com/gin-gonic/gin" "github.com/sirupsen/logrus" "github.com/tensorchord/openmodelz/agent/pkg/consts" "github.com/tensorchord/openmodelz/agent/pkg/server/static" ) // @Summary Reverse proxy to the backend gradio. // @Description Reverse proxy to the backend gradio. // @Tags inference // @Accept */* // @Produce json // @Param id path string true "Deployment ID" // @Router /gradio/{id} [get] // @Router /gradio/{id} [post] // @Success 201 func (s *Server) proxyGradio(c *gin.Context) error { remote, err := url.Parse(fmt.Sprintf("http://0.0.0.0:%d", s.config.Server.ServerPort)) if err != nil { return err } proxy := httputil.NewSingleHostReverseProxy(remote) proxy.Transport = &http.Transport{ Proxy: http.ProxyFromEnvironment, DialContext: (&net.Dialer{ Timeout: s.config.ModelZCloud.UpstreamTimeout, KeepAlive: s.config.ModelZCloud.UpstreamTimeout, DualStack: true, }).DialContext, MaxIdleConns: s.config.ModelZCloud.MaxIdleConnections, MaxIdleConnsPerHost: s.config.ModelZCloud.MaxIdleConnectionsPerHost, IdleConnTimeout: 90 * time.Second, TLSHandshakeTimeout: 10 * time.Second, ExpectContinueTimeout: 1 * time.Second, } uid, deployment, err := s.proxyNoAuth(c) if err != nil { return err } ns := consts.DefaultPrefix + uid proxy.Director = func(req *http.Request) { req.Header = c.Request.Header req.Host = remote.Host req.URL.Scheme = remote.Scheme req.URL.Host = remote.Host req.URL.Path = path.Join( "/", "inference", fmt.Sprintf("%s.%s", deployment, ns), c.Param("proxyPath")) logrus.WithFields(logrus.Fields{ "deployment": deployment, "uid": uid, "ns": ns, "path": req.URL.Path, "remote": remote.String(), }).Debug("proxying to gradio") } proxy.ModifyResponse = func(resp *http.Response) error { // http.StatusSeeOther indicates that the server is still loading. if resp.StatusCode == http.StatusSeeOther { resp.StatusCode = http.StatusOK instances, err := s.runtime.InferenceInstanceList(ns, deployment) if err != nil { return NewError(http.StatusInternalServerError, err, "instance-list") } buf, err := static.RenderDeploymentLoadingPage("gradio", resp.Header.Get("X-Call-Id"), "We are currently processing your request.", deployment, instances) if err != nil { return NewError(http.StatusInternalServerError, err, "render-loading-page") } resp.Body = io.NopCloser(buf) resp.ContentLength = int64(buf.Len()) resp.Header.Set("Content-Length", strconv.Itoa(buf.Len())) resp.Header.Set("Content-Type", "text/html") resp.StatusCode = http.StatusServiceUnavailable } return nil } proxy.ServeHTTP(c.Writer, c.Request) return nil } ================================================ FILE: agent/pkg/server/handler_healthz.go ================================================ package server import ( "net/http" "github.com/gin-gonic/gin" ) // @Summary Healthz // @Description Healthz // @Tags system // @Accept json // @Produce json // @Success 200 // @Router /healthz [get] func (s *Server) handleHealthz(c *gin.Context) error { c.Status(http.StatusOK) return nil } ================================================ FILE: agent/pkg/server/handler_healthz_test.go ================================================ package server import ( "github.com/gin-gonic/gin" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) var _ = Describe("healthz", func() { BeforeEach(func() { server = &Server{ router: gin.New(), metricsRouter: gin.New(), runtime: mockRuntime, } }) It("healthz", func() { c := mkContext("GET", "/", nil, nil) err := server.handleHealthz(c) Expect(err).NotTo(HaveOccurred()) }) }) ================================================ FILE: agent/pkg/server/handler_image_cache.go ================================================ package server import ( "net/http" "github.com/gin-gonic/gin" "github.com/tensorchord/openmodelz/agent/api/types" ) // @Summary Create the image cache. // @Description Create the image cache. // @Tags image-cache // @Accept json // @Produce json // @Param body body types.ImageCache true "image-cache" // @Success 201 {object} types.ImageCache // @Router /system/image-cache [post] func (s *Server) handleImageCacheCreate(c *gin.Context) error { var req types.ImageCache if err := c.ShouldBindJSON(&req); err != nil { return NewError( http.StatusBadRequest, err, "image-cache-create") } if err := s.validator.ValidateImageCacheRequest(&req); err != nil { return NewError( http.StatusBadRequest, err, "image-cache-create") } inference, err := s.runtime.InferenceGetCRD(req.Namespace, req.Name) if err != nil { return errFromErrDefs(err, "inference-instance-list") } if err := s.runtime.ImageCacheCreate(c.Request.Context(), req, inference); err != nil { return errFromErrDefs(err, "image-cache-create") } c.JSON(http.StatusOK, req) return nil } ================================================ FILE: agent/pkg/server/handler_inference_create.go ================================================ package server import ( "fmt" "net/http" "github.com/gin-gonic/gin" "github.com/tensorchord/openmodelz/agent/api/types" "github.com/tensorchord/openmodelz/agent/client" ) // @Summary Create the inferences. // @Description Create the inferences. // @Tags inference // @Accept json // @Produce json // @Param request body types.InferenceDeployment true "query params" // @Success 201 {object} types.InferenceDeployment // @Router /system/inferences [post] func (s *Server) handleInferenceCreate(c *gin.Context) error { event := types.DeploymentCreateEvent var req types.InferenceDeployment if err := c.ShouldBindJSON(&req); err != nil { return NewError(http.StatusBadRequest, err, event) } if s.config.ModelZCloud.Enabled { ns := req.Spec.Namespace user, err := client.GetUserIDFromNamespace(ns) if err != nil { return err } else if user == "" { return fmt.Errorf("user id is empty") } s.cache.SetWithTTL(req.Spec.Name, user, 1, 0) exist := s.runtime.NamespaceGet(c.Request.Context(), ns) if !exist { if err := s.runtime.NamespaceCreate(c.Request.Context(), ns); err != nil { return err } } } // Set the default values. s.validator.DefaultDeployRequest(&req) // Validate the request. if err := s.validator.ValidateDeployRequest(&req); err != nil { return NewError(http.StatusBadRequest, err, event) } // Create the inference. if err := s.runtime.InferenceCreate(c.Request.Context(), req, s.config.Ingress, event, s.config.Server.ServerPort); err != nil { return errFromErrDefs(err, event) } c.JSON(http.StatusCreated, req) return nil } ================================================ FILE: agent/pkg/server/handler_inference_create_test.go ================================================ package server import ( "github.com/gin-gonic/gin" "github.com/golang/mock/gomock" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/tensorchord/openmodelz/agent/api/types" "github.com/tensorchord/openmodelz/agent/pkg/server/validator" . "github.com/tensorchord/openmodelz/modelzetes/pkg/pointer" ) var _ = Describe("inference create", func() { BeforeEach(func() { server = &Server{ router: gin.New(), metricsRouter: gin.New(), runtime: mockRuntime, validator: validator.New(), } }) It("invalid request - nil", func() { c := mkContext("GET", "/", nil, nil) err := server.handleInferenceCreate(c) Expect(err).To(HaveOccurred()) }) It("invalid request - empty", func() { c := mkJsonBodyContext("GET", "/", nil, types.InferenceDeployment{}) err := server.handleInferenceCreate(c) Expect(err).To(HaveOccurred()) }) It("good request", func() { mockRuntime.EXPECT().InferenceCreate(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(nil) c := mkJsonBodyContext("GET", "/", nil, types.InferenceDeployment{ Spec: types.InferenceDeploymentSpec{ Name: "abc", Image: "mock-image", Port: Ptr(int32(123)), }, }) err := server.handleInferenceCreate(c) Expect(err).NotTo(HaveOccurred()) }) }) ================================================ FILE: agent/pkg/server/handler_inference_delete.go ================================================ package server import ( "errors" "net/http" "github.com/gin-gonic/gin" "github.com/tensorchord/openmodelz/agent/api/types" ) // @Summary Delete the inferences. // @Description Delete the inferences. // @Tags inference // @Accept json // @Produce json // @Param request body types.DeleteFunctionRequest true "query params" // @Param namespace query string true "Namespace" // @Success 202 {object} types.DeleteFunctionRequest // @Router /system/inferences [delete] func (s *Server) handleInferenceDelete(c *gin.Context) error { event := types.DeploymentDeleteEvent var req types.DeleteFunctionRequest if err := c.ShouldBindJSON(&req); err != nil { return NewError(http.StatusBadRequest, err, event) } namespace := c.Query("namespace") if namespace == "" { return NewError( http.StatusBadRequest, errors.New("namespace is required"), event) } if req.FunctionName == "" { return NewError( http.StatusBadRequest, errors.New("function name is required"), event) } if err := s.runtime.InferenceDelete(c.Request.Context(), namespace, req.FunctionName, s.config.Ingress.Namespace, event); err != nil { return errFromErrDefs(err, event) } c.JSON(http.StatusAccepted, req) return nil } ================================================ FILE: agent/pkg/server/handler_inference_delete_test.go ================================================ package server import ( "github.com/gin-gonic/gin" "github.com/golang/mock/gomock" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/tensorchord/openmodelz/agent/api/types" "github.com/tensorchord/openmodelz/agent/pkg/server/validator" ) var _ = Describe("inference delete", func() { BeforeEach(func() { server = &Server{ router: gin.New(), metricsRouter: gin.New(), runtime: mockRuntime, validator: validator.New(), } }) It("invalid request - nil", func() { c := mkContext("GET", "/", nil, nil) err := server.handleInferenceDelete(c) Expect(err).To(HaveOccurred()) }) It("invalid request - empty", func() { c := mkJsonBodyContext("GET", "/", nil, types.DeleteFunctionRequest{}) err := server.handleInferenceDelete(c) Expect(err).To(HaveOccurred()) }) It("good request", func() { mockRuntime.EXPECT().InferenceDelete(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(nil) c := mkJsonBodyContext("GET", "/", nil, types.DeleteFunctionRequest{ FunctionName: "mock-inference", }) setQuery(c, map[string]string{"namespace": "mock-namespace"}) err := server.handleInferenceDelete(c) Expect(err).NotTo(HaveOccurred()) }) }) ================================================ FILE: agent/pkg/server/handler_inference_get.go ================================================ package server import ( "errors" "net/http" "github.com/gin-gonic/gin" _ "github.com/tensorchord/openmodelz/agent/api/types" ) // @Summary Get the inference by name. // @Description Get the inference by name. // @Tags inference // @Accept json // @Produce json // @Param namespace query string true "Namespace" // @Param name path string true "inference id" // @Success 200 {object} types.InferenceDeployment // @Router /system/inference/{name} [get] func (s *Server) handleInferenceGet(c *gin.Context) error { namespace := c.Query("namespace") if namespace == "" { return NewError( http.StatusBadRequest, errors.New("namespace is required"), "inference-get") } name := c.Param("name") if name == "" { return NewError( http.StatusBadRequest, errors.New("name is required"), "inference-get") } function, err := s.runtime.InferenceGet(namespace, name) if err != nil { return errFromErrDefs(err, "inference-get") } c.JSON(http.StatusOK, function) return nil } ================================================ FILE: agent/pkg/server/handler_inference_get_test.go ================================================ package server import ( "errors" "github.com/gin-gonic/gin" "github.com/golang/mock/gomock" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/tensorchord/openmodelz/agent/api/types" "github.com/tensorchord/openmodelz/agent/pkg/server/validator" . "github.com/tensorchord/openmodelz/modelzetes/pkg/pointer" ) var _ = Describe("inference get", func() { BeforeEach(func() { server = &Server{ router: gin.New(), metricsRouter: gin.New(), runtime: mockRuntime, validator: validator.New(), } }) It("invalid request - no namespace", func() { c := mkContext("GET", "/", nil, nil) err := server.handleInferenceGet(c) Expect(err).To(HaveOccurred()) }) It("invalid request - no name", func() { c := mkJsonBodyContext("GET", "/", nil, nil) setQuery(c, map[string]string{"namespace": "mock-namespace"}) err := server.handleInferenceGet(c) Expect(err).To(HaveOccurred()) }) It("invalid request - mock error", func() { mockRuntime.EXPECT().InferenceGet(gomock.Any(), gomock.Any()).Times(1).Return(nil, errors.New("mock-error")) c := mkJsonBodyContext("GET", "/", nil, nil) setQuery(c, map[string]string{"namespace": "mock-namespace"}) setParam(c, map[string]string{"name": "mock-name"}) err := server.handleInferenceGet(c) Expect(err).To(HaveOccurred()) }) It("good request", func() { mockRuntime.EXPECT().InferenceGet(gomock.Any(), gomock.Any()).Times(1).Return(Ptr(types.InferenceDeployment{}), nil) c := mkJsonBodyContext("GET", "/", nil, nil) setQuery(c, map[string]string{"namespace": "mock-namespace"}) setParam(c, map[string]string{"name": "mock-name"}) err := server.handleInferenceGet(c) Expect(err).NotTo(HaveOccurred()) }) }) ================================================ FILE: agent/pkg/server/handler_inference_instance.go ================================================ package server import ( "errors" "net/http" "github.com/gin-gonic/gin" _ "github.com/tensorchord/openmodelz/agent/api/types" ) // @Summary List the inference instances. // @Description List the inference instances. // @Tags inference // @Accept json // @Produce json // @Param namespace query string true "Namespace" // @Param name path string true "Name" // @Success 200 {object} []types.InferenceDeployment // @Router /system/inference/{name}/instances [get] func (s *Server) handleInferenceInstance(c *gin.Context) error { namespace := c.Query("namespace") if namespace == "" { return NewError(http.StatusBadRequest, errors.New("namespace is required"), "inference-instance-list") } name := c.Param("name") if name == "" { return NewError(http.StatusBadRequest, errors.New("name is required"), "inference-instance-list") } instances, err := s.runtime.InferenceInstanceList(namespace, name) if err != nil { return errFromErrDefs(err, "inference-instance-list") } c.JSON(http.StatusOK, instances) return nil } ================================================ FILE: agent/pkg/server/handler_inference_instance_exec.go ================================================ package server import ( "errors" "net/http" "strconv" "strings" "github.com/gin-gonic/gin" _ "github.com/tensorchord/openmodelz/agent/api/types" ) // @Summary Attach to the inference instance. // @Description Attach to the inference instance. // @Tags inference // @Accept json // @Produce json // @Param namespace query string true "Namespace" // @Param name path string true "Name" // @Param instance path string true "Instance name" // @Success 200 {object} []types.InferenceDeployment // @Router /system/inference/{name}/instance/{instance} [post] func (s *Server) handleInferenceInstanceExec(c *gin.Context) error { namespace := c.Query("namespace") if namespace == "" { return NewError(http.StatusBadRequest, errors.New("namespace is required"), "inference-instance-list") } name := c.Param("name") if name == "" { return NewError(http.StatusBadRequest, errors.New("name is required"), "inference-instance-list") } instance := c.Param("instance") if name == "" { return NewError(http.StatusBadRequest, errors.New("instance is required"), "inference-instance-list") } tty := c.Query("tty") if tty == "" { tty = "false" } ttyBoolean, err := strconv.ParseBool(tty) if err != nil { return NewError(http.StatusBadRequest, err, "inference-instance-exec") } command := c.Query("command") commandSlice := strings.Split(command, ",") if err := s.runtime.InferenceExec( c, namespace, instance, commandSlice, ttyBoolean); err != nil { return errFromErrDefs(err, "inference-instance-exec") } return nil } ================================================ FILE: agent/pkg/server/handler_inference_list.go ================================================ package server import ( "errors" "net/http" "github.com/gin-gonic/gin" _ "github.com/tensorchord/openmodelz/agent/api/types" ) // @Summary List the inferences. // @Description List the inferences. // @Tags inference // @Accept json // @Produce json // @Param namespace query string true "Namespace" // @Success 200 {object} []types.InferenceDeployment // @Router /system/inferences [get] func (s *Server) handleInferenceList(c *gin.Context) error { namespace := c.Query("namespace") if namespace == "" { return NewError( http.StatusBadRequest, errors.New("namespace is required"), "inference-list") } inferenes, err := s.runtime.InferenceList(namespace) if err != nil { return errFromErrDefs(err, "inference-list") } // Add invocation count metrics into the body. // TODO: https://github.com/tensorchord/openmodelz/issues/203 s.prometheusClient.AddMetrics(inferenes) c.JSON(http.StatusOK, inferenes) return nil } ================================================ FILE: agent/pkg/server/handler_inference_logs.go ================================================ package server import ( "context" "encoding/json" "net/http" "time" "github.com/cockroachdb/errors" "github.com/gin-gonic/gin" "github.com/tensorchord/openmodelz/agent/api/types" "github.com/tensorchord/openmodelz/agent/pkg/log" ) // @Summary Get the inference logs. // @Description Get the inference logs. // @Tags log // @Accept json // @Produce json // @Param namespace query string true "Namespace" // @Param name query string true "Name" // @Param instance query string false "Instance" // @Param tail query int false "Tail" // @Param follow query bool false "Follow" // @Param since query string false "Since" // @Param end query string false "End" // @Success 200 {object} []types.Message // @Router /system/logs/inference [get] func (s *Server) handleInferenceLogs(c *gin.Context) error { return s.getLogsFromRequester(c, s.deploymentLogRequester) } func (s Server) getLogsFromRequester(c *gin.Context, requester log.Requester) error { cn, ok := c.Writer.(http.CloseNotifier) if !ok { return NewError(http.StatusNotFound, errors.New("LogHandler: response is not a CloseNotifier, required for streaming response"), "log-get") } flusher, ok := c.Writer.(http.Flusher) if !ok { return NewError(http.StatusNotFound, errors.New("LogHandler: response is not a Flusher, required for streaming response"), "log-get") } var req types.LogRequest if err := c.ShouldBindQuery(&req); err != nil { return NewError(http.StatusBadRequest, err, "log-get") } _ = cn timeout := s.config.Inference.LogTimeout if req.Follow { // use a much larger timeout for streaming log timeout = time.Hour } ctx, cancelQuery := context.WithTimeout(c.Request.Context(), timeout) defer cancelQuery() messages, err := requester.Query(ctx, req) if err != nil { return errFromErrDefs(err, "log-get") } // Send the initial headers saying we're gonna stream the response. c.Header("Content-Type", "application/x-ndjson") c.Header("Transfer-Encoding", "chunked") c.Header("Connection", "Keep-Alive") flusher.Flush() defer flusher.Flush() defer c.Writer.Write([]byte{}) defer flusher.Flush() jsonEncoder := json.NewEncoder(c.Writer) for messages != nil { select { case <-cn.CloseNotify(): s.logger.WithField("req", req). Debug("client closed connection") return nil case msg, ok := <-messages: if !ok { s.logger.WithField("req", req). Debug("log stream closed") messages = nil return nil } // serialize and write the msg to the http ResponseWriter err := jsonEncoder.Encode(msg) if err != nil { // can't actually write the status header here so we should json serialize an error // and return that because we have already sent the content type and status code s.logger.WithError(err).Error("LogHandler: failed to serialize log message") // write json error message here ? jsonEncoder.Encode(types.Message{Text: "failed to serialize log message"}) flusher.Flush() return nil } flusher.Flush() } } return nil } ================================================ FILE: agent/pkg/server/handler_inference_proxy.go ================================================ package server import ( "errors" "fmt" "net" "net/http" "net/http/httputil" "strconv" "strings" "time" "github.com/gin-gonic/gin" "github.com/prometheus/client_golang/prometheus" "github.com/tensorchord/openmodelz/agent/errdefs" ) // @Summary Inference. // @Description Inference proxy. // @Tags inference-proxy // @Accept json // @Produce json // @Param name path string true "inference id" // @Router /inference/{name} [post] // @Router /inference/{name} [get] // @Router /inference/{name} [put] // @Router /inference/{name} [delete] // @Success 200 // @Failure 303 // @Failure 400 // @Failure 404 // @Failure 500 func (s *Server) handleInferenceProxy(c *gin.Context) error { namespacedName := c.Param("name") if namespacedName == "" { return NewError( http.StatusBadRequest, errors.New("name is required"), "inference-proxy") } namespace, name, err := getNamespaceAndName(namespacedName) if err != nil { return NewError( http.StatusBadRequest, err, "inference-proxy") } // Update metrics. s.metricsOptions.GatewayInferenceInvocationStarted. WithLabelValues(namespacedName).Inc() s.metricsOptions.GatewayInferenceInvocationInflight. WithLabelValues(namespacedName).Inc() start := time.Now() label := prometheus.Labels{"inference_name": namespacedName, "code": strconv.Itoa(http.StatusProcessing)} defer func() { s.metricsOptions.GatewayInferenceInvocationInflight. WithLabelValues(namespacedName).Dec() s.metricsOptions.GatewayInferencesHistogram.With(label). Observe(time.Since(start).Seconds()) s.metricsOptions.GatewayInferenceInvocation.With(label).Inc() }() res := s.scaler.Scale(c.Request.Context(), namespace, name) if !res.Found { label["code"] = strconv.Itoa(http.StatusNotFound) return NewError( http.StatusNotFound, errors.New("inference not found"), "inference-proxy") } else if res.Error != nil { label["code"] = strconv.Itoa(http.StatusInternalServerError) return NewError( http.StatusInternalServerError, res.Error, "inference-proxy") } if res.Available { statusCode, err := s.forward(c, namespace, name) if err != nil { label["code"] = strconv.Itoa(statusCode) return NewError(statusCode, err, "inference-proxy") } label["code"] = strconv.Itoa(statusCode) return nil } else { // The inference is still being created. label["code"] = strconv.Itoa(http.StatusSeeOther) return NewError(http.StatusSeeOther, fmt.Errorf("inference %s is not available", name), "inference-proxy") } } func (s *Server) forward(c *gin.Context, namespace, name string) (int, error) { backendURL, err := s.endpointResolver.Resolve(namespace, name) if err != nil { return 0, errdefs.InvalidParameter(err) } defer s.endpointResolver.Close(backendURL) proxyServer := httputil.ReverseProxy{} proxyServer.Transport = &http.Transport{ Proxy: http.ProxyFromEnvironment, DialContext: (&net.Dialer{ Timeout: s.config.Server.ReadTimeout, KeepAlive: s.config.Server.ReadTimeout, DualStack: true, }).DialContext, } proxyServer.Director = func(req *http.Request) { targetQuery := backendURL.RawQuery req.URL.Scheme = backendURL.Scheme req.URL.Host = backendURL.Host if targetQuery == "" || req.URL.RawQuery == "" { req.URL.RawQuery = targetQuery + req.URL.RawQuery } else { req.URL.RawQuery = targetQuery + "&" + req.URL.RawQuery } req.URL.Path = c.Param("proxyPath") if req.URL.Path == "" { req.URL.Path = "/" } s.logger.WithField("url", backendURL.String()). WithField("path", req.URL.Path). WithField("header", req.Header). WithField("raw-query", req.URL.RawQuery).Debug("reverse proxy") } var statusCode int proxyServer.ModifyResponse = func(resp *http.Response) error { statusCode = resp.StatusCode return nil } proxyServer.ServeHTTP(c.Writer, c.Request) return statusCode, nil } func getNamespaceAndName(name string) (string, string, error) { if !strings.Contains(name, ".") { return "", "", fmt.Errorf("name is not namespaced") } namespace := name[strings.LastIndexAny(name, ".")+1:] infName := strings.TrimSuffix(name, "."+namespace) if namespace == "" { return "", "", fmt.Errorf("namespace is empty") } if infName == "" { return "", "", fmt.Errorf("inference name is empty") } return namespace, infName, nil } ================================================ FILE: agent/pkg/server/handler_inference_scale.go ================================================ package server import ( "errors" "net/http" "github.com/gin-gonic/gin" "github.com/tensorchord/openmodelz/agent/api/types" ) // @Summary Scale the inferences. // @Description Scale the inferences. // @Tags inference // @Accept json // @Produce json // @Param namespace query string true "Namespace" // @Param request body types.ScaleServiceRequest true "query params" // @Success 202 {object} []types.ScaleServiceRequest // @Failure 400 // @Router /system/scale-inference [post] func (s *Server) handleInferenceScale(c *gin.Context) error { var req types.ScaleServiceRequest if err := c.ShouldBindJSON(&req); err != nil { return NewError( http.StatusBadRequest, err, "inference-scale") } namespace := c.Query("namespace") if namespace == "" { return NewError( http.StatusBadRequest, errors.New("namespace is required"), "inference-scale") } inf, err := s.runtime.InferenceGet(namespace, req.ServiceName) if err != nil { return errFromErrDefs(err, "inference-scale") } if err := s.runtime.InferenceScale(c.Request.Context(), namespace, req, inf); err != nil { return errFromErrDefs(err, "inference-scale") } c.JSON(http.StatusAccepted, req) return nil } ================================================ FILE: agent/pkg/server/handler_inference_update.go ================================================ package server import ( "errors" "net/http" "github.com/gin-gonic/gin" "github.com/tensorchord/openmodelz/agent/api/types" ) // @Summary Update the inferences. // @Description Update the inferences. // @Tags inference // @Accept json // @Produce json // @Param request body types.InferenceDeployment true "query params" // @Param namespace query string true "Namespace" // @Success 202 {object} types.InferenceDeployment // @Router /system/inferences [put] func (s *Server) handleInferenceUpdate(c *gin.Context) error { event := types.DeploymentUpdateEvent var req types.InferenceDeployment if err := c.ShouldBindJSON(&req); err != nil { return NewError(http.StatusBadRequest, err, event) } namespace := c.Query("namespace") if namespace == "" { return NewError( http.StatusBadRequest, errors.New("namespace is required"), event) } if err := s.validator.ValidateDeployRequest(&req); err != nil { return NewError(http.StatusBadRequest, err, event) } if err := s.runtime.InferenceUpdate(c.Request.Context(), namespace, req, event); err != nil { return errFromErrDefs(err, event) } c.JSON(http.StatusAccepted, req) return nil } ================================================ FILE: agent/pkg/server/handler_info.go ================================================ package server import ( "net/http" "github.com/gin-gonic/gin" "github.com/tensorchord/openmodelz/agent/api/types" "github.com/tensorchord/openmodelz/agent/pkg/version" ) // @Summary Get system info. // @Description Get system info. // @Tags system // @Accept json // @Produce json // @Success 200 {object} types.ProviderInfo // @Router /system/info [get] func (s *Server) handleInfo(c *gin.Context) error { v := version.GetVersion() c.JSON(http.StatusOK, types.ProviderInfo{ Name: "agent", Orchestration: "kubernetes", Version: &types.VersionInfo{ Version: v.Version, BuildDate: v.BuildDate, GitCommit: v.GitCommit, GitTag: v.GitTag, GitTreeState: v.GitTreeState, GoVersion: v.GoVersion, Compiler: v.Compiler, Platform: v.Platform, }, }) return nil } ================================================ FILE: agent/pkg/server/handler_mosec_proxy.go ================================================ package server import ( "fmt" "path" "github.com/gin-gonic/gin" "github.com/tensorchord/openmodelz/agent/pkg/consts" ) // @Summary Proxy to the backend mosec. // @Description Proxy to the backend mosec. // @Tags inference // @Accept */* // @Produce json // @Param id path string true "Deployment ID" // @Router /mosec/{id} [get] // @Router /mosec/{id}/metrics [get] // @Router /mosec/{id}/inference [post] // @Success 201 func (s *Server) proxyMosec(c *gin.Context) error { uid, deployment, err := s.proxyAuth(c) if err != nil { return err } c.Request.URL.Path = path.Join( "/", "inference", fmt.Sprintf("%s.%s", deployment, consts.DefaultPrefix+uid), c.Param("proxyPath")) return s.handleInferenceProxy(c) } ================================================ FILE: agent/pkg/server/handler_namespace_create.go ================================================ package server import ( "net/http" "github.com/gin-gonic/gin" "github.com/tensorchord/openmodelz/agent/api/types" ) // @Summary Create the namespace. // @Description Create the namespace. // @Tags namespace // @Accept json // @Produce json // @Param body body types.NamespaceRequest true "Namespace name" // @Success 200 {object} types.NamespaceRequest // @Router /system/namespaces [post] func (s *Server) handleNamespaceCreate(c *gin.Context) error { var req types.NamespaceRequest if err := c.ShouldBindJSON(&req); err != nil { return NewError(http.StatusBadRequest, err, "namespace-create") } if err := s.runtime.NamespaceCreate(c.Request.Context(), req.Name); err != nil { return errFromErrDefs(err, "namespace-create") } c.JSON(http.StatusOK, req) return nil } ================================================ FILE: agent/pkg/server/handler_namespace_delete.go ================================================ package server import ( "net/http" "github.com/gin-gonic/gin" "github.com/tensorchord/openmodelz/agent/api/types" ) // @Summary Delete the namespace. // @Description Delete the namespace. // @Tags namespace // @Accept json // @Produce json // @Param body body types.NamespaceRequest true "Namespace name" // @Success 200 {object} types.NamespaceRequest // @Router /system/namespaces [delete] func (s *Server) handleNamespaceDelete(c *gin.Context) error { var req types.NamespaceRequest if err := c.ShouldBindJSON(&req); err != nil { return NewError(http.StatusBadRequest, err, "namespace-delete") } if err := s.runtime.NamespaceDelete(c.Request.Context(), req.Name); err != nil { return errFromErrDefs(err, "namespace-delete") } c.JSON(http.StatusOK, req) return nil } ================================================ FILE: agent/pkg/server/handler_namespace_delete_test.go ================================================ package server import ( "errors" "github.com/gin-gonic/gin" "github.com/golang/mock/gomock" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/tensorchord/openmodelz/agent/api/types" "github.com/tensorchord/openmodelz/agent/pkg/server/validator" ) var _ = Describe("namespace delete", func() { BeforeEach(func() { server = &Server{ router: gin.New(), metricsRouter: gin.New(), runtime: mockRuntime, validator: validator.New(), } }) It("invalid request - nil", func() { c := mkContext("GET", "/", nil, nil) err := server.handleNamespaceDelete(c) Expect(err).To(HaveOccurred()) }) It("invalid request - mock error", func() { mockRuntime.EXPECT().NamespaceDelete(gomock.Any(), gomock.Any()).Times(1).Return(errors.New("mock-error")) c := mkJsonBodyContext("GET", "/", nil, types.NamespaceRequest{ Name: "mock-ns", }) err := server.handleNamespaceDelete(c) Expect(err).To(HaveOccurred()) }) It("good request", func() { mockRuntime.EXPECT().NamespaceDelete(gomock.Any(), gomock.Any()).Times(1).Return(nil) c := mkJsonBodyContext("GET", "/", nil, types.NamespaceRequest{ Name: "mock-ns", }) err := server.handleNamespaceDelete(c) Expect(err).NotTo(HaveOccurred()) }) }) ================================================ FILE: agent/pkg/server/handler_namespace_list.go ================================================ package server import ( "net/http" "github.com/gin-gonic/gin" ) // @Summary List the namespaces. // @Description List the namespaces. // @Tags namespace // @Accept json // @Produce json // @Success 200 {object} []string // @Router /system/namespaces [get] func (s *Server) handleNamespaceList(c *gin.Context) error { ns, err := s.runtime.NamespaceList(c.Request.Context()) if err != nil { return errFromErrDefs(err, "namespace-list") } c.JSON(http.StatusOK, ns) return nil } ================================================ FILE: agent/pkg/server/handler_other_proxy.go ================================================ package server import ( "fmt" "net" "net/http" "net/http/httputil" "net/url" "path" "time" "github.com/gin-gonic/gin" "github.com/tensorchord/openmodelz/agent/pkg/consts" ) // @Summary Reverse proxy to the backend other. // @Description Reverse proxy to the backend other. // @Tags inference // @Accept */* // @Produce json // @Param id path string true "Deployment ID" // @Router /other/{id} [get] // @Router /other/{id} [post] // @Success 201 func (s *Server) proxyOther(c *gin.Context) error { remote, err := url.Parse(fmt.Sprintf("http://0.0.0.0:%d", s.config.Server.ServerPort)) if err != nil { return err } proxy := httputil.NewSingleHostReverseProxy(remote) proxy.Transport = &http.Transport{ Proxy: http.ProxyFromEnvironment, DialContext: (&net.Dialer{ Timeout: s.config.ModelZCloud.UpstreamTimeout, KeepAlive: s.config.ModelZCloud.UpstreamTimeout, DualStack: true, }).DialContext, MaxIdleConns: s.config.ModelZCloud.MaxIdleConnections, MaxIdleConnsPerHost: s.config.ModelZCloud.MaxIdleConnectionsPerHost, IdleConnTimeout: 90 * time.Second, TLSHandshakeTimeout: 10 * time.Second, ExpectContinueTimeout: 1 * time.Second, } uid, deployment, err := s.proxyAuth(c) if err != nil { return err } proxy.Director = func(req *http.Request) { req.Header = c.Request.Header req.Host = remote.Host req.URL.Scheme = remote.Scheme req.URL.Host = remote.Host req.URL.Path = path.Join( "/", "inference", fmt.Sprintf("%s.%s", deployment, consts.DefaultPrefix+uid), c.Param("proxyPath")) } proxy.ServeHTTP(c.Writer, c.Request) return nil } ================================================ FILE: agent/pkg/server/handler_root.go ================================================ package server import ( "github.com/gin-gonic/gin" "github.com/tensorchord/openmodelz/agent/pkg/server/static" ) func (s *Server) handleRoot(c *gin.Context) error { lp, err := static.RenderLoadingPage() if err != nil { return err } c.Data(200, "text/html; charset=utf-8", lp.Bytes()) return nil } ================================================ FILE: agent/pkg/server/handler_server_delete.go ================================================ package server import ( "errors" "net/http" "github.com/gin-gonic/gin" ) // @Summary Delete a node from the cluster. // @Description Delete a node. // @Tags namespace // @Param name path string true "Server Name" // @Accept json // @Produce json // @Success 200 // @Router /system/server/{name}/delete [delete] func (s *Server) handleServerDelete(c *gin.Context) error { name := c.Param("name") if name == "" { return NewError(http.StatusBadRequest, errors.New("name is required"), "server-delete-node") } err := s.runtime.ServerDeleteNode(c.Request.Context(), name) if err != nil { return errFromErrDefs(err, "server-delete-node") } return nil } ================================================ FILE: agent/pkg/server/handler_server_label_create.go ================================================ package server import ( "errors" "net/http" "github.com/gin-gonic/gin" "github.com/tensorchord/openmodelz/agent/api/types" ) // @Summary List the servers. // @Description List the servers. // @Tags namespace // @Param name path string true "Server Name" // @Param request body types.ServerSpec true "query params" // @Accept json // @Produce json // @Success 200 {object} []string // @Router /system/server/{name}/labels [post] func (s *Server) handleServerLabelCreate(c *gin.Context) error { name := c.Param("name") if name == "" { return NewError(http.StatusBadRequest, errors.New("name is required"), "server-label-create") } var req types.ServerSpec if err := c.ShouldBindJSON(&req); err != nil { return NewError(http.StatusBadRequest, err, "server-label-create") } err := s.runtime.ServerLabelCreate(c.Request.Context(), name, req) if err != nil { return errFromErrDefs(err, "namespace-list") } c.JSON(http.StatusOK, req) return nil } ================================================ FILE: agent/pkg/server/handler_server_list.go ================================================ package server import ( "net/http" "github.com/gin-gonic/gin" "github.com/tensorchord/openmodelz/agent/api/types" ) // @Summary List the servers. // @Description List the servers. // @Tags namespace // @Accept json // @Produce json // @Success 200 {object} []types.Server // @Router /system/servers [get] func (s *Server) handleServerList(c *gin.Context) error { ns := []types.Server{} ns, err := s.runtime.ServerList(c.Request.Context()) if err != nil { return errFromErrDefs(err, "namespace-list") } c.JSON(http.StatusOK, ns) return nil } ================================================ FILE: agent/pkg/server/handler_streamlit_proxy.go ================================================ package server import ( "fmt" "io" "net" "net/http" "net/http/httputil" "net/url" "path" "strconv" "time" "github.com/gin-gonic/gin" "github.com/sirupsen/logrus" "github.com/tensorchord/openmodelz/agent/pkg/consts" "github.com/tensorchord/openmodelz/agent/pkg/server/static" ) // @Summary Reverse proxy to streamlit. // @Description Reverse proxy to streamlit. // @Tags inference // @Accept */* // @Produce json // @Param id path string true "Deployment ID" // @Router /streamlit/{id} [get] // @Router /streamlit/{id} [post] // @Success 201 func (s *Server) proxyStreamlit(c *gin.Context) error { remote, err := url.Parse(fmt.Sprintf("http://0.0.0.0:%d", s.config.Server.ServerPort)) if err != nil { return err } proxy := httputil.NewSingleHostReverseProxy(remote) proxy.Transport = &http.Transport{ Proxy: http.ProxyFromEnvironment, DialContext: (&net.Dialer{ Timeout: s.config.ModelZCloud.UpstreamTimeout, KeepAlive: s.config.ModelZCloud.UpstreamTimeout, DualStack: true, }).DialContext, MaxIdleConns: s.config.ModelZCloud.MaxIdleConnections, MaxIdleConnsPerHost: s.config.ModelZCloud.MaxIdleConnectionsPerHost, IdleConnTimeout: 90 * time.Second, TLSHandshakeTimeout: 10 * time.Second, ExpectContinueTimeout: 1 * time.Second, } uid, deployment, err := s.proxyNoAuth(c) if err != nil { return err } ns := consts.DefaultPrefix + uid proxy.Director = func(req *http.Request) { req.Header = c.Request.Header req.Host = remote.Host req.URL.Scheme = remote.Scheme req.URL.Host = remote.Host req.URL.Path = path.Join( "/", "inference", fmt.Sprintf("%s.%s", deployment, ns), c.Param("proxyPath")) logrus.WithFields(logrus.Fields{ "deployment": deployment, "uid": uid, "ns": ns, "path": req.URL.Path, "remote": remote.String(), }).Debug("proxying to streamlit") } proxy.ModifyResponse = func(resp *http.Response) error { if resp.StatusCode == http.StatusSeeOther { resp.StatusCode = http.StatusOK instances, err := s.runtime.InferenceInstanceList(ns, deployment) if err != nil { return NewError(http.StatusInternalServerError, err, "instance-list") } buf, err := static.RenderDeploymentLoadingPage("streamlit", resp.Header.Get("X-Call-Id"), "We are currently processing your request.", deployment, instances) if err != nil { return NewError(http.StatusInternalServerError, err, "render-loading-page") } resp.Body = io.NopCloser(buf) resp.ContentLength = int64(buf.Len()) resp.Header.Set("Content-Length", strconv.Itoa(buf.Len())) resp.Header.Set("Content-Type", "text/html") resp.StatusCode = http.StatusServiceUnavailable } return nil } proxy.ServeHTTP(c.Writer, c.Request) return nil } ================================================ FILE: agent/pkg/server/middleware_callid.go ================================================ package server import ( "fmt" "time" "github.com/gin-gonic/gin" "github.com/google/uuid" ) func (s Server) middlewareCallID(c *gin.Context) error { start := time.Now() if len(c.Request.Header.Get("X-Call-Id")) == 0 { callID := uuid.New().String() c.Request.Header.Add("X-Call-Id", callID) c.Writer.Header().Add("X-Call-Id", callID) } c.Request.Header.Add("X-Start-Time", fmt.Sprintf("%d", start.UTC().UnixNano())) c.Writer.Header().Add("X-Start-Time", fmt.Sprintf("%d", start.UTC().UnixNano())) c.Next() return nil } ================================================ FILE: agent/pkg/server/proxy_auth.go ================================================ package server import ( "context" "fmt" "strings" "github.com/gin-gonic/gin" "github.com/sirupsen/logrus" "github.com/tensorchord/openmodelz/agent/errdefs" "github.com/tensorchord/openmodelz/agent/pkg/consts" ) func (s *Server) proxyAuth(c *gin.Context) (string, string, error) { var uid string var valid bool deployment := c.Param("id") if len(deployment) == 0 { return "", "", errFromErrDefs( fmt.Errorf("cannot find the deployment name in %s", c.Request.RequestURI), "get-deployment") } key := c.GetHeader("X-API-Key") // Be compatible with the OpenAI API. rawKeyStr := c.GetHeader("Authorization") logrus.Debug("proxyOther: key: ", key, ", rawKeyStr: ", rawKeyStr) if s.validateUnifiedKey(key) { // uid 0 means to use unified api key uid = "00000000-0000-0000-0000-000000000000" } else if len(key) > 0 { uid, valid = s.validateAPIKey(key) if !valid { return "", "", errdefs.Unauthorized(fmt.Errorf("invalid API key")) } } else if len(rawKeyStr) > 0 { strs := strings.Split(rawKeyStr, " ") if len(strs) != 2 { return "", "", errdefs.Unauthorized(fmt.Errorf("invalid Authorization API key")) } if strs[0] != "Bearer" { return "", "", errdefs.Unauthorized(fmt.Errorf("invalid Authorization API key")) } uid, valid = s.validateAPIKey(strs[1]) if !valid { return "", "", errdefs.Unauthorized(fmt.Errorf("invalid Authorization API key")) } } if len(uid) == 0 { return "", "", errdefs.Unauthorized(fmt.Errorf("invalid API key")) } return uid, deployment, nil } func (s *Server) proxyNoAuth(c *gin.Context) (string, string, error) { deployment := c.Param("id") if len(deployment) == 0 { return "", "", errdefs.InvalidParameter( fmt.Errorf("cannot find the deployment name in %s", c.Request.RequestURI)) } uid, found := s.getUIDFromDeploymentID(c.Request.Context(), deployment) if !found { return "", "", errdefs.InvalidParameter( fmt.Errorf("cannot find the user id from the deployment id")) } return uid, deployment, nil } func (s *Server) validateAPIKey(key string) (string, bool) { if !strings.HasPrefix(key, consts.APIKEY_PREFIX) { return "", false } apikeys := s.config.ModelZCloud.APIKeys uid, exit := apikeys[key] if exit { return uid, true } apiServerReady := make(chan struct{}) go func() { if err := s.modelzCloudClient.WaitForAPIServerReady(); err != nil { logrus.Fatalf("failed to wait for apiserver ready: %v", err) } close(apiServerReady) }() // Get from apiserver apikeys, err := s.modelzCloudClient.GetAPIKeys(context.Background(), apiServerReady, s.config.ModelZCloud.AgentToken, s.config.ModelZCloud.ID) if err != nil { logrus.Errorf("failed to get apikeys: %v", err) return "", false } uid, exit = apikeys[key] if exit { return uid, true } return "", false } func (s *Server) validateUnifiedKey(key string) bool { if !strings.HasPrefix(key, consts.APIKEY_PREFIX) { return false } if len(s.config.ModelZCloud.UnifiedAPIKey) != 0 && s.config.ModelZCloud.UnifiedAPIKey == key { return true } return false } ================================================ FILE: agent/pkg/server/server_factory.go ================================================ package server import ( "net/http" "github.com/dgraph-io/ristretto" "github.com/gin-contrib/cors" "github.com/gin-gonic/gin" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/tensorchord/openmodelz/agent/client" ginlogrus "github.com/toorop/gin-logrus" "github.com/tensorchord/openmodelz/agent/pkg/config" "github.com/tensorchord/openmodelz/agent/pkg/event" "github.com/tensorchord/openmodelz/agent/pkg/k8s" "github.com/tensorchord/openmodelz/agent/pkg/log" "github.com/tensorchord/openmodelz/agent/pkg/metrics" "github.com/tensorchord/openmodelz/agent/pkg/prom" "github.com/tensorchord/openmodelz/agent/pkg/runtime" "github.com/tensorchord/openmodelz/agent/pkg/scaling" "github.com/tensorchord/openmodelz/agent/pkg/server/validator" ) type Server struct { router *gin.Engine metricsRouter *gin.Engine logger *logrus.Entry validator *validator.Validator runtime runtime.Runtime // endpointResolver resolves the requests from the client to the // corresponding inference kubernetes service. endpointResolver k8s.Resolver buildLogRequester log.Requester deploymentLogRequester log.Requester // prometheusClient is the client to query the prometheus server. // It is used in inference list. prometheusClient prom.PrometheusQuery metricsOptions metrics.MetricOptions // scaler scales the inference from 0 to 1. scaler *scaling.InferenceScaler config config.Config eventRecorder event.Interface modelzCloudClient *client.Client cache ristretto.Cache } func New(c config.Config) (Server, error) { router := gin.New() router.Use(ginlogrus.Logger(logrus.StandardLogger(), "/healthz")) router.Use(gin.Recovery()) // metrics server metricsRouter := gin.New() metricsRouter.Use(gin.Recovery()) if gin.Mode() == gin.DebugMode { logrus.SetLevel(logrus.DebugLevel) logrus.Debug("Allow CORS") router.Use(cors.New(cors.Config{ AllowOrigins: []string{"*"}, AllowMethods: []string{"GET", "POST", "PUT", "PATCH", "DELETE"}, AllowHeaders: []string{"*"}, })) } promCli := prom.NewPrometheusQuery(c.Metrics.PrometheusHost, c.Metrics.PrometheusPort, http.DefaultClient) logger := logrus.WithField("component", "server") s := Server{ router: router, metricsRouter: metricsRouter, config: c, logger: logger, validator: validator.New(), prometheusClient: promCli, } cache, err := ristretto.NewCache(&ristretto.Config{ NumCounters: 1e7, MaxCost: 1 << 28, BufferItems: 64, }) if err != nil { return s, err } s.cache = *cache if s.config.ModelZCloud.EventEnabled { logrus.Info("Event recording is enabled") cli, err := client.NewClientWithOpts( client.WithHost(s.config.ModelZCloud.URL)) if err != nil { return s, errors.Wrap(err, "failed to create modelz cloud client") } s.eventRecorder = event.NewEventRecorder(cli, s.config.ModelZCloud.AgentToken) } else { s.eventRecorder = event.NewFake() } s.registerRoutes() s.registerMetricsRoutes() if err := s.initKubernetesResources(); err != nil { return s, err } if c.ModelZCloud.Enabled { err := s.initModelZCloud(c.ModelZCloud.URL, c.ModelZCloud.AgentToken, c.ModelZCloud.Region) if err != nil { return s, err } } if err := s.initMetrics(); err != nil { return s, err } s.initLogs() return s, nil } ================================================ FILE: agent/pkg/server/server_handlerfunc.go ================================================ package server import ( "errors" "net/http" "github.com/gin-gonic/gin" "github.com/sirupsen/logrus" ) type HandlerFunc func(c *gin.Context) error func WrapHandler(handler HandlerFunc) gin.HandlerFunc { return func(c *gin.Context) { err := handler(c) if err != nil { var serverErr *Error if !errors.As(err, &serverErr) { serverErr = &Error{ HTTPStatusCode: http.StatusInternalServerError, Err: err, Message: err.Error(), } } serverErr.Request = c.Request.Method + " " + c.Request.URL.String() if gin.Mode() == "debug" { logrus.Debugf("error: %+v", err) } else { // Remove detailed info when in the release mode serverErr.Op = "" serverErr.Err = nil } c.JSON(serverErr.HTTPStatusCode, serverErr) c.Abort() return } } } ================================================ FILE: agent/pkg/server/server_init_kubernetes.go ================================================ package server import ( "context" "fmt" "time" "github.com/prometheus/client_golang/prometheus" "github.com/sirupsen/logrus" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" kubeinformers "k8s.io/client-go/informers" kubeinformersv1 "k8s.io/client-go/informers/core/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/clientcmd" kubefledged "github.com/senthilrch/kube-fledged/pkg/client/clientset/versioned" "github.com/tensorchord/openmodelz/agent/api/types" "github.com/tensorchord/openmodelz/agent/pkg/event" "github.com/tensorchord/openmodelz/agent/pkg/k8s" "github.com/tensorchord/openmodelz/agent/pkg/log" "github.com/tensorchord/openmodelz/agent/pkg/runtime" "github.com/tensorchord/openmodelz/agent/pkg/scaling" ingressclient "github.com/tensorchord/openmodelz/ingress-operator/pkg/client/clientset/versioned" clientset "github.com/tensorchord/openmodelz/modelzetes/pkg/client/clientset/versioned" informers "github.com/tensorchord/openmodelz/modelzetes/pkg/client/informers/externalversions" "github.com/tensorchord/openmodelz/modelzetes/pkg/consts" "github.com/tensorchord/openmodelz/modelzetes/pkg/signals" ) func (s *Server) initKubernetesResources() error { clientCmdConfig, err := clientcmd.BuildConfigFromFlags( s.config.KubeConfig.MasterURL, s.config.KubeConfig.Kubeconfig) if err != nil { return err } clientCmdConfig.QPS = float32(s.config.KubeConfig.QPS) clientCmdConfig.Burst = s.config.KubeConfig.Burst kubeClient, err := kubernetes.NewForConfig(clientCmdConfig) if err != nil { return err } inferenceClient, err := clientset.NewForConfig(clientCmdConfig) if err != nil { return err } var ingressClient ingressclient.Interface if s.config.Ingress.IngressEnabled { ingressClient, err = ingressclient.NewForConfig(clientCmdConfig) if err != nil { return err } } kubefledgedClient, err := kubefledged.NewForConfig(clientCmdConfig) if err != nil { return err } kubeInformerFactory := kubeinformers.NewSharedInformerFactoryWithOptions( kubeClient, s.config.KubeConfig.ResyncPeriod) inferenceInformerFactory := informers.NewSharedInformerFactoryWithOptions( inferenceClient, s.config.KubeConfig.ResyncPeriod) // set up signals so we handle the first shutdown signal gracefully stopCh := signals.SetupSignalHandler() inferences := inferenceInformerFactory.Tensorchord().V2alpha1().Inferences() go inferences.Informer().Run(stopCh) if ok := cache.WaitForNamedCacheSync( fmt.Sprintf("%s:inferences", consts.ProviderName), stopCh, inferences.Informer().HasSynced); !ok { s.logger.Errorf("failed to wait for cache to sync") } deployments := kubeInformerFactory.Apps().V1().Deployments() go deployments.Informer().Run(stopCh) if ok := cache.WaitForNamedCacheSync( fmt.Sprintf("%s:deployments", consts.ProviderName), stopCh, deployments.Informer().HasSynced); !ok { s.logger.Errorf("failed to wait for cache to sync") } pods := kubeInformerFactory.Core().V1().Pods() s.podStartWatch(pods, kubeClient) go pods.Informer().Run(stopCh) if ok := cache.WaitForNamedCacheSync( fmt.Sprintf("%s:pods", consts.ProviderName), stopCh, pods.Informer().HasSynced); !ok { s.logger.Errorf("failed to wait for cache to sync") } endpoints := kubeInformerFactory.Core().V1().Endpoints() go endpoints.Informer().Run(stopCh) if ok := cache.WaitForNamedCacheSync( fmt.Sprintf("%s:endpoints", consts.ProviderName), stopCh, endpoints.Informer().HasSynced); !ok { s.logger.Errorf("failed to wait for cache to sync") } runtime, err := runtime.New(clientCmdConfig, endpoints, deployments, inferences, pods, kubeClient, ingressClient, kubefledgedClient, inferenceClient, s.eventRecorder, s.config.Ingress.IngressEnabled, s.config.ModelZCloud.EventEnabled, s.config.Build.BuildEnabled, s.config.Ingress.AnyIPToDomain, ) if err != nil { return err } s.runtime = runtime if s.config.Server.Dev { logrus.Warn("running in dev mode, using port forwarding to access pods, please do not use dev mode in production") s.endpointResolver = k8s.NewPortForwardingResolver(clientCmdConfig, kubeClient) } else { s.endpointResolver = k8s.NewEndpointResolver(endpoints.Lister()) } s.deploymentLogRequester = log.NewK8sAPIRequestor(kubeClient) s.scaler, err = scaling.NewInferenceScaler(runtime, s.config.Inference.CacheTTL) if err != nil { return err } if s.scaler == nil { return fmt.Errorf("scaler is nil") } return nil } // podStartWatch log event when pod start began and finished func (s *Server) podStartWatch(pods kubeinformersv1.PodInformer, client *kubernetes.Clientset) { pods.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: func(obj interface{}) { new := obj.(*v1.Pod) controlPlane, exist := new.Annotations[consts.AnnotationControlPlaneKey] // for inference created by modelz apiserver if !exist || controlPlane != consts.ModelzAnnotationValue { return } podWatchEventLog(s.eventRecorder, new, types.PodCreateEvent) start := time.Now() // Ticker will keep watching until pod start or timeout ticker := time.NewTicker(time.Second * 2) timeout := time.After(5 * time.Minute) go func() { for { select { case <-timeout: podWatchEventLog(s.eventRecorder, new, types.PodTimeoutEvent) return case <-ticker.C: pod, err := client.CoreV1().Pods(new.Namespace).Get(context.TODO(), new.Name, metav1.GetOptions{}) if err != nil { logrus.WithFields(logrus.Fields{ "namespace": pod.Namespace, "deployment": pod.Labels["app"], "name": pod.Name, }).Errorf("failed to get pod: %s", err) return } for _, c := range pod.Status.Conditions { if c.Type == v1.PodReady && c.Status == v1.ConditionTrue { podWatchEventLog(s.eventRecorder, pod, types.PodReadyEvent) label := prometheus.Labels{ "inference_name": fmt.Sprintf("%s.%s", pod.Labels["app"], pod.Namespace), "source_image": pod.Annotations[consts.AnnotationDockerImage]} s.metricsOptions.PodStartHistogram.With(label). Observe(time.Since(start).Seconds()) return } } } } }() }, }) } // log status for pod watch status transfer func podWatchEventLog(recorder event.Interface, obj *v1.Pod, event string) { deployment := obj.Labels["app"] err := recorder.CreateDeploymentEvent(obj.Namespace, deployment, event, obj.Name) if err != nil { logrus.WithFields(logrus.Fields{ "namespace": obj.Namespace, "deployment": deployment, "name": obj.Name, "event": event, }).Errorf("failed to create deployment event: %s", err) } } ================================================ FILE: agent/pkg/server/server_init_logs.go ================================================ package server import ( "github.com/tensorchord/openmodelz/agent/pkg/log" ) func (s *Server) initLogs() { if len(s.config.Logs.LokiURL) > 0 { s.logger.Info("enable Loki logs requester") s.buildLogRequester = log.NewLokiAPIRequestor( s.config.Logs.LokiURL, s.config.Logs.LokiUser, s.config.Logs.LokiToken) } } ================================================ FILE: agent/pkg/server/server_init_metrics.go ================================================ package server import ( "context" "github.com/tensorchord/openmodelz/agent/pkg/metrics" ) func (s *Server) initMetrics() error { metricsOptions := metrics.BuildMetricsOptions() s.metricsOptions = metricsOptions exporter := metrics.NewExporter(metricsOptions, s.runtime) metrics.RegisterExporter(exporter) exporter.StartServiceWatcher(context.TODO(), s.config.Metrics.PollingInterval) return nil } ================================================ FILE: agent/pkg/server/server_init_modelz_cloud.go ================================================ package server import ( "context" "fmt" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/tensorchord/openmodelz/agent/api/types" "github.com/tensorchord/openmodelz/agent/client" ) func (s *Server) initModelZCloud(url, token, region string) error { cluster := types.ManagedCluster{ Region: region, PrometheusURL: fmt.Sprintf("%s:%d", s.config.Metrics.PrometheusHost, s.config.Metrics.PrometheusPort), } cli, err := client.NewClientWithOpts( client.WithHost(url)) if err != nil { return errors.Wrap(err, "failed to create modelz cloud client") } s.modelzCloudClient = cli err = s.runtime.GetClusterInfo(&cluster) if err != nil { return errors.Wrap(err, "failed to get managed cluster info") } apiServerReady := make(chan struct{}) go func() { if err := s.modelzCloudClient.WaitForAPIServerReady(); err != nil { logrus.Fatalf("failed to wait for apiserver ready: %v", err) } close(apiServerReady) }() cluster.Status = types.ClusterStatusInit // after init modelz cloud client, register agent err = cli.RegisterAgent(context.Background(), token, &cluster) if err != nil { return errors.Wrap(err, "failed to register agent to modelz cloud") } s.config.ModelZCloud.ID = cluster.ID s.config.ModelZCloud.TokenID = cluster.TokenID s.config.ModelZCloud.Name = cluster.Name apikeys, err := s.modelzCloudClient.GetAPIKeys(context.Background(), apiServerReady, s.config.ModelZCloud.AgentToken, s.config.ModelZCloud.ID) if err != nil { logrus.Errorf("failed to get apikeys: %v", err) } s.config.ModelZCloud.APIKeys = apikeys namespaces, err := s.modelzCloudClient.GetNamespaces(context.Background(), apiServerReady, s.config.ModelZCloud.AgentToken, s.config.ModelZCloud.ID) if err != nil { logrus.Errorf("failed to get namespaces: %v", err) } nss := []string{} for _, ns := range namespaces.Items { nss = append(nss, ns) err = s.runtime.NamespaceCreate(context.Background(), ns) if err != nil { logrus.Errorf("failed to create namespace %s: %v", ns, err) continue } } s.config.ModelZCloud.UserNamespaces = nss return nil } ================================================ FILE: agent/pkg/server/server_init_route.go ================================================ package server import ( "github.com/gin-gonic/gin" swaggerfiles "github.com/swaggo/files" ginSwagger "github.com/swaggo/gin-swagger" _ "github.com/tensorchord/openmodelz/agent/pkg/docs" "github.com/tensorchord/openmodelz/agent/pkg/metrics" ) const ( endpointInferencePlural = "/inferences" endpointInference = "/inference" endpointServerPlural = "/servers" endpointServer = "/server" endpointScaleInference = "/scale-inference" endpointInfo = "/info" endpointLogPlural = "/logs" endpointNamespacePlural = "/namespaces" endpointHealthz = "/healthz" endpointBuild = "/build" endpointImageCache = "/image-cache" ) func (s *Server) registerRoutes() { root := s.router.Group("/") v1 := s.router.Group("/api/v1") // swagger root.GET("/swagger/*any", ginSwagger.WrapHandler(swaggerfiles.Handler)) // dataplane root.Any("/inference/:name", WrapHandler(s.middlewareCallID), WrapHandler(s.handleInferenceProxy)) root.Any("/inference/:name/*proxyPath", WrapHandler(s.middlewareCallID), WrapHandler(s.handleInferenceProxy)) v1.Any("/mosec/:id/*proxyPath", WrapHandler(s.proxyMosec)) v1.Any("/gradio/:id/*proxyPath", WrapHandler(s.proxyGradio)) v1.Any("/streamlit/:id/*proxyPath", WrapHandler(s.proxyStreamlit)) v1.Any("/other/:id/*proxyPath", WrapHandler(s.proxyOther)) // healthz root.GET(endpointHealthz, WrapHandler(s.handleHealthz)) // landing page root.GET("/", WrapHandler(s.handleRoot)) // control plane controlPlane := root.Group("/system") // inferences controlPlane.GET(endpointInferencePlural, WrapHandler(s.handleInferenceList)) controlPlane.POST(endpointInferencePlural, WrapHandler(s.handleInferenceCreate)) controlPlane.PUT(endpointInferencePlural, WrapHandler(s.handleInferenceUpdate)) controlPlane.DELETE(endpointInferencePlural, WrapHandler(s.handleInferenceDelete)) controlPlane.POST(endpointScaleInference, WrapHandler(s.handleInferenceScale)) controlPlane.GET(endpointInference+"/:name", WrapHandler(s.handleInferenceGet)) // instances controlPlane.GET(endpointInference+"/:name/instances", WrapHandler(s.handleInferenceInstance)) controlPlane.GET(endpointInference+"/:name/instance/:instance/exec", WrapHandler(s.handleInferenceInstanceExec)) // info controlPlane.GET(endpointInfo, WrapHandler(s.handleInfo)) // servers controlPlane.GET(endpointServerPlural, WrapHandler(s.handleServerList)) controlPlane.POST(endpointServer+"/:name/labels", WrapHandler(s.handleServerLabelCreate)) controlPlane.DELETE(endpointServer+"/:name/delete", WrapHandler(s.handleServerDelete)) // logs controlPlane.GET(endpointLogPlural+endpointInference, WrapHandler(s.handleInferenceLogs)) controlPlane.GET(endpointLogPlural+endpointBuild, WrapHandler(s.handleBuildLogs)) // namespaces controlPlane.GET(endpointNamespacePlural, WrapHandler(s.handleNamespaceList)) controlPlane.POST(endpointNamespacePlural, WrapHandler(s.handleNamespaceCreate)) controlPlane.DELETE(endpointNamespacePlural, WrapHandler(s.handleNamespaceDelete)) // TODO(gaocegege): Support secrets // controlPlane.GET("/secrets") // builds if s.config.Build.BuildEnabled { controlPlane.GET(endpointBuild, WrapHandler(s.handleBuildList)) controlPlane.GET(endpointBuild+"/:name", WrapHandler(s.handleBuildGet)) controlPlane.POST(endpointBuild, WrapHandler(s.handleBuildCreate)) } // TODO(gaocegege): Support metrics // metrics // image cache controlPlane.POST(endpointImageCache, WrapHandler(s.handleImageCacheCreate)) } // registerMetricsRoutes registers the metrics routes. func (s *Server) registerMetricsRoutes() { s.metricsRouter.GET("/metrics", gin.WrapH(metrics.PrometheusHandler())) s.metricsRouter.GET(endpointHealthz, WrapHandler(s.handleHealthz)) } ================================================ FILE: agent/pkg/server/server_run.go ================================================ package server import ( "context" "errors" "fmt" "net/http" "os" "os/signal" "syscall" "time" "github.com/sirupsen/logrus" "github.com/tensorchord/openmodelz/agent/api/types" "k8s.io/apimachinery/pkg/util/wait" ) func (s *Server) Run() error { srv := &http.Server{ Addr: fmt.Sprintf(":%d", s.config.Server.ServerPort), Handler: s.router, WriteTimeout: s.config.Server.WriteTimeout, ReadTimeout: s.config.Server.ReadTimeout, } go func() { if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { logrus.Errorf("listen on port %d error: %v", s.config.Server.ServerPort, err) } }() metricsSrv := &http.Server{ Addr: fmt.Sprintf(":%d", s.config.Metrics.ServerPort), Handler: s.metricsRouter, ReadTimeout: s.config.Metrics.PollingInterval, WriteTimeout: s.config.Metrics.PollingInterval, } go func() { if err := metricsSrv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { logrus.Errorf("listen on port %d error: %v", s.config.Metrics.ServerPort, err) } }() logrus.WithField("port", s.config.Server.ServerPort). Info("server is running...") logrus.WithField("metrics-port", s.config.Metrics.ServerPort). Info("metrics server is running...") if s.config.ModelZCloud.Enabled { // check apiserver is ready apiServerReady := make(chan struct{}) go func() { if err := s.modelzCloudClient.WaitForAPIServerReady(); err != nil { logrus.Fatalf("failed to wait for apiserver ready: %v", err) } close(apiServerReady) }() // websocket // build websocket go s.connect(apiServerReady) // heartbeat with apiserver go wait.UntilWithContext(context.Background(), func(ctx context.Context) { cluster := types.ManagedCluster{ Name: s.config.ModelZCloud.Name, ID: s.config.ModelZCloud.ID, Status: types.ClusterStatusActive, UpdatedAt: time.Now().UTC(), TokenID: s.config.ModelZCloud.TokenID, Region: s.config.ModelZCloud.Region, PrometheusURL: fmt.Sprintf("%s:%d", s.config.Metrics.PrometheusHost, s.config.Metrics.PrometheusPort), } err := s.runtime.GetClusterInfo(&cluster) if err != nil { logrus.Errorf("failed to get managed cluster info: %v", err) } err = s.modelzCloudClient.UpdateAgentStatus(ctx, apiServerReady, s.config.ModelZCloud.AgentToken, cluster) if err != nil { logrus.Errorf("failed to update agent status: %v", err) } logrus.Debugf("update agent status: %v", cluster) }, s.config.ModelZCloud.HeartbeatInterval) go wait.UntilWithContext(context.Background(), func(ctx context.Context) { apikeys, err := s.modelzCloudClient.GetAPIKeys(ctx, apiServerReady, s.config.ModelZCloud.AgentToken, s.config.ModelZCloud.ID) if err != nil { logrus.Errorf("failed to get apikeys: %v", err) } s.config.ModelZCloud.APIKeys = apikeys logrus.Debugf("update apikeys") }, s.config.ModelZCloud.HeartbeatInterval) // default 1min update, TODO(xieydd) make it configurable go wait.UntilWithContext(context.Background(), func(ctx context.Context) { namespaces, err := s.modelzCloudClient.GetNamespaces(ctx, apiServerReady, s.config.ModelZCloud.AgentToken, s.config.ModelZCloud.ID) if err != nil { logrus.Errorf("failed to get namespaces: %v", err) } for _, ns := range namespaces.Items { if ContainString(ns, s.config.ModelZCloud.UserNamespaces) { continue } err = s.runtime.NamespaceCreate(ctx, ns) if err != nil { logrus.Errorf("failed to create namespace %s: %v", ns, err) continue } s.config.ModelZCloud.UserNamespaces = append(s.config.ModelZCloud.UserNamespaces, ns) logrus.Debugf("update namespaces") } }, s.config.ModelZCloud.HeartbeatInterval) // default 1h update, make it configurable } quit := make(chan os.Signal, 1) signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM) <-quit logrus.Info("shutdown server") ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() return srv.Shutdown(ctx) } func ContainString(target string, strs []string) bool { for _, str := range strs { if str == target { return true } } return false } ================================================ FILE: agent/pkg/server/server_websocket.go ================================================ package server import ( "context" "net/http" "net/url" "time" "github.com/rancher/remotedialer" "github.com/sirupsen/logrus" "github.com/tensorchord/openmodelz/agent/api/types" ) func (s *Server) connect(apiServerReady <-chan struct{}) { <-apiServerReady var clusterDialEndpoint string headers := http.Header{ "X-Cluster-ID": {s.config.ModelZCloud.ID}, "Agent-Token": {s.config.ModelZCloud.AgentToken}, } u, err := url.Parse(s.config.ModelZCloud.URL) if err != nil { logrus.Errorf("failed to parse url: %v", err) } switch u.Scheme { case "http": clusterDialEndpoint = "ws://" + u.Host + types.DailEndPointSuffix case "https": clusterDialEndpoint = "wss://" + u.Host + types.DailEndPointSuffix } ctx := context.Background() go func() { for { remotedialer.ClientConnect(ctx, clusterDialEndpoint, headers, nil, func(proto, address string) bool { return true }, nil) select { case <-ctx.Done(): return case <-time.After(s.config.ModelZCloud.HeartbeatInterval): // retry connect after interval } } }() // retry( // s.config.ModelZCloud.HeartbeatInterval, // func() error { // logrus.Debugf("run websocket server") // ctx := context.Background() // err := remotedialer.ClientConnect(ctx, clusterDialEndpoint, headers, nil, // func(proto, address string) bool { return true }, nil) // if err != nil { // logrus.Errorf("failed to connect to apiserver: %v", err) // return err // } // return nil // }, // ) } func retry(sleep time.Duration, f func() error) { i := 1 for { err := f() if err == nil { return } else { logrus.Errorf("retry %d times, still failed", i) time.Sleep(sleep) i++ } } } ================================================ FILE: agent/pkg/server/static/index.html ================================================ OpenModelZ Serving | Running

OpenModelZ server is running Version: {{.Version}}

Please check out the documentation for the next steps. Please contact us on discord if there is any issue.

================================================ FILE: agent/pkg/server/static/landing.go ================================================ package static import ( "bytes" _ "embed" "html/template" "github.com/tensorchord/openmodelz/agent/pkg/version" ) //go:embed index.html var htmlTemplate string type htmlStruct struct { Version string } func RenderLoadingPage() (*bytes.Buffer, error) { tmpl, err := template.New("root").Parse(htmlTemplate) if err != nil { return nil, err } data := htmlStruct{ Version: version.GetAgentVersion(), } var buffer bytes.Buffer if err := tmpl.Execute(&buffer, data); err != nil { return nil, err } return &buffer, nil } ================================================ FILE: agent/pkg/server/static/page_loading.go ================================================ package static import ( "bytes" "html/template" "github.com/tensorchord/openmodelz/agent/api/types" ) const htmlDeploymentTemplate = `Loading - {{.Framework}}

{{.StatusString}}Framework: {{.Framework}}Deployment: {{.Deployment}}Scheduling, ContainerCreating, Initializing, RunningStatus: {{.InstanceStatus}}The page will auto refresh once the request is completed. Kindly wait for the page to reload automatically. If the issue persists, please contact modelz support team on discord for assistance.

` type htmlDeploymentStruct struct { Deployment string Framework string ID string StatusString string InstanceStatus string } func RenderDeploymentLoadingPage(framework, id, statusString, deployment string, instances []types.InferenceDeploymentInstance) (*bytes.Buffer, error) { tmpl, err := template.New("root").Parse(htmlDeploymentTemplate) if err != nil { return nil, err } data := htmlDeploymentStruct{ Deployment: deployment, Framework: framework, ID: id, StatusString: statusString, InstanceStatus: "Scaling", } if len(instances) > 0 { data.InstanceStatus = string(instances[0].Status.Phase) } var buffer bytes.Buffer if err := tmpl.Execute(&buffer, data); err != nil { return nil, err } return &buffer, nil } ================================================ FILE: agent/pkg/server/suite_test.go ================================================ package server import ( "bytes" "encoding/json" "io" "net/http" "net/http/httptest" "net/url" "testing" "github.com/gin-gonic/gin" "github.com/golang/mock/gomock" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" runtimemock "github.com/tensorchord/openmodelz/agent/pkg/runtime/mock" ) var ( ctrl *gomock.Controller mockRuntime *runtimemock.MockRuntime server *Server ) func mkContext(method string, path string, header map[string][]string, body io.Reader) *gin.Context { c, _ := gin.CreateTestContext(httptest.NewRecorder()) if c == nil { panic(c) } req, _ := http.NewRequest(method, path, body) for k, vs := range header { for _, v := range vs { req.Header.Add(k, v) } } c.Request = req return c } func mkJsonBodyContext(method string, path string, header map[string][]string, body any) *gin.Context { jsonValue, err := json.Marshal(body) if err != nil { panic(err) } return mkContext(method, path, header, bytes.NewBuffer(jsonValue)) } func setQuery(c *gin.Context, query map[string]string) { params, _ := url.ParseQuery(c.Request.URL.RawQuery) for k, v := range query { params.Set(k, v) } c.Request.URL.RawQuery = params.Encode() } func setParam(c *gin.Context, param map[string]string) { for k, v := range param { c.Params = []gin.Param{{Key: k, Value: v}} } } func TestBuilder(t *testing.T) { gin.SetMode(gin.ReleaseMode) RegisterFailHandler(Fail) RunSpecs(t, "server") } var _ = BeforeSuite(func() { ctrl = gomock.NewController(GinkgoT()) mockRuntime = runtimemock.NewMockRuntime(ctrl) }) ================================================ FILE: agent/pkg/server/user.go ================================================ package server import ( "context" "github.com/sirupsen/logrus" ) func (s *Server) getUIDFromDeploymentID(ctx context.Context, id string) (string, bool) { uid, exit := s.cache.Get(id) if exit { return uid.(string), true } uid, err := s.modelzCloudClient.GetUIDFromDeploymentID(ctx, s.config.ModelZCloud.AgentToken, s.config.ModelZCloud.ID, id) if err != nil { logrus.Errorf("failed to get uid from deployment id: %v", err) return "", false } // no expiration s.cache.SetWithTTL(id, uid, 1, 0) return uid.(string), true } ================================================ FILE: agent/pkg/server/validator/validator.go ================================================ package validator import ( "fmt" "regexp" "k8s.io/apimachinery/pkg/util/rand" "github.com/tensorchord/openmodelz/agent/api/types" ) const ( defaultMinReplicas = 0 defaultMaxReplicas = 1 maxReplicas = 5 defaultTargetLoad = 100 defaultZeroDuration = 300 defaultStartupDuration = 600 defaultBuildDuration = "40m" defaultHTTPProbePath = "/" ) var ( dnsValidRegex = `^[a-z0-9]([-a-z0-9]*[a-z0-9])?$` ) type Validator struct { validDNS *regexp.Regexp } func New() *Validator { return &Validator{ validDNS: regexp.MustCompile(dnsValidRegex), } } // Validates that the service name is valid for Kubernetes func (v Validator) ValidateService(service string) error { matched := v.validDNS.MatchString(service) if matched { return nil } return fmt.Errorf("service: (%s) is invalid, must be a valid DNS entry", service) } // DefaultDeployRequest sets default values for the deploy request. func (v Validator) DefaultDeployRequest(request *types.InferenceDeployment) { if request.Spec.Scaling == nil { request.Spec.Scaling = &types.ScalingConfig{} } if request.Spec.Scaling.MinReplicas == nil { request.Spec.Scaling.MinReplicas = new(int32) *request.Spec.Scaling.MinReplicas = defaultMinReplicas } if request.Spec.Scaling.MaxReplicas == nil { request.Spec.Scaling.MaxReplicas = new(int32) *request.Spec.Scaling.MaxReplicas = defaultMinReplicas } if request.Spec.Scaling.TargetLoad == nil { request.Spec.Scaling.TargetLoad = new(int32) *request.Spec.Scaling.TargetLoad = defaultTargetLoad } if request.Spec.Scaling.Type == nil { request.Spec.Scaling.Type = new(types.ScalingType) *request.Spec.Scaling.Type = types.ScalingTypeCapacity } if request.Spec.Scaling.ZeroDuration == nil { request.Spec.Scaling.ZeroDuration = new(int32) *request.Spec.Scaling.ZeroDuration = defaultZeroDuration } if request.Spec.Scaling.StartupDuration == nil { request.Spec.Scaling.StartupDuration = new(int32) *request.Spec.Scaling.StartupDuration = defaultStartupDuration } if request.Spec.Framework == "" { request.Spec.Framework = types.FrameworkOther } } // ValidateDeployRequest validates that the service name is valid for Kubernetes func (v Validator) ValidateDeployRequest(request *types.InferenceDeployment) error { if request.Spec.Name == "" { return fmt.Errorf("service: is required") } err := v.ValidateService(request.Spec.Name) if err != nil { return err } if request.Spec.Image == "" { return fmt.Errorf("image: is required") } if request.Spec.Scaling == nil { return fmt.Errorf("scaling: is required") } if request.Spec.Framework == types.FrameworkOther { if request.Spec.Port == nil { return fmt.Errorf("port: is required for other framework") } } return nil } func (v Validator) ValidateBuildRequest(request *types.Build) error { if request.Spec.Name == "" { return fmt.Errorf("name: is required") } if request.Spec.BuildTarget.ArtifactImage == "" { return fmt.Errorf("artifact image: is required") } return nil } func (v Validator) ValidateImageCacheRequest(request *types.ImageCache) error { if request.Name == "" { return fmt.Errorf("name: is required") } if request.Namespace == "" { return fmt.Errorf("namespace: is required") } if request.Image == "" { return fmt.Errorf("image: is required") } if request.NodeSelector == "" { return fmt.Errorf("node selector: is required") } return nil } func (v Validator) DefaultBuildRequest(request *types.Build) { if request.Spec.BuildTarget.Builder == "" { request.Spec.BuildTarget.Builder = types.BuilderTypeImage } if request.Spec.BuildTarget.Builder != types.BuilderTypeImage { if request.Spec.Branch == "" && request.Spec.Revision == "" { request.Spec.Branch = "main" } if request.Spec.BuildTarget.Duration == "" { request.Spec.BuildTarget.Duration = defaultBuildDuration } } if request.Spec.BuildTarget.ArtifactImageTag == "" { request.Spec.BuildTarget.ArtifactImageTag = rand.String(8) } } ================================================ FILE: agent/pkg/version/version.go ================================================ /* Copyright The TensorChord Inc. Copyright The BuildKit Authors. Copyright The containerd Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package version import ( "fmt" "regexp" "runtime" "strings" "sync" ) var ( // Package is filled at linking time Package = "github.com/tensorchord/openmodelz/agent" // Revision is filled with the VCS (e.g. git) revision being used to build // the program at linking time. Revision = "" version = "0.0.0+unknown" buildDate = "1970-01-01T00:00:00Z" // output from `date -u +'%Y-%m-%dT%H:%M:%SZ'` gitCommit = "" // output from `git rev-parse HEAD` gitTag = "" // output from `git describe --exact-match --tags HEAD` (if clean tree state) gitTreeState = "" // determined from `git status --porcelain`. either 'clean' or 'dirty' developmentFlag = "false" ) // Version contains version information type Version struct { Version string BuildDate string GitCommit string GitTag string GitTreeState string GoVersion string Compiler string Platform string } func (v Version) String() string { return v.Version } // SetGitTagForE2ETest sets the gitTag for test purpose. func SetGitTagForE2ETest(tag string) { gitTag = tag } // GetAgentVersion gets version information func GetAgentVersion() string { var versionStr string if gitCommit != "" && gitTag != "" && gitTreeState == "clean" && developmentFlag == "false" { // if we have a clean tree state and the current commit is tagged, // this is an official release. versionStr = gitTag } else { // otherwise formulate a version string based on as much metadata // information we have available. if strings.HasPrefix(version, "v") { versionStr = version } else { versionStr = "v" + version } if len(gitCommit) >= 7 { versionStr += "+" + gitCommit[0:7] if gitTreeState != "clean" { versionStr += ".dirty" } } else { versionStr += "+unknown" } } return versionStr } // GetVersion returns the version information func GetVersion() Version { return Version{ Version: GetAgentVersion(), BuildDate: buildDate, GitCommit: gitCommit, GitTag: gitTag, GitTreeState: gitTreeState, GoVersion: runtime.Version(), Compiler: runtime.Compiler, Platform: fmt.Sprintf("%s/%s", runtime.GOOS, runtime.GOARCH), } } var ( reRelease *regexp.Regexp reDev *regexp.Regexp reOnce sync.Once ) func UserAgent() string { version := GetVersion().String() reOnce.Do(func() { reRelease = regexp.MustCompile(`^(v[0-9]+\.[0-9]+)\.[0-9]+$`) reDev = regexp.MustCompile(`^(v[0-9]+\.[0-9]+)\.[0-9]+`) }) if matches := reRelease.FindAllStringSubmatch(version, 1); len(matches) > 0 { version = matches[0][1] } else if matches := reDev.FindAllStringSubmatch(version, 1); len(matches) > 0 { version = matches[0][1] + "-dev" } return "openmodelz/agent/" + version } ================================================ FILE: agent/sqlc.yaml ================================================ version: "2" sql: - engine: "postgresql" queries: "sql/query/" schema: "./sql/schema.sql" gen: go: package: "query" sql_package: "pgx/v4" out: "pkg/query" emit_prepared_queries: true emit_interface: true emit_exact_table_names: false emit_json_tags: true ================================================ FILE: autoscaler/.gitignore ================================================ # If you prefer the allow list template instead of the deny list, see community template: # https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore # # Binaries for programs and plugins *.exe *.exe~ *.dll *.so *.dylib # Test binary, built with `go test -c` *.test # Output of the go coverage tool, specifically when used with LiteIDE *.out *.report # Dependency directories (remove the comment below to include it) vendor/ # Go workspace file go.work .vscode/* .idea # Local History for Visual Studio Code .history/ # Built Visual Studio Code Extensions *.vsix __debug_bin bin/ debug-bin/ /build.envd .ipynb_checkpoints/ cover.html cmd/test/ # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] *$py.class # Distribution / packaging .Python build/ develop-eggs/ dist/ downloads/ eggs/ .eggs/ lib/ lib64/ parts/ sdist/ var/ wheels/ wheelhouse/ pip-wheel-metadata/ share/python-wheels/ *.egg-info/ .installed.cfg *.egg MANIFEST .demo/ pkg/docs/swagger.* ================================================ FILE: autoscaler/Dockerfile ================================================ FROM ubuntu:22.04 LABEL maintainer="modelz-support@tensorchord.ai" COPY autoscaler /usr/bin/autoscaler ENTRYPOINT ["/usr/bin/autoscaler"] ================================================ FILE: autoscaler/Makefile ================================================ # Copyright 2022 TensorChord Inc. # # The old school Makefile, following are required targets. The Makefile is written # to allow building multiple binaries. You are free to add more targets or change # existing implementations, as long as the semantics are preserved. # # make - default to 'build' target # make lint - code analysis # make test - run unit test (or plus integration test) # make build - alias to build-local target # make build-local - build local binary targets # make build-linux - build linux binary targets # make container - build containers # $ docker login registry -u username -p xxxxx # make push - push containers # make clean - clean up targets # # Not included but recommended targets: # make e2e-test # # The makefile is also responsible to populate project version information. # # # Tweak the variables based on your project. # # This repo's root import path (under GOPATH). ROOT := github.com/tensorchord/openmodelz/autoscaler # Target binaries. You can build multiple binaries for a single project. TARGETS := autoscaler # Container image prefix and suffix added to targets. # The final built images are: # $[REGISTRY]/$[IMAGE_PREFIX]$[TARGET]$[IMAGE_SUFFIX]:$[VERSION] # $[REGISTRY] is an item from $[REGISTRIES], $[TARGET] is an item from $[TARGETS]. IMAGE_PREFIX ?= $(strip ) IMAGE_SUFFIX ?= $(strip ) # Container registries. REGISTRY ?= ghcr.io/tensorchord # Container registry for base images. BASE_REGISTRY ?= docker.io BASE_REGISTRY_USER ?= modelzai # Disable CGO by default. CGO_ENABLED ?= 0 GOOS ?= $(shell go env GOOS) GOARCH ?= $(shell go env GOARCH) # # These variables should not need tweaking. # # It's necessary to set this because some environments don't link sh -> bash. export SHELL := bash # It's necessary to set the errexit flags for the bash shell. export SHELLOPTS := errexit PACKAGE_NAME := github.com/tensorchord/openmodelz GOLANG_CROSS_VERSION ?= v1.17.6 # Project main package location (can be multiple ones). CMD_DIR := ./cmd # Project output directory. OUTPUT_DIR := ./bin DEBUG_DIR := ./debug-bin # Build directory. BUILD_DIR := ./build # Current version of the project. VERSION ?= $(shell git describe --match 'v[0-9]*' --always --tags --abbrev=0) BUILD_DATE=$(shell date -u +'%Y-%m-%dT%H:%M:%SZ') GIT_COMMIT=$(shell git rev-parse HEAD) GIT_TAG=$(shell if [ -z "`git status --porcelain`" ]; then git describe --exact-match --tags HEAD 2>/dev/null; fi) GIT_TREE_STATE=$(shell if [ -z "`git status --porcelain`" ]; then echo "clean" ; else echo "dirty"; fi) GITSHA ?= $(shell git rev-parse --short HEAD) # Track code version with Docker Label. DOCKER_LABELS ?= git-describe="$(shell date -u +v%Y%m%d)-$(shell git describe --tags --always --dirty)" # Golang standard bin directory. GOPATH ?= $(shell go env GOPATH) GOROOT ?= $(shell go env GOROOT) BIN_DIR := $(GOPATH)/bin GOLANGCI_LINT := $(BIN_DIR)/golangci-lint # check if we need embed the dashboard DASHBOARD_BUILD ?= debug # Default golang flags used in build and test # -mod=vendor: force go to use the vendor files instead of using the `$GOPATH/pkg/mod` # -p: the number of programs that can be run in parallel # -count: run each test and benchmark 1 times. Set this flag to disable test cache export GOFLAGS ?= -count=1 # # Define all targets. At least the following commands are required: # # All targets. .PHONY: help lint test build container push addlicense debug debug-local build-local generate clean test-local addlicense-install release build-image .DEFAULT_GOAL:=build build: build-local ## Build the release version of envd help: ## Display this help @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z0-9_-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) debug: debug-local ## Build the debug version of envd # more info about `GOGC` env: https://github.com/golangci/golangci-lint#memory-usage-of-golangci-lint lint: $(GOLANGCI_LINT) ## Lint GO code @$(GOLANGCI_LINT) run $(GOLANGCI_LINT): curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $$(go env GOPATH)/bin mockgen-install: go install github.com/golang/mock/mockgen@v1.6.0 addlicense-install: go install github.com/google/addlicense@latest build-local: @for target in $(TARGETS); do \ CGO_ENABLED=$(CGO_ENABLED) GOOS=$(GOOS) GOARCH=$(GOARCH) go build -tags $(DASHBOARD_BUILD) -trimpath -v -o $(OUTPUT_DIR)/$${target} \ -ldflags "-s -w -X $(ROOT)/pkg/version.version=$(VERSION) -X $(ROOT)/pkg/version.buildDate=$(BUILD_DATE) -X $(ROOT)/pkg/version.gitCommit=$(GIT_COMMIT) -X $(ROOT)/pkg/version.gitTreeState=$(GIT_TREE_STATE)" \ $(CMD_DIR)/$${target}; \ done # It is used by vscode to attach into the process. debug-local: @for target in $(TARGETS); do \ CGO_ENABLED=$(CGO_ENABLED) go build -tags $(DASHBOARD_BUILD) -trimpath \ -v -o $(DEBUG_DIR)/$${target} \ -gcflags='all=-N -l' \ $(CMD_DIR)/$${target}; \ done addlicense: addlicense-install ## Add license to GO code files addlicense -l mpl -c "TensorChord Inc." $$(find . -type f -name '*.go' | grep -v pkg/docs/docs.go) test-local: @go test -tags=$(DASHBOARD_BUILD) -v -race -coverprofile=coverage.out ./... test: ## Run the tests @go test -tags=$(DASHBOARD_BUILD) -race -coverpkg=./pkg/... -coverprofile=coverage.out ./... @go tool cover -func coverage.out | tail -n 1 | awk '{ print "Total coverage: " $$3 }' clean: ## Clean the outputs and artifacts @-rm -vrf ${OUTPUT_DIR} @-rm -vrf ${DEBUG_DIR} @-rm -vrf build dist .eggs *.egg-info fmt: ## Run go fmt against code. go fmt ./... vet: ## Run go vet against code. go vet ./... build-image: build-local docker build -t ${BASE_REGISTRY}/${BASE_REGISTRY_USER}/modelz-autoscaler:dev -f Dockerfile ./bin docker push ${BASE_REGISTRY}/${BASE_REGISTRY_USER}/modelz-autoscaler:dev release: @if [ ! -f ".release-env" ]; then \ echo "\033[91m.release-env is required for release\033[0m";\ exit 1;\ fi docker run \ --rm \ --privileged \ -e CGO_ENABLED=1 \ --env-file .release-env \ -v /var/run/docker.sock:/var/run/docker.sock \ -v `pwd`:/go/src/$(PACKAGE_NAME) \ -v `pwd`/sysroot:/sysroot \ -w /go/src/$(PACKAGE_NAME) \ goreleaser/goreleaser-cross:${GOLANG_CROSS_VERSION} \ release --rm-dist tsschema: swag @cd dashboard; pnpm tsschema generate: mockgen-install sqlc-install swag tsschema @mockgen -source pkg/query/querier.go -destination pkg/query/mock/mock.go -package mock @sqlc generate dashboard-build: @cd dashboard; pnpm build ================================================ FILE: autoscaler/cmd/autoscaler/main.go ================================================ package main import ( "fmt" "os" "github.com/sirupsen/logrus" "github.com/urfave/cli/v2" "github.com/tensorchord/openmodelz/autoscaler/pkg/autoscalerapp" "github.com/tensorchord/openmodelz/autoscaler/pkg/version" ) func run(args []string) error { cli.VersionPrinter = func(c *cli.Context) { fmt.Println(c.App.Name, version.Package, c.App.Version, version.Revision) } app := autoscalerapp.New() return app.Run(args) } func handleErr(err error) { if err == nil { return } logrus.Error(err) } func main() { err := run(os.Args) handleErr(err) } ================================================ FILE: autoscaler/pkg/autoscaler/factory.go ================================================ package autoscaler import ( "net/http" "net/url" "time" "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "github.com/tensorchord/openmodelz/agent/client" "github.com/tensorchord/openmodelz/autoscaler/pkg/prom" ) type Opt struct { GatewayHost string PrometheusHost string BasicAuthEnabled bool SecretPath string PrometheusPort int Interval time.Duration } func New(opt Opt) (*Scaler, error) { logrus.Info("Creating autoscaler with options: ", opt) gatewayURL, err := url.Parse(opt.GatewayHost) if err != nil { return nil, errors.Wrap(err, "failed to parse gateway host") } client, err := client.NewClientWithOpts( client.WithHost(gatewayURL.String()), ) if err != nil { return nil, errors.Wrap(err, "failed to create client") } prometheusQuery := prom.NewPrometheusQuery(opt.PrometheusHost, opt.PrometheusPort, &http.Client{}) as := newScaler(client, &prometheusQuery, newLoadCache(), newInferenceCache()) return as, nil } ================================================ FILE: autoscaler/pkg/autoscaler/inferencecache.go ================================================ package autoscaler import ( "time" "github.com/tensorchord/openmodelz/agent/api/types" ) type Inference struct { Deployment types.InferenceDeployment Timestamp time.Time } type InferenceCache struct { inference map[string]Inference } func newInferenceCache() *InferenceCache { return &InferenceCache{ inference: make(map[string]Inference), } } func (i *InferenceCache) Set(key string, inference Inference) { i.inference[key] = inference } func (i *InferenceCache) Get(key string, expireTime time.Duration) (types.InferenceDeployment, bool) { inference, ok := i.inference[key] // expired if !ok || time.Since(inference.Timestamp) > expireTime { return types.InferenceDeployment{}, false } return inference.Deployment, ok } ================================================ FILE: autoscaler/pkg/autoscaler/loadcache.go ================================================ package autoscaler import "time" // LoadCache is a cache for load metrics. type LoadCache struct { load map[string]Load } type Load struct { ScalingType string CurrentStartedRequests float64 CurrentLoad float64 Timestamp time.Time } func newLoadCache() *LoadCache { return &LoadCache{ load: make(map[string]Load), } } func (l *LoadCache) Get(key string) (Load, bool) { load, ok := l.load[key] return load, ok } func (l *LoadCache) Set(key string, load Load) { l.load[key] = load } ================================================ FILE: autoscaler/pkg/autoscaler/scaler.go ================================================ package autoscaler import ( "context" "fmt" "math" "net/url" "strconv" "strings" "time" "github.com/sirupsen/logrus" "github.com/tensorchord/openmodelz/agent/client" "github.com/tensorchord/openmodelz/agent/pkg/scaling" "github.com/tensorchord/openmodelz/autoscaler/pkg/prom" ) type Scaler struct { PromQuery *prom.PrometheusQuery client *client.Client LoadCache *LoadCache ZeroCache map[string]time.Time InferenceCache *InferenceCache } func newScaler(c *client.Client, promQuery *prom.PrometheusQuery, loadCache *LoadCache, inferanceCache *InferenceCache) *Scaler { return &Scaler{ client: c, PromQuery: promQuery, LoadCache: loadCache, ZeroCache: make(map[string]time.Time), InferenceCache: inferanceCache, } } func (s *Scaler) AutoScale(interval time.Duration) { ticker := time.NewTicker(interval) quit := make(chan struct{}) TTL := 1 * time.Minute for { select { case <-ticker.C: // Detect if the instance pod always restart, // if pod restart count in 10 minutes before last update time is more than 2, will scale it down. results, err := s.GetRestartMetrics() if err != nil { logrus.Info("Get Restart Metrics of inference Failed") continue } inferenceCount := make(map[string]int) for _, ts := range results { labels := ts.Labels podName, inferenceName, namespace := "", "", "" for _, label := range labels { switch label.Name { case "pod": podName = label.Value case "inference_name": inferenceName = label.Value case "namespace": namespace = label.Value } } if len(ts.Samples) < 1 { logrus.Infof("Sample not found for inference %s.", inferenceName) continue } strs := strings.Split(inferenceName, ".") if len(strs) != 2 { logrus.Infof("Invalid inference name: %s", inferenceName) continue } name := strs[0] resp, err := s.client.InstanceList(context.TODO(), namespace, name) if err != nil { logrus.WithFields(logrus.Fields{ "service": inferenceName, "error": err, }).Error("failed to get instance list") continue } for _, instance := range resp { if instance.Spec.Name == podName { if instance.Status.Phase == "CrashLoopBackOff" { inferenceCount[inferenceName] += 1 } } } } if len(inferenceCount) != 0 { for inferenceName, count := range inferenceCount { strs := strings.Split(inferenceName, ".") if len(strs) != 2 { logrus.Infof("Invalid inference name: %s", inferenceName) continue } name := strs[0] namespace := strs[1] resp, ok := s.InferenceCache.Get(inferenceName, TTL) if !ok { resp, err = s.client.InferenceGet(context.TODO(), namespace, name) if err != nil { logrus.WithFields(logrus.Fields{ "service": inferenceName, "error": err, }).Error("failed to get inference") continue } // update inference cache inference := Inference{ Timestamp: time.Now(), Deployment: resp, } s.InferenceCache.Set(inferenceName, inference) } // check if the instance already exists var expectedReplicas int totalReplicas := resp.Status.Replicas if count > int(totalReplicas) { expectedReplicas = 0 } else { expectedReplicas = int(totalReplicas) - count } if expectedReplicas != int(totalReplicas) { logrus.Infof("Scaling inference %s to %d replicas", inferenceName, expectedReplicas) // Add event to record the scale down operation eventMessage := fmt.Sprintf("Deployment %d replicas always CrashLoopBackOff, system scale down the deployment replicas to %d", count, expectedReplicas) if err := s.client.InferenceScale(context.TODO(), namespace, name, expectedReplicas, eventMessage); err != nil { logrus.WithFields(logrus.Fields{ "service": inferenceName, "expected": expectedReplicas, "error": err, }).Error("failed to scale inference") continue } // update the inference, set minReplicas to expectedReplicas if resp.Spec.Scaling.MinReplicas != nil && *resp.Spec.Scaling.MinReplicas > int32(expectedReplicas) { resp.Status.EventMessage = fmt.Sprintf("Deployment %d replicas always CrashLoopBackOff, system scales down the replicas to %d, original min replicas is %d, reset it to %d", count, expectedReplicas, resp.Spec.Scaling.MinReplicas, expectedReplicas) *resp.Spec.Scaling.MinReplicas = int32(expectedReplicas) if _, err := s.client.DeploymentUpdate(context.TODO(), namespace, resp); err != nil { logrus.WithFields(logrus.Fields{ "service": inferenceName, "expected": expectedReplicas, "error": err, }).Error("failed to update inference") continue } } } } } s.LoadCache = newLoadCache() s.GetLoadMetrics() for service, lc := range s.LoadCache.load { // if instances of inference are restarting, do not scale it. if value, ok := inferenceCount[service]; ok && value > 0 { continue } strs := strings.Split(service, ".") if len(strs) != 2 { logrus.Infof("Invalid inference name: %s", service) continue } name := strs[0] namespace := strs[1] resp, ok := s.InferenceCache.Get(service, TTL) if !ok { resp, err = s.client.InferenceGet(context.TODO(), namespace, name) if err != nil { logrus.WithFields(logrus.Fields{ "service": service, "error": err, }).Error("failed to get inference") continue } // update inference cache inference := Inference{ Timestamp: time.Now(), Deployment: resp, } s.InferenceCache.Set(service, inference) } if resp.Spec.Labels == nil { logrus.WithFields(logrus.Fields{ "service": service, "error": err, }).Error("failed to get inference labels") continue } var expectedReplicas int var targetLoad int // If the inference has a target load label, use that instead. if resp.Spec.Scaling != nil && resp.Spec.Scaling.TargetLoad != nil { targetLoad = int(*resp.Spec.Scaling.TargetLoad) expectedReplicas = int(math.Ceil( lc.CurrentLoad / float64(*resp.Spec.Scaling.TargetLoad))) } if expectedReplicas == 0 { // Check the current start requests to see if the inference is being used. if lc.CurrentStartedRequests > 0 { logrus.WithFields(logrus.Fields{ "service": service, "current_started_requests": lc.CurrentStartedRequests, "target_load": lc.CurrentLoad, }).Debug("inference is being used") expectedReplicas = 1 } } var maxReplicas, minReplicas int var zeroDuration time.Duration if resp.Spec.Scaling != nil { if resp.Spec.Scaling.MinReplicas != nil { minReplicas = int(*resp.Spec.Scaling.MinReplicas) } else { minReplicas = scaling.DefaultMinReplicas } if resp.Spec.Scaling.MaxReplicas != nil { maxReplicas = int(*resp.Spec.Scaling.MaxReplicas) } else { maxReplicas = scaling.DefaultMaxReplicas } if resp.Spec.Scaling.ZeroDuration != nil { zeroDuration = time.Duration(*resp.Spec.Scaling.ZeroDuration) * time.Second } else { zeroDuration = scaling.DefaultZeroDuration } } if expectedReplicas > maxReplicas { logrus.Infof("Expected replicas (%d) exceeds max replicas (%d) for inference %s", expectedReplicas, maxReplicas, service) expectedReplicas = maxReplicas } if expectedReplicas < minReplicas { logrus.Infof("Expected replicas (%d) is less than min replicas (%d) for inference %s", expectedReplicas, minReplicas, service) expectedReplicas = minReplicas } availableReplicas := resp.Status.AvailableReplicas totalReplicas := resp.Status.Replicas if expectedReplicas == int(totalReplicas) { // If the expected replicas is the same as the current replicas, remove the entry from the zero cache. delete(s.ZeroCache, service) logrus.WithFields(logrus.Fields{ "service": service, "replicas": totalReplicas, "expectedReplicas": expectedReplicas, }).Debug("delete zero cache") } if expectedReplicas == 0 && totalReplicas != 0 { if availableReplicas == 0 { // If the expected replicas is 0 and there are no available replicas, // set the expected replicas to 1 to prevent the inference from being scaled to zero. expectedReplicas = 1 } else { // If the expected replicas is 0 and there is no entry in the zero cache, add one. if _, ok := s.ZeroCache[service]; !ok { s.ZeroCache[service] = time.Now() } // If the inference has been idle for longer than the zero duration, scale to zero. if time.Since(s.ZeroCache[service]) > zeroDuration { logrus.Infof("Inference %s has been idle for %s, scaling to zero", service, zeroDuration) } else { // If the inference has not been idle for longer than the zero duration, scale to 1. expectedReplicas = 1 } } } if expectedReplicas == 1 && totalReplicas == 0 { // If the expected replicas is 1 and the current replicas is 0, do nothing since the scaling handler in gateway will take care of this situation. expectedReplicas = 0 } logrus.WithFields(logrus.Fields{ "service": service, "replicas": totalReplicas, "expectedReplicas": expectedReplicas, "availableReplicas": availableReplicas, "currentLoad": lc.CurrentLoad, "targetLoad": targetLoad, "zeroDuration": zeroDuration, "zeroCache": s.ZeroCache[service], }).Debug("start scaling (replicas)") if expectedReplicas != int(totalReplicas) { delete(s.ZeroCache, service) logrus.Infof("Scaling inference %s to %d replicas", service, expectedReplicas) eventMessage := fmt.Sprintf("Scaling inference based load, current %f, target %d", lc.CurrentLoad, targetLoad) if err := s.client.InferenceScale(context.TODO(), namespace, name, expectedReplicas, eventMessage); err != nil { logrus.WithFields(logrus.Fields{ "service": service, "expected": expectedReplicas, "error": err, }).Error("failed to scale inference") continue } } } case <-quit: return } } } func (s *Scaler) GetLoadMetrics() { results, err := s.PromQuery.Fetch(url.QueryEscape("job:inference_current_load:sum")) if err != nil { // log the error but continue, the mixIn will correctly handle the empty results. logrus.Infof("Error querying Prometheus: %s\n", err.Error()) } currentSumResults, err := s.PromQuery.Fetch( url.QueryEscape("job:inference_current_started:max_sum")) if err != nil { // log the error but continue, the mixIn will correctly handle the empty results. logrus.Infof("Error querying Prometheus: %s\n", err.Error()) } for _, result := range results.Data.Result { currentLoad := 0.0 switch val := result.Value[1].(type) { case string: f, err := strconv.ParseFloat(val, 64) if err != nil { logrus.Infof("add_metrics: unable to convert value %q for metric: %s", val, err) continue } currentLoad = f } timestamp := time.Now() switch val := result.Value[0].(type) { case float64: timestamp = time.Unix(int64(val), 0) } if l, ok := s.LoadCache.Get(result.Metric.InferenceName); ok { l.CurrentLoad = currentLoad l.Timestamp = timestamp s.LoadCache.Set(result.Metric.InferenceName, l) } else { s.LoadCache.Set(result.Metric.InferenceName, Load{ CurrentLoad: currentLoad, Timestamp: timestamp, }) } } for _, result := range currentSumResults.Data.Result { currentSum := 0.0 switch val := result.Value[1].(type) { case string: f, err := strconv.ParseFloat(val, 64) if err != nil { logrus.Infof("add_metrics: unable to convert value %q for metric: %s", val, err) continue } currentSum = f } timestamp := time.Now() switch val := result.Value[0].(type) { case float64: timestamp = time.Unix(int64(val), 0) } if l, ok := s.LoadCache.Get(result.Metric.InferenceName); ok { l.CurrentStartedRequests = currentSum l.Timestamp = timestamp s.LoadCache.Set(result.Metric.InferenceName, l) } else { s.LoadCache.Set(result.Metric.InferenceName, Load{ CurrentStartedRequests: currentSum, Timestamp: timestamp, }) } } } func (s *Scaler) GetRestartMetrics() ([]*prom.TimeSeries, error) { // record this rule in prometheus // (sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{namespace=~"modelz-(.*)"}[10m])) > 2) * on (pod) group_left(inference_name) (label_join(label_replace(kube_pod_info{created_by_kind="ReplicaSet",namespace=~"modelz-(.*)"}, "inference", "$1", "created_by_name", "(.+)-.+"), "inference_name",".","inference","namespace")) query := "pod_restart_count_over_2_10m" tsList, err := s.PromQuery.Query(query, time.Now()) if err != nil { logrus.Infof("Error querying Prometheus: %s\n", err.Error()) return nil, err } return tsList, nil } ================================================ FILE: autoscaler/pkg/autoscalerapp/root.go ================================================ // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. package autoscalerapp import ( "time" "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" cli "github.com/urfave/cli/v2" "github.com/tensorchord/openmodelz/autoscaler/pkg/autoscaler" "github.com/tensorchord/openmodelz/autoscaler/pkg/server" "github.com/tensorchord/openmodelz/autoscaler/pkg/version" ) type EnvdServerApp struct { *cli.App } func New() EnvdServerApp { internalApp := cli.NewApp() internalApp.EnableBashCompletion = true internalApp.Name = "modelz-autoscaler" internalApp.Usage = "Autoscaler for modelz serverless inference platform" internalApp.HideHelpCommand = true internalApp.HideVersion = false internalApp.Version = version.GetVersion().String() internalApp.Flags = []cli.Flag{ &cli.BoolFlag{ Name: "debug", Usage: "enable debug output in logs", }, &cli.StringFlag{ Name: "gateway-host", Usage: "host for gateway", EnvVars: []string{"MODELZ_GATEWAY_HOST"}, Aliases: []string{"gh"}, }, &cli.StringFlag{ Name: "prometheus-host", Usage: "host for prometheus", Value: "prometheus", EnvVars: []string{"MODELZ_PROMETHEUS_HOST"}, Aliases: []string{"ph"}, }, &cli.IntFlag{ Name: "prometheus-port", Usage: "port for prometheus", Value: 9090, EnvVars: []string{"MODELZ_PROMETHEUS_PORT"}, Aliases: []string{"pp"}, }, &cli.BoolFlag{ Name: "basic-auth", Usage: "enable basic auth", EnvVars: []string{"MODELZ_BASIC_AUTH"}, Aliases: []string{"ba"}, Value: true, }, &cli.PathFlag{ Name: "secret-path", Usage: "path to secrets", Value: "/var/modelz/secrets", EnvVars: []string{"MODELZ_SECRET_PATH"}, Aliases: []string{"sp"}, }, &cli.DurationFlag{ Name: "interval", Usage: "interval for autoscaling", Value: time.Second, EnvVars: []string{"MODELZ_INTERVAL"}, Aliases: []string{"i"}, }, } internalApp.Action = runServer // Deal with debug flag. var debugEnabled bool internalApp.Before = func(context *cli.Context) error { debugEnabled = context.Bool("debug") if debugEnabled { logrus.SetReportCaller(true) logrus.SetFormatter(&logrus.TextFormatter{FullTimestamp: true}) logrus.SetLevel(logrus.DebugLevel) } else { logrus.SetFormatter(&logrus.JSONFormatter{}) } return nil } return EnvdServerApp{ App: internalApp, } } func runServer(clicontext *cli.Context) error { opt := autoscaler.Opt{ GatewayHost: clicontext.String("gateway-host"), PrometheusHost: clicontext.String("prometheus-host"), BasicAuthEnabled: clicontext.Bool("basic-auth"), SecretPath: clicontext.Path("secret-path"), PrometheusPort: clicontext.Int("prometheus-port"), Interval: clicontext.Duration("interval"), } as, err := autoscaler.New(opt) if err != nil { return errors.Wrap(err, "failed to create autoscaler") } logrus.Info("starting system info server") go server.RunInfoServe() logrus.Info("starting autoscaler") as.AutoScale(opt.Interval) return nil } ================================================ FILE: autoscaler/pkg/prom/prom.go ================================================ package prom import ( "context" "encoding/json" "fmt" "io/ioutil" "net/http" "time" "github.com/prometheus/client_golang/api" promapiv1 "github.com/prometheus/client_golang/api/prometheus/v1" prommodel "github.com/prometheus/common/model" "github.com/sirupsen/logrus" ) // PrometheusQuery represents parameters for querying Prometheus type PrometheusQuery struct { Port int Host string Client *http.Client } type PrometheusQueryFetcher interface { Fetch(query string) (*VectorQueryResponse, error) } // NewPrometheusQuery create a NewPrometheusQuery func NewPrometheusQuery(host string, port int, client *http.Client) PrometheusQuery { return PrometheusQuery{ Client: client, Host: host, Port: port, } } // Fetch queries aggregated stats func (q PrometheusQuery) Fetch(query string) (*VectorQueryResponse, error) { req, reqErr := http.NewRequest(http.MethodGet, fmt.Sprintf("http://%s:%d/api/v1/query?query=%s", q.Host, q.Port, query), nil) if reqErr != nil { return nil, reqErr } res, getErr := q.Client.Do(req) if getErr != nil { return nil, getErr } if res.Body != nil { defer res.Body.Close() } bytesOut, readErr := ioutil.ReadAll(res.Body) if readErr != nil { return nil, readErr } if res.StatusCode != http.StatusOK { return nil, fmt.Errorf("unexpected status code from Prometheus want: %d, got: %d, body: %s", http.StatusOK, res.StatusCode, string(bytesOut)) } var values VectorQueryResponse unmarshalErr := json.Unmarshal(bytesOut, &values) if unmarshalErr != nil { return nil, fmt.Errorf("error unmarshalling result: %s, '%s'", unmarshalErr, string(bytesOut)) } return &values, nil } // TODO(xieydd) Refactor PrometheusQuery // Query queries Prometheus with given query string and time func (q PrometheusQuery) Query(query string, time time.Time) ([]*TimeSeries, error) { var ts []*TimeSeries client, err := api.NewClient(api.Config{ Address: fmt.Sprintf("http://%s:%d", q.Host, q.Port), }) if err != nil { return ts, err } api := promapiv1.NewAPI(client) results, warnings, err := api.Query(context.TODO(), query, time) if len(warnings) != 0 { logrus.Info("Prom query warnings", "warnings", warnings) } if err != nil { return ts, err } logrus.Info("Prom query result", "result", results.String(), "resultsType", results.Type()) return convertPromResultsToTimeSeries(results) } func convertPromResultsToTimeSeries(value prommodel.Value) ([]*TimeSeries, error) { var results []*TimeSeries typeValue := value.Type() switch typeValue { case prommodel.ValMatrix: if matrix, ok := value.(prommodel.Matrix); ok { for _, sampleStream := range matrix { if sampleStream == nil { continue } ts := NewTimeSeries() for key, val := range sampleStream.Metric { ts.AppendLabel(string(key), string(val)) } for _, pair := range sampleStream.Values { ts.AppendSample(int64(pair.Timestamp/1000), float64(pair.Value)) } results = append(results, ts) } return results, nil } else { return results, fmt.Errorf("prometheus value type is %v, but assert failed", typeValue) } case prommodel.ValVector: if vector, ok := value.(prommodel.Vector); ok { for _, sample := range vector { if sample == nil { continue } ts := NewTimeSeries() for key, val := range sample.Metric { ts.AppendLabel(string(key), string(val)) } // for vector, all the sample has the same timestamp. just one point for each metric ts.AppendSample(int64(sample.Timestamp/1000), float64(sample.Value)) results = append(results, ts) } return results, nil } else { return results, fmt.Errorf("prometheus value type is %v, but assert failed", typeValue) } case prommodel.ValScalar: return results, fmt.Errorf("not support for scalar when use timeseries") case prommodel.ValString: return results, fmt.Errorf("not support for string when use timeseries") case prommodel.ValNone: return results, fmt.Errorf("prometheus return value type is none") } return results, fmt.Errorf("prometheus return unknown model value type %v", typeValue) } ================================================ FILE: autoscaler/pkg/prom/types.go ================================================ package prom import ( "fmt" "sort" ) type VectorQueryResponse struct { Data struct { Result []struct { Metric struct { InferenceName string `json:"inference_name"` } Value []interface{} `json:"value"` } } } // Ref: https://github.com/gocrane/crane/blob/9aaeb2aa9cf9f43a31842b4663e48bc47ac05f17/pkg/common/types.go // TimeSeries is a stream of samples that belong to a metric with a set of labels type TimeSeries struct { // A collection of Labels that are attached by monitoring system as metadata // for the metrics, which are known as dimensions. Labels []Label // A collection of Samples in chronological order. Samples []Sample } // Sample pairs a Value with a Timestamp. type Sample struct { Value float64 Timestamp int64 } // A Label is a Name and Value pair that provides additional information about the metric. // It is metadata for the metric. For example, Kubernetes pod metrics always have // 'namespace' label that represents which namespace the pod belongs to. type Label struct { Name string Value string } func (s *Sample) String() string { return fmt.Sprintf("%d %f", s.Timestamp, s.Value) } func (l *Label) String() string { return l.Name + "=" + l.Value } func (ts *TimeSeries) SetLabels(labels []Label) { ts.Labels = labels } func (ts *TimeSeries) SetSamples(samples []Sample) { ts.Samples = samples } func (ts *TimeSeries) AppendLabel(key, val string) { ts.Labels = append(ts.Labels, Label{key, val}) } func (ts *TimeSeries) AppendSample(timestamp int64, val float64) { ts.Samples = append(ts.Samples, Sample{Timestamp: timestamp, Value: val}) } func (ts *TimeSeries) SortSampleAsc() { sort.Slice(ts.Samples, func(i, j int) bool { if ts.Samples[i].Timestamp < ts.Samples[j].Timestamp { return true } else { return false } }) } func NewTimeSeries() *TimeSeries { return &TimeSeries{ Labels: make([]Label, 0), Samples: make([]Sample, 0), } } ================================================ FILE: autoscaler/pkg/server/status.go ================================================ // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. package server import ( "encoding/json" "fmt" "net/http" "time" "github.com/sirupsen/logrus" "github.com/tensorchord/openmodelz/autoscaler/pkg/version" ) func getInfo(w http.ResponseWriter, r *http.Request) { scalerInfo := map[string]string{"version": version.GetEnvdVersion()} jsonOut, marshalErr := json.Marshal(scalerInfo) if marshalErr != nil { logrus.Infof("Error during unmarshal of autoscaler info request %s\n", marshalErr.Error()) w.WriteHeader(http.StatusInternalServerError) return } w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) w.Write(jsonOut) } func RunInfoServe() { tcpPort := 8080 serverMux := http.NewServeMux() serverMux.HandleFunc("/system/info", getInfo) s := &http.Server{ Addr: fmt.Sprintf(":%d", tcpPort), ReadTimeout: 10 * time.Second, WriteTimeout: 10 * time.Second, MaxHeaderBytes: http.DefaultMaxHeaderBytes, // 1MB - can be overridden by setting Server.MaxHeaderBytes. Handler: serverMux, } s.ListenAndServe() } ================================================ FILE: autoscaler/pkg/version/version.go ================================================ /* Copyright The TensorChord Inc. Copyright The BuildKit Authors. Copyright The containerd Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package version import ( "fmt" "regexp" "runtime" "strings" "sync" ) var ( // Package is filled at linking time Package = "github.com/tensorchord/openmodelz/autoscaler" // Revision is filled with the VCS (e.g. git) revision being used to build // the program at linking time. Revision = "" version = "0.0.0+unknown" buildDate = "1970-01-01T00:00:00Z" // output from `date -u +'%Y-%m-%dT%H:%M:%SZ'` gitCommit = "" // output from `git rev-parse HEAD` gitTag = "" // output from `git describe --exact-match --tags HEAD` (if clean tree state) gitTreeState = "" // determined from `git status --porcelain`. either 'clean' or 'dirty' developmentFlag = "false" ) // Version contains envd version information type Version struct { Version string BuildDate string GitCommit string GitTag string GitTreeState string GoVersion string Compiler string Platform string } func (v Version) String() string { return v.Version } // SetGitTagForE2ETest sets the gitTag for test purpose. func SetGitTagForE2ETest(tag string) { gitTag = tag } // GetEnvdVersion gets Envd version information func GetEnvdVersion() string { var versionStr string if gitCommit != "" && gitTag != "" && gitTreeState == "clean" && developmentFlag == "false" { // if we have a clean tree state and the current commit is tagged, // this is an official release. versionStr = gitTag } else { // otherwise formulate a version string based on as much metadata // information we have available. if strings.HasPrefix(version, "v") { versionStr = version } else { versionStr = "v" + version } if len(gitCommit) >= 7 { versionStr += "+" + gitCommit[0:7] if gitTreeState != "clean" { versionStr += ".dirty" } } else { versionStr += "+unknown" } } return versionStr } // GetVersion returns the version information func GetVersion() Version { return Version{ Version: GetEnvdVersion(), BuildDate: buildDate, GitCommit: gitCommit, GitTag: gitTag, GitTreeState: gitTreeState, GoVersion: runtime.Version(), Compiler: runtime.Compiler, Platform: fmt.Sprintf("%s/%s", runtime.GOOS, runtime.GOARCH), } } var ( reRelease *regexp.Regexp reDev *regexp.Regexp reOnce sync.Once ) func UserAgent() string { version := GetVersion().String() reOnce.Do(func() { reRelease = regexp.MustCompile(`^(v[0-9]+\.[0-9]+)\.[0-9]+$`) reDev = regexp.MustCompile(`^(v[0-9]+\.[0-9]+)\.[0-9]+`) }) if matches := reRelease.FindAllStringSubmatch(version, 1); len(matches) > 0 { version = matches[0][1] } else if matches := reDev.FindAllStringSubmatch(version, 1); len(matches) > 0 { version = matches[0][1] + "-dev" } return "envd/" + version } ================================================ FILE: go.mod ================================================ module github.com/tensorchord/openmodelz go 1.20 replace ( github.com/anthhub/forwarder => github.com/tensorchord/forwarder v0.0.0-20230713171536-b1b52b398d3a github.com/senthilrch/kube-fledged v0.10.0 => github.com/tensorchord/kube-fledged v0.2.0 ) require ( github.com/anthhub/forwarder v1.1.0 github.com/cockroachdb/errors v1.10.0 github.com/dchest/uniuri v1.2.0 github.com/dgraph-io/ristretto v0.1.1 github.com/docker/docker v23.0.2+incompatible github.com/docker/go-connections v0.4.0 github.com/dustinkirkland/golang-petname v0.0.0-20230626224747-e794b9370d49 github.com/gin-contrib/cors v1.4.0 github.com/gin-gonic/gin v1.9.1 github.com/golang/mock v1.5.0 github.com/google/go-cmp v0.5.9 github.com/google/uuid v1.3.0 github.com/gorilla/websocket v1.4.2 github.com/jackc/pgconn v1.14.1 github.com/jackc/pgx/v4 v4.18.1 github.com/jedib0t/go-pretty/v6 v6.4.6 github.com/moby/moby v24.0.4+incompatible github.com/moby/term v0.5.0 github.com/onsi/ginkgo/v2 v2.11.0 github.com/onsi/gomega v1.27.8 github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.16.0 github.com/prometheus/common v0.44.0 github.com/segmentio/analytics-go/v3 v3.2.1 github.com/senthilrch/kube-fledged v0.10.0 github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.7.0 github.com/swaggo/files v1.0.1 github.com/swaggo/gin-swagger v1.6.0 github.com/swaggo/swag v1.8.12 github.com/toorop/gin-logrus v0.0.0-20210225092905-2c785434f26f github.com/urfave/cli/v2 v2.3.0 golang.org/x/net v0.14.0 golang.org/x/term v0.11.0 k8s.io/api v0.27.4 k8s.io/apimachinery v0.27.4 k8s.io/client-go v0.27.4 k8s.io/code-generator v0.27.4 k8s.io/klog v1.0.0 ) require ( github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect ) require ( github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect github.com/KyleBanks/depth v1.2.1 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect github.com/bytedance/sonic v1.9.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect github.com/cockroachdb/redact v1.1.5 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/docker/distribution v2.8.2+incompatible // indirect github.com/docker/go-units v0.5.0 // indirect github.com/dustin/go-humanize v1.0.0 // indirect github.com/emicklei/go-restful/v3 v3.9.0 // indirect github.com/evanphx/json-patch v5.6.0+incompatible // indirect github.com/gabriel-vasile/mimetype v1.4.2 // indirect github.com/getsentry/sentry-go v0.18.0 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/go-errors/errors v1.4.2 // indirect github.com/go-logr/logr v1.2.4 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect github.com/go-openapi/jsonreference v0.20.1 // indirect github.com/go-openapi/spec v0.20.8 // indirect github.com/go-openapi/swag v0.22.3 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.14.0 // indirect github.com/goccy/go-json v0.10.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/glog v1.0.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/google/btree v1.1.2 // indirect github.com/google/gnostic v0.6.9 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect github.com/imdario/mergo v0.3.13 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jackc/chunkreader/v2 v2.0.1 // indirect github.com/jackc/pgio v1.0.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgproto3/v2 v2.3.2 // indirect github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect github.com/jackc/pgtype v1.14.0 // indirect github.com/jackc/puddle v1.3.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/cpuid/v2 v2.2.4 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/leodido/go-urn v1.2.4 // indirect github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-isatty v0.0.19 // indirect github.com/mattn/go-runewidth v0.0.14 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/moby/spdystream v0.2.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect github.com/morikuni/aec v1.0.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.0-rc2 // indirect github.com/pelletier/go-toml/v2 v2.0.8 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect github.com/prometheus/client_model v0.4.0 // indirect github.com/prometheus/procfs v0.10.1 // indirect github.com/rancher/remotedialer v0.3.0 github.com/rivo/uniseg v0.4.2 // indirect github.com/rogpeppe/go-internal v1.11.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/segmentio/backo-go v1.0.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.11 // indirect github.com/xlab/treeprint v1.1.0 // indirect go.starlark.net v0.0.0-20221019144234-6ce4ce37fe55 // indirect golang.org/x/arch v0.3.0 // indirect golang.org/x/crypto v0.12.0 // indirect golang.org/x/mod v0.10.0 // indirect golang.org/x/oauth2 v0.8.0 // indirect golang.org/x/sync v0.2.0 // indirect golang.org/x/sys v0.11.0 // indirect golang.org/x/text v0.12.0 // indirect golang.org/x/time v0.1.0 // indirect golang.org/x/tools v0.9.3 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.30.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect gotest.tools/v3 v3.5.0 // indirect k8s.io/cli-runtime v0.27.4 // indirect k8s.io/gengo v0.0.0-20220902162205-c0856e24416d // indirect k8s.io/klog/v2 v2.90.1 // indirect k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // indirect k8s.io/utils v0.0.0-20230209194617-a36077c30491 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/kustomize/api v0.13.2 // indirect sigs.k8s.io/kustomize/kyaml v0.14.1 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect sigs.k8s.io/yaml v1.3.0 // indirect ) ================================================ FILE: go.sum ================================================ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s= github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams= github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= github.com/cockroachdb/errors v1.10.0 h1:lfxS8zZz1+OjtV4MtNWgboi/W5tyLEB6VQZBXN+0VUU= github.com/cockroachdb/errors v1.10.0/go.mod h1:lknhIsEVQ9Ss/qKDBQS/UqFSvPQjOwNq2qyKAxtHRqE= github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE= github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30= github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dchest/uniuri v1.2.0 h1:koIcOUdrTIivZgSLhHQvKgqdWZq5d7KdMEWF1Ud6+5g= github.com/dchest/uniuri v1.2.0/go.mod h1:fSzm4SLHzNZvWLvWJew423PhAzkpNQYq+uNLq4kxhkY= github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v23.0.2+incompatible h1:q81C2qQ/EhPm8COZMUGOQYh4qLv4Xu6CXELJ3WK/mlU= github.com/docker/docker v23.0.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustinkirkland/golang-petname v0.0.0-20230626224747-e794b9370d49 h1:6SNWi8VxQeCSwmLuTbEvJd7xvPmdS//zvMBWweZLgck= github.com/dustinkirkland/golang-petname v0.0.0-20230626224747-e794b9370d49/go.mod h1:V+Qd57rJe8gd4eiGzZyg4h54VLHmYVVw54iMnlAMrF8= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful/v3 v3.9.0 h1:XwGDlfxEnQZzuopoqxwSEllNcCOM9DhhFyhFIIGKwxE= github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/flowstack/go-jsonschema v0.1.1/go.mod h1:yL7fNggx1o8rm9RlgXv7hTBWxdBM0rVwpMwimd3F3N0= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg= github.com/getsentry/sentry-go v0.18.0 h1:MtBW5H9QgdcJabtZcuJG80BMOwaBpkRDZkxRkNC1sN0= github.com/getsentry/sentry-go v0.18.0/go.mod h1:Kgon4Mby+FJ7ZWHFUAZgVaIa8sxHtnRJRLTXZr51aKQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/cors v1.4.0 h1:oJ6gwtUl3lqV0WEIwM/LxPF1QZ5qe2lGWdY2+bz7y0g= github.com/gin-contrib/cors v1.4.0/go.mod h1:bs9pNM0x/UsmHPBWT2xZz9ROh8xYjYkiURUfmBoMlcs= github.com/gin-contrib/gzip v0.0.6 h1:NjcunTcGAj5CO1gn4N8jHOSIeRFHIbn51z6K+xaN4d4= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.7.3/go.mod h1:jD2toBW3GZUr5UMcdrwQA10I7RuaFOl/SGeDjXkfUtY= github.com/gin-gonic/gin v1.8.1/go.mod h1:ji8BvRH1azfM+SYow9zQ6SZMvR8qOMZHmsCuWR9tTTk= github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg= github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= github.com/go-openapi/jsonreference v0.20.0/go.mod h1:Ag74Ico3lPc+zR+qjn4XBUmXymS4zJbYVCZmcgkasdo= github.com/go-openapi/jsonreference v0.20.1 h1:FBLnyygC4/IZZr893oiomc9XaghoveYTrLC1F86HID8= github.com/go-openapi/jsonreference v0.20.1/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= github.com/go-openapi/spec v0.20.8 h1:ubHmXNY3FCIOinT8RNrrPfGc9t7I1qhPtdOGoG2AxRU= github.com/go-openapi/spec v0.20.8/go.mod h1:2OpW+JddWPrpXSCIX8eOx7lZ5iyuWj3RYR6VaaBKcWA= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= github.com/go-playground/validator/v10 v10.10.0/go.mod h1:74x4gJWsvQexRdW8Pn3dXSGrTK4nAUsbPlLADvpJkos= github.com/go-playground/validator/v10 v10.14.0 h1:vgvQWe3XCz3gIeFDm/HnTIbj6UGmg/+t63MyGU2n5js= github.com/go-playground/validator/v10 v10.14.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw= github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/mock v1.5.0 h1:jlYHihg//f7RRwuPfptm04yp4s7O6Kw8EZiVYIGcH0g= github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/gnostic v0.6.9 h1:ZK/5VhkoX835RikCHpSUJV9a+S3e1zLh59YnyWeBW+0= github.com/google/gnostic v0.6.9/go.mod h1:Nm8234We1lq6iB9OmlgNv3nH91XLLVZHCDayfA3xq+E= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 h1:+ngKgrYPPJrOjhax5N+uePQ0Fh1Z7PheYoUI/0nzkPA= github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA= github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE= github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsUgOEh9hBm+xYTstcNHg7UPMVJqRfQxq4s= github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o= github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY= github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= github.com/jackc/pgconn v1.14.0/go.mod h1:9mBNlny0UvkgJdCDvdVHYSjI+8tD2rnKK69Wz8ti++E= github.com/jackc/pgconn v1.14.1 h1:smbxIaZA08n6YuxEX1sDyjV/qkbtUtkH20qLkR9MUR4= github.com/jackc/pgconn v1.14.1/go.mod h1:9mBNlny0UvkgJdCDvdVHYSjI+8tD2rnKK69Wz8ti++E= github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= github.com/jackc/pgmock v0.0.0-20201204152224-4fe30f7445fd/go.mod h1:hrBW0Enj2AZTNpt/7Y5rr2xe/9Mn757Wtb2xeBzPv2c= github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65 h1:DadwsjnMwFjfWc9y5Wi/+Zz7xoE5ALHsRQlOctkOiHc= github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65/go.mod h1:5R2h2EEX+qri8jOWMbJCtaPWkrrNc7OHwsp2TCqp7ak= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg= github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/jackc/pgproto3/v2 v2.3.2 h1:7eY55bdBeCz1F2fTzSz69QC+pG46jYq9/jtSPiJ5nn0= github.com/jackc/pgproto3/v2 v2.3.2/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM= github.com/jackc/pgtype v1.14.0 h1:y+xUdabmyMkJLyApYuPj38mW+aAIqCe5uuBB51rH3Vw= github.com/jackc/pgtype v1.14.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs= github.com/jackc/pgx/v4 v4.18.1 h1:YP7G1KABtKpB5IHrO9vYwSrCOhs7p3uqhvhhQBptya0= github.com/jackc/pgx/v4 v4.18.1/go.mod h1:FydWkUyadDmdNH/mHnGob881GawxeEm7TcMCzkb+qQE= github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.3.0 h1:eHK/5clGOatcjX3oWGBO/MpxpbHzSwud5EWTSCI+MX0= github.com/jackc/puddle v1.3.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jedib0t/go-pretty/v6 v6.4.6 h1:v6aG9h6Uby3IusSSEjHaZNXpHFhzqMmjXcPq1Rjl9Jw= github.com/jedib0t/go-pretty/v6 v6.4.6/go.mod h1:Ndk3ase2CkQbXLLNf5QDHoYb6J9WtVfmHZu9n8rk2xs= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/moby/moby v24.0.4+incompatible h1:20Bf1sfJpspHMAUrxRFplG31Sriaw7Z9/jUEuJk6mqI= github.com/moby/moby v24.0.4+incompatible/go.mod h1:fDXVQ6+S340veQPv35CzDahGBmHsiclFwfEygB/TWMc= github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0= github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/namsral/flag v1.7.4-pre h1:b2ScHhoCUkbsq0d2C15Mv+VU8bl8hAXV8arnWiOHNZs= github.com/namsral/flag v1.7.4-pre/go.mod h1:OXldTctbM6SWH1K899kPZcf65KxJiD7MsceFUpB5yDo= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.27.8 h1:gegWiwZjBsf2DgiSbf5hpokZ98JVDMcWkUiigk6/KXc= github.com/onsi/gomega v1.27.8/go.mod h1:2J8vzI/s+2shY9XHRApDkdgPo1TKT7P2u6fXeJKFnNQ= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.0-rc2 h1:2zx/Stx4Wc5pIPDvIxHXvXtQFW/7XWJGmnM7r3wg034= github.com/opencontainers/image-spec v1.1.0-rc2/go.mod h1:3OVijpioIKYWTqjiG0zfF6wvoJ4fAXGbjdZuI2NgsRQ= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo= github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4= github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5 h1:Ii+DKncOVM8Cu1Hc+ETb5K+23HdAMvESYE3ZJ5b5cMI= github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/profile v1.6.0/go.mod h1:qBsxPvzyUincmltOk6iyRVxHYg4adc0OFOv72ZdLa18= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rancher/remotedialer v0.3.0 h1:y1EO8JCsgZo0RcqTUp6U8FXcBAv27R+TLnWRcpvX1sM= github.com/rancher/remotedialer v0.3.0/go.mod h1:BwwztuvViX2JrLLUwDlsYt5DiyUwHLlzynRwkZLAY0Q= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.2 h1:YwD0ulJSJytLpiaWua0sBDusfsCZohxjxzVTYjwxfV8= github.com/rivo/uniseg v0.4.2/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/segmentio/analytics-go/v3 v3.2.1 h1:G+f90zxtc1p9G+WigVyTR0xNfOghOGs/PYAlljLOyeg= github.com/segmentio/analytics-go/v3 v3.2.1/go.mod h1:p8owAF8X+5o27jmvUognuXxdtqvSGtD0ZrfY2kcS9bE= github.com/segmentio/backo-go v1.0.0 h1:kbOAtGJY2DqOR0jfRkYEorx/b18RgtepGtY3+Cpe6qA= github.com/segmentio/backo-go v1.0.0/go.mod h1:kJ9mm9YmoWSkk+oQ+5Cj8DEoRCX2JT6As4kEtIIOp1M= github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk= github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.4/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/swaggo/files v1.0.1 h1:J1bVJ4XHZNq0I46UU90611i9/YzdrF7x92oX1ig5IdE= github.com/swaggo/files v1.0.1/go.mod h1:0qXmMNH6sXNf+73t65aKeB+ApmgxdnkQzVTAj2uaMUg= github.com/swaggo/gin-swagger v1.6.0 h1:y8sxvQ3E20/RCyrXeFfg60r6H0Z+SwpTjMYsMm+zy8M= github.com/swaggo/gin-swagger v1.6.0/go.mod h1:BG00cCEy294xtVpyIAHG6+e2Qzj/xKlRdOqDkvq0uzo= github.com/swaggo/swag v1.8.12 h1:pctzkNPu0AlQP2royqX3apjKCQonAnf7KGoxeO4y64w= github.com/swaggo/swag v1.8.12/go.mod h1:lNfm6Gg+oAq3zRJQNEMBE66LIJKM44mxFqhEEgy2its= github.com/tensorchord/forwarder v0.0.0-20230713171536-b1b52b398d3a h1:q4GoeuagHfbdl7JGSU0AcArYXnsA3p2+dzBdx7AZHP0= github.com/tensorchord/forwarder v0.0.0-20230713171536-b1b52b398d3a/go.mod h1:PfpNmyy0g95SWDoSxXH5MPAlFJ9S04w7cBmIMS6U89U= github.com/tensorchord/kube-fledged v0.2.0 h1:wNJNcot0/CxLRtdnG34/EVwVI8WgLoJ2uxAfiUtVfNg= github.com/tensorchord/kube-fledged v0.2.0/go.mod h1:m4ncbmr05mQDRdJAl+UuCcgf6cpipU333En8lbob2u4= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/toorop/gin-logrus v0.0.0-20210225092905-2c785434f26f h1:oqdnd6OGlOUu1InG37hWcCB3a+Jy3fwjylyVboaNMwY= github.com/toorop/gin-logrus v0.0.0-20210225092905-2c785434f26f/go.mod h1:X3Dd1SB8Gt1V968NTzpKFjMM6O8ccta2NPC6MprOxZQ= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go v1.2.7/go.mod h1:nF9osbDWLy6bDVv/Rtoh6QgnvNDpmCalQV5urGCCS6M= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY= github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M= github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= github.com/xlab/treeprint v1.1.0 h1:G/1DjNkPpfZCFt9CSh6b5/nY4VimlbHF3Rh4obvtzDk= github.com/xlab/treeprint v1.1.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5/go.mod h1:nmDLcffg48OtT/PSW0Hg7FvpRQsQh5OSqIylirxKC7o= go.starlark.net v0.0.0-20221019144234-6ce4ce37fe55 h1:UETCDFV7xVE6L29SnwA1vzkJEYGwffjjmxURPkstP6A= go.starlark.net v0.0.0-20221019144234-6ce4ce37fe55/go.mod h1:kIVgS18CjmEC3PqMd5kaJSGEifyV/CeB9x506ZJ1Vbk= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k= golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.8.0 h1:6dkIjl3j3LtZ/O3sTgZTMsLKSftL/B8Zgq4huOIIUu8= golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191002063906-3421d5a6bb1c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0= golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.1.0 h1:xYY+Bajn2a7VBmTM5GikTmnK8ZuX8YgnQCqZpbBNtmA= golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.9.3 h1:Gn1I8+64MsuTb/HpH+LmQtNas23LhUVr3rYZ0eKuaMM= golang.org/x/tools v0.9.3/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= google.golang.org/api v0.44.0/go.mod h1:EBOGZqzyhtvMDoxwS97ctnh0zUmYY6CxqXsc1AvkYD8= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.5.0 h1:Ljk6PdHdOhAb5aDMWXjDLMMhph+BpztA4v1QdqEW2eY= gotest.tools/v3 v3.5.0/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= k8s.io/api v0.23.5/go.mod h1:Na4XuKng8PXJ2JsploYYrivXrINeTaycCGcYgF91Xm8= k8s.io/api v0.27.4 h1:0pCo/AN9hONazBKlNUdhQymmnfLRbSZjd5H5H3f0bSs= k8s.io/api v0.27.4/go.mod h1:O3smaaX15NfxjzILfiln1D8Z3+gEYpjEpiNA/1EVK1Y= k8s.io/apimachinery v0.23.5/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM= k8s.io/apimachinery v0.27.4 h1:CdxflD4AF61yewuid0fLl6bM4a3q04jWel0IlP+aYjs= k8s.io/apimachinery v0.27.4/go.mod h1:XNfZ6xklnMCOGGFNqXG7bUrQCoR04dh/E7FprV6pb+E= k8s.io/cli-runtime v0.23.5/go.mod h1:oY6QDF2qo9xndSq32tqcmRp2UyXssdGrLfjAVymgbx4= k8s.io/cli-runtime v0.27.4 h1:Zb0eci+58eHZNnoHhjRFc7W88s8dlG12VtIl3Nv2Hto= k8s.io/cli-runtime v0.27.4/go.mod h1:k9Z1xiZq2xNplQmehpDquLgc+rE+pubpO1cK4al4Mlw= k8s.io/client-go v0.23.5/go.mod h1:flkeinTO1CirYgzMPRWxUCnV0G4Fbu2vLhYCObnt/r4= k8s.io/client-go v0.27.4 h1:vj2YTtSJ6J4KxaC88P4pMPEQECWMY8gqPqsTgUKzvjk= k8s.io/client-go v0.27.4/go.mod h1:ragcly7lUlN0SRPk5/ZkGnDjPknzb37TICq07WhI6Xc= k8s.io/code-generator v0.27.4 h1:bw2xFEBnthhCSC7Bt6FFHhPTfWX21IJ30GXxOzywsFE= k8s.io/code-generator v0.27.4/go.mod h1:DPung1sI5vBgn4AGKtlPRQAyagj/ir/4jI55ipZHVww= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/gengo v0.0.0-20220902162205-c0856e24416d h1:U9tB195lKdzwqicbJvyJeOXV7Klv+wNAWENRnXEGi08= k8s.io/gengo v0.0.0-20220902162205-c0856e24416d/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/klog/v2 v2.90.1 h1:m4bYOKall2MmOiRaR1J+We67Do7vm9KiQVlT96lnHUw= k8s.io/klog/v2 v2.90.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f h1:2kWPakN3i/k81b0gvD5C5FJ2kxm1WrQFanWchyKuqGg= k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f/go.mod h1:byini6yhqGC14c3ebc/QwanvYwhuMWF6yz2F8uwW8eg= k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20230209194617-a36077c30491 h1:r0BAOLElQnnFhE/ApUsg3iHdVYYPBjNSSOMowRZxxsY= k8s.io/utils v0.0.0-20230209194617-a36077c30491/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/kustomize/api v0.10.1/go.mod h1:2FigT1QN6xKdcnGS2Ppp1uIWrtWN28Ms8A3OZUZhwr8= sigs.k8s.io/kustomize/api v0.13.2 h1:kejWfLeJhUsTGioDoFNJET5LQe/ajzXhJGYoU+pJsiA= sigs.k8s.io/kustomize/api v0.13.2/go.mod h1:DUp325VVMFVcQSq+ZxyDisA8wtldwHxLZbr1g94UHsw= sigs.k8s.io/kustomize/kyaml v0.13.0/go.mod h1:FTJxEZ86ScK184NpGSAQcfEqee0nul8oLCK30D47m4E= sigs.k8s.io/kustomize/kyaml v0.14.1 h1:c8iibius7l24G2wVAGZn/Va2wNys03GXLjYVIcFVxKA= sigs.k8s.io/kustomize/kyaml v0.14.1/go.mod h1:AN1/IpawKilWD7V+YvQwRGUvuUOOWpjsHu6uHwonSF4= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= ================================================ FILE: ingress-operator/.DEREK.yml ================================================ redirect: https://raw.githubusercontent.com/openfaas/faas/master/.DEREK.yml ================================================ FILE: ingress-operator/.dockerignore ================================================ .git .github .vscode .tools artifacts examples hack ================================================ FILE: ingress-operator/.gitignore ================================================ faas-o6s # Binaries for programs and plugins *.exe *.dll *.so *.dylib # Test binary, build with `go test -c` *.test # Output of the go coverage tool, specifically when used with LiteIDE *.out # Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736 .glide/ .idea/ bin/ password.txt faas-netes/** test.yaml cluster.yaml ================================================ FILE: ingress-operator/.tools/README.md ================================================ # Tools folder The tools folder allows us a space to define and pin external dependencies. If they are go based tools we can create individual `mod` files that allow us to download or install these tools independent of the main package. ## Tools 1. `k8s.io/code-gen` this package needs to be _downloaded_ not installed. But we can not use the copy created in the `vendor` folder because vendor does not make a complete clone, it only keeps the Go files, and the code-gen project has several bash scripts that we need to reference. The main project `Makefile` will attempt to keep the `code-generator.mod` file in sync with the `go.mod`. It should not need to be manually edited, but it does need to be committed. ================================================ FILE: ingress-operator/.tools/code-generator.mod ================================================ module _ // Fake module so that we can install code-generator separate from the project go 1.16 require ( k8s.io/code-generator v0.21.3 ) ================================================ FILE: ingress-operator/.tools/code-generator.sum ================================================ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= github.com/go-openapi/spec v0.19.5/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk= github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= k8s.io/code-generator v0.21.3/go.mod h1:K3y0Bv9Cz2cOW2vXUrNZlFbflhuPvuadW6JdnN6gGKo= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.8.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7/go.mod h1:wXW5VT87nVfh/iLV8FpR2uDvrFyomxbtb1KivDbvPTE= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= ================================================ FILE: ingress-operator/.vscode/settings.json ================================================ { "go.inferGopath": false, "cSpell.words": [ "Infof", "appsv", "corev", "faas", "faasclientset", "faasscheme", "faasv", "handler", "klog", "kube", "kubeclientset", "kubeconfig", "kubeinformers", "logtostderr", "metav", "netv", "networkingv", "sync", "syncer", "threadiness", "traefik" ] } ================================================ FILE: ingress-operator/Dockerfile ================================================ FROM ubuntu:22.04 LABEL maintainer="modelz-support@tensorchord.ai" COPY ingress-operator /usr/bin/ingress-operator ENTRYPOINT ["/usr/bin/ingress-operator"] ================================================ FILE: ingress-operator/LICENSE ================================================ MIT License Copyright (c) 2023 TensorChord Inc. Copyright (c) 2017-2019 OpenFaaS Author(s) Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: ingress-operator/Makefile ================================================ # Copyright 2022 TensorChord Inc. # # The old school Makefile, following are required targets. The Makefile is written # to allow building multiple binaries. You are free to add more targets or change # existing implementations, as long as the semantics are preserved. # # make - default to 'build' target # make lint - code analysis # make test - run unit test (or plus integration test) # make build - alias to build-local target # make build-local - build local binary targets # make build-linux - build linux binary targets # make container - build containers # $ docker login registry -u username -p xxxxx # make push - push containers # make clean - clean up targets # # Not included but recommended targets: # make e2e-test # # The makefile is also responsible to populate project version information. # # # Tweak the variables based on your project. # # This repo's root import path (under GOPATH). ROOT := github.com/tensorchord/openmodelz/ingress-operator # Target binaries. You can build multiple binaries for a single project. TARGETS := ingress-operator # Container image prefix and suffix added to targets. # The final built images are: # $[REGISTRY]/$[IMAGE_PREFIX]$[TARGET]$[IMAGE_SUFFIX]:$[VERSION] # $[REGISTRY] is an item from $[REGISTRIES], $[TARGET] is an item from $[TARGETS]. IMAGE_PREFIX ?= $(strip ) IMAGE_SUFFIX ?= $(strip ) # Container registries. REGISTRY ?= ghcr.io/tensorchord # Container registry for base images. BASE_REGISTRY ?= docker.io BASE_REGISTRY_USER ?= modelzai # Disable CGO by default. CGO_ENABLED ?= 0 # # These variables should not need tweaking. # # It's necessary to set this because some environments don't link sh -> bash. export SHELL := bash # It's necessary to set the errexit flags for the bash shell. export SHELLOPTS := errexit PACKAGE_NAME := github.com/tensorchord/openmodelz/ingress-operator GOLANG_CROSS_VERSION ?= v1.17.6 # Project main package location (can be multiple ones). CMD_DIR := ./cmd # Project output directory. OUTPUT_DIR := ./bin DEBUG_DIR := ./debug-bin # Build directory. BUILD_DIR := ./build # Current version of the project. VERSION ?= $(shell git describe --match 'v[0-9]*' --always --tags --abbrev=0) BUILD_DATE=$(shell date -u +'%Y-%m-%dT%H:%M:%SZ') GIT_COMMIT=$(shell git rev-parse HEAD) GIT_TAG=$(shell if [ -z "`git status --porcelain`" ]; then git describe --exact-match --tags HEAD 2>/dev/null; fi) GIT_TREE_STATE=$(shell if [ -z "`git status --porcelain`" ]; then echo "clean" ; else echo "dirty"; fi) GITSHA ?= $(shell git rev-parse --short HEAD) # Track code version with Docker Label. DOCKER_LABELS ?= git-describe="$(shell date -u +v%Y%m%d)-$(shell git describe --tags --always --dirty)" # Golang standard bin directory. GOPATH ?= $(shell go env GOPATH) GOROOT ?= $(shell go env GOROOT) BIN_DIR := $(GOPATH)/bin GOLANGCI_LINT := $(BIN_DIR)/golangci-lint # check if we need embed the dashboard DASHBOARD_BUILD ?= debug # Default golang flags used in build and test # -mod=vendor: force go to use the vendor files instead of using the `$GOPATH/pkg/mod` # -p: the number of programs that can be run in parallel # -count: run each test and benchmark 1 times. Set this flag to disable test cache export GOFLAGS ?= -count=1 # # Define all targets. At least the following commands are required: # # All targets. .PHONY: help lint test build container push addlicense debug debug-local build-local generate clean test-local addlicense-install release build-image .DEFAULT_GOAL:=build build: build-local ## Build the release version of envd help: ## Display this help @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z0-9_-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) debug: debug-local ## Build the debug version of envd # more info about `GOGC` env: https://github.com/golangci/golangci-lint#memory-usage-of-golangci-lint lint: $(GOLANGCI_LINT) ## Lint GO code @$(GOLANGCI_LINT) run $(GOLANGCI_LINT): curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $$(go env GOPATH)/bin mockgen-install: go install github.com/golang/mock/mockgen@v1.6.0 addlicense-install: go install github.com/google/addlicense@latest build-local: @for target in $(TARGETS); do \ CGO_ENABLED=$(CGO_ENABLED) GOOS=$(GOOS) GOARCH=$(GOARCH) go build -tags $(DASHBOARD_BUILD) -trimpath -v -o $(OUTPUT_DIR)/$${target} \ -ldflags "-s -w -X $(ROOT)/pkg/version.version=$(VERSION) -X $(ROOT)/pkg/version.buildDate=$(BUILD_DATE) -X $(ROOT)/pkg/version.gitCommit=$(GIT_COMMIT) -X $(ROOT)/pkg/version.gitTreeState=$(GIT_TREE_STATE)" \ $(CMD_DIR)/$${target}; \ done # It is used by vscode to attach into the process. debug-local: @for target in $(TARGETS); do \ CGO_ENABLED=$(CGO_ENABLED) go build -tags $(DASHBOARD_BUILD) -trimpath \ -v -o $(DEBUG_DIR)/$${target} \ -gcflags='all=-N -l' \ $(CMD_DIR)/$${target}; \ done addlicense: addlicense-install ## Add license to GO code files addlicense -l mpl -c "TensorChord Inc." $$(find . -type f -name '*.go' | grep -v pkg/docs/docs.go) test-local: @go test -tags=$(DASHBOARD_BUILD) -v -race -coverprofile=coverage.out ./... test: ## Run the tests @go test -tags=$(DASHBOARD_BUILD) -race -coverpkg=./pkg/... -coverprofile=coverage.out ./... @go tool cover -func coverage.out | tail -n 1 | awk '{ print "Total coverage: " $$3 }' clean: ## Clean the outputs and artifacts @-rm -vrf ${OUTPUT_DIR} @-rm -vrf ${DEBUG_DIR} @-rm -vrf build dist .eggs *.egg-info fmt: ## Run go fmt against code. go fmt ./... vet: ## Run go vet against code. go vet ./... build-image: build-local docker build -t ${BASE_REGISTRY}/${BASE_REGISTRY_USER}/ingress-operator:dev -f Dockerfile ./bin docker push ${BASE_REGISTRY}/${BASE_REGISTRY_USER}/ingress-operator:dev release: @if [ ! -f ".release-env" ]; then \ echo "\033[91m.release-env is required for release\033[0m";\ exit 1;\ fi docker run \ --rm \ --privileged \ -e CGO_ENABLED=1 \ --env-file .release-env \ -v /var/run/docker.sock:/var/run/docker.sock \ -v `pwd`:/go/src/$(PACKAGE_NAME) \ -v `pwd`/sysroot:/sysroot \ -w /go/src/$(PACKAGE_NAME) \ goreleaser/goreleaser-cross:${GOLANG_CROSS_VERSION} \ release --rm-dist ================================================ FILE: ingress-operator/artifacts/.gitignore ================================================ ================================================ FILE: ingress-operator/artifacts/crds/tensorchord.ai_inferenceingresses.yaml ================================================ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.5.0 creationTimestamp: null name: inferenceingresses.tensorchord.ai spec: group: tensorchord.ai names: kind: InferenceIngress listKind: InferenceIngressList plural: inferenceingresses singular: inferenceingress scope: Namespaced versions: - additionalPrinterColumns: - jsonPath: .spec.domain name: Domain type: string name: v1 schema: openAPIV3Schema: description: InferenceIngress describes an OpenFaaS function type: object required: - spec properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object spec: description: InferenceIngressSpec is the spec for a InferenceIngressSpec resource. It must be created in the same namespace as the gateway, i.e. openfaas. type: object required: - domain - framework - function properties: bypassGateway: description: BypassGateway, when true creates an Ingress record directly for the Function name without using the gateway in the hot path type: boolean domain: description: Domain such as "api.example.com" type: string framework: type: string function: description: Function such as "nodeinfo" type: string ingressType: description: IngressType such as "nginx" type: string path: description: Path such as "/v1/profiles/view/(.*)", or leave empty for default type: string tls: description: Enable TLS via cert-manager type: object properties: enabled: type: boolean issuerRef: description: ObjectReference is a reference to an object with a given name and kind. type: object required: - name properties: kind: type: string name: type: string served: true storage: true subresources: {} status: acceptedNames: kind: "" plural: "" conditions: [] storedVersions: [] ================================================ FILE: ingress-operator/artifacts/operator-amd64.yaml ================================================ --- apiVersion: apps/v1 kind: Deployment metadata: name: ingress-operator namespace: openfaas spec: replicas: 1 selector: matchLabels: app: ingress-operator template: metadata: labels: app: ingress-operator annotations: prometheus.io.scrape: 'false' spec: serviceAccountName: ingress-operator containers: - name: operator image: docker.io/alexellis2/ingress-operator:1 imagePullPolicy: Always command: - ./ingress-operator env: - name: ingress_namespace value: openfaas resources: limits: memory: 128Mi requests: memory: 25Mi ================================================ FILE: ingress-operator/artifacts/operator-rbac.yaml ================================================ --- apiVersion: v1 kind: ServiceAccount metadata: name: ingress-operator namespace: openfaas --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: ingress-operator-rw namespace: openfaas rules: - apiGroups: ["openfaas.com"] resources: ["functioningresses"] verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] - apiGroups: ["extensions", "networking", "networking.k8s.io"] resources: ["ingresses"] verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] - apiGroups: [""] resources: ["events"] verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] # - apiGroups: ["certmanager.k8s.io"] # resources: ["certificates"] # verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: ingress-operator-rw namespace: openfaas roleRef: apiGroup: rbac.authorization.k8s.io kind: Role name: ingress-operator-rw subjects: - kind: ServiceAccount name: ingress-operator namespace: openfaas ================================================ FILE: ingress-operator/cmd/ingress-operator/main.go ================================================ package main import ( "fmt" "os" cli "github.com/urfave/cli/v2" klog "k8s.io/klog" // required to authenticate against GKE clusters _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" "github.com/tensorchord/openmodelz/ingress-operator/pkg/app" "github.com/tensorchord/openmodelz/ingress-operator/pkg/version" ) func run(args []string) error { cli.VersionPrinter = func(c *cli.Context) { fmt.Println(c.App.Name, version.Package, c.App.Version, version.Revision) } klog.InitFlags(nil) a := app.New() return a.Run(args) } func handleErr(err error) { if err == nil { return } klog.Error(err) os.Exit(1) } func main() { err := run(os.Args) handleErr(err) } ================================================ FILE: ingress-operator/hack/boilerplate.go.txt ================================================ /* Copyright 2023 TensorChord Inc. Licensed under the MIT license. See LICENSE file in the project root for full license information. */ ================================================ FILE: ingress-operator/hack/custom-boilerplate.go.txt ================================================ /* Copyright 2023 OpenFaaS Author(s) Licensed under the MIT license. See LICENSE file in the project root for full license information. */ ================================================ FILE: ingress-operator/hack/print-codegen-version.sh ================================================ #!/bin/bash # This scripts exists primarily so that it can be used in the Makefile. # It is needed because the `($shell ...)` command was having issues with the pipe. # Extracting it to a script was the simplest solution. grep 'k8s.io/code-generator' go.mod | awk '{print $2}' ================================================ FILE: ingress-operator/hack/update-codegen.sh ================================================ #!/usr/bin/env bash # copied from: https://github.com/weaveworks/flagger/tree/master/hack set -o errexit set -o nounset set -o pipefail SCRIPT_ROOT=$(git rev-parse --show-toplevel)/ingress-operator echo "SCRIPT_ROOT is ${SCRIPT_ROOT}" # Grab code-generator version from go.sum. CODEGEN_VERSION=$(grep 'k8s.io/code-generator' go.sum | awk '{print $2}' | head -1) CODEGEN_PKG=$(echo `go env GOPATH`"/pkg/mod/k8s.io/code-generator@${CODEGEN_VERSION}") echo ">> Using ${CODEGEN_PKG}" # code-generator does work with go.mod but makes assumptions about # the project living in `$GOPATH/src`. To work around this and support # any location; create a temporary directory, use this as an output # base, and copy everything back once generated. TEMP_DIR=$(mktemp -d) cleanup() { echo ">> Removing ${TEMP_DIR}" rm -rf ${TEMP_DIR} } trap "cleanup" EXIT SIGINT echo ">> Temporary output directory ${TEMP_DIR}" # Ensure we can execute. chmod +x ${CODEGEN_PKG}/generate-groups.sh ${CODEGEN_PKG}/generate-groups.sh all \ github.com/tensorchord/openmodelz/ingress-operator/pkg/client github.com/tensorchord/openmodelz/ingress-operator/pkg/apis \ modelzetes:v1 \ --output-base "${TEMP_DIR}" \ --go-header-file ${SCRIPT_ROOT}/hack/boilerplate.go.txt # Copy everything back. cp -r "${TEMP_DIR}/github.com/tensorchord/openmodelz/ingress-operator/." "${SCRIPT_ROOT}/" ================================================ FILE: ingress-operator/hack/update-crds.sh ================================================ #!/bin/bash export controllergen="$GOPATH/bin/controller-gen" export PKG=sigs.k8s.io/controller-tools/cmd/controller-gen@v0.7.0 if [ ! -e "$controllergen" ]; then echo "Getting $PKG" go install $PKG fi "$controllergen" \ crd \ schemapatch:manifests=./artifacts/crds \ paths=./pkg/apis/... \ output:dir=./artifacts/crds ================================================ FILE: ingress-operator/hack/verify-codegen.sh ================================================ #!/usr/bin/env bash set -o errexit set -o nounset set -o pipefail SCRIPT_ROOT=$(git rev-parse --show-toplevel) DIFFROOT="${SCRIPT_ROOT}/pkg" TMP_DIFFROOT="${SCRIPT_ROOT}/_tmp/pkg" _tmp="${SCRIPT_ROOT}/_tmp" cleanup() { rm -rf "${_tmp}" } trap "cleanup" EXIT SIGINT cleanup mkdir -p "${TMP_DIFFROOT}" cp -a "${DIFFROOT}"/* "${TMP_DIFFROOT}" "${SCRIPT_ROOT}/hack/update-codegen.sh" echo "diffing ${DIFFROOT} against freshly generated codegen" ret=0 diff -Naupr "${DIFFROOT}" "${TMP_DIFFROOT}" || ret=$? cp -a "${TMP_DIFFROOT}"/* "${DIFFROOT}" if [[ $ret -eq 0 ]] then echo "${DIFFROOT} up to date." else echo "${DIFFROOT} is out of date. Please run hack/update-codegen.sh" exit 1 fi ================================================ FILE: ingress-operator/pkg/apis/modelzetes/register.go ================================================ package modelzetes const ( GroupName = "tensorchord.ai" ) ================================================ FILE: ingress-operator/pkg/apis/modelzetes/v1/doc.go ================================================ // +k8s:deepcopy-gen=package,register // Package v1 is the OpenFaaS v1 version of the API. // +groupName=tensorchord.ai package v1 ================================================ FILE: ingress-operator/pkg/apis/modelzetes/v1/register.go ================================================ package v1 import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" controller "github.com/tensorchord/openmodelz/ingress-operator/pkg/apis/modelzetes" ) // SchemeGroupVersion is group version used to register these objects var SchemeGroupVersion = schema.GroupVersion{Group: controller.GroupName, Version: "v1"} // Resource takes an unqualified resource and returns a Group qualified GroupResource func Resource(resource string) schema.GroupResource { return SchemeGroupVersion.WithResource(resource).GroupResource() } var ( // localSchemeBuilder and AddToScheme will stay in k8s.io/kubernetes. SchemeBuilder runtime.SchemeBuilder localSchemeBuilder = &SchemeBuilder AddToScheme = localSchemeBuilder.AddToScheme ) func init() { // We only register manually written functions here. The registration of the // generated functions takes place in the generated files. The separation // makes the code compile even when the generated files are missing. localSchemeBuilder.Register(addKnownTypes) } // Adds the list of known types to api.Scheme. func addKnownTypes(scheme *runtime.Scheme) error { scheme.AddKnownTypes(SchemeGroupVersion, &InferenceIngress{}, &InferenceIngressList{}, ) metav1.AddToGroupVersion(scheme, SchemeGroupVersion) return nil } ================================================ FILE: ingress-operator/pkg/apis/modelzetes/v1/types.go ================================================ package v1 import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // +genclient // +genclient:noStatus // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object // +kubebuilder:printcolumn:name="Domain",type=string,JSONPath=`.spec.domain` // InferenceIngress describes an OpenFaaS function type InferenceIngress struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` Spec InferenceIngressSpec `json:"spec"` } // InferenceIngressSpec is the spec for a InferenceIngressSpec resource. It must // be created in the same namespace as the gateway, i.e. openfaas. type InferenceIngressSpec struct { // Domain such as "api.example.com" Domain string `json:"domain"` // Function such as "nodeinfo" Function string `json:"function"` Framework string `json:"framework"` // Path such as "/v1/profiles/view/(.*)", or leave empty for default // +optional Path string `json:"path"` // IngressType such as "nginx" // +optional IngressType string `json:"ingressType,omitempty"` // Enable TLS via cert-manager // +optional TLS *InferenceIngressTLS `json:"tls,omitempty"` // BypassGateway, when true creates an Ingress record // directly for the Function name without using the gateway // in the hot path // +optional BypassGateway bool `json:"bypassGateway,omitempty"` } // InferenceIngressSpec TLS options type InferenceIngressTLS struct { // +optional Enabled bool `json:"enabled"` // +optional IssuerRef ObjectReference `json:"issuerRef"` } // UseTLS if TLS is enabled func (f *InferenceIngressSpec) UseTLS() bool { return f.TLS != nil && f.TLS.Enabled } // ObjectReference is a reference to an object with a given name and kind. type ObjectReference struct { Name string `json:"name"` // +optional Kind string `json:"kind,omitempty"` } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object // InferenceIngress is a list of Function resources type InferenceIngressList struct { metav1.TypeMeta `json:",inline"` metav1.ListMeta `json:"metadata"` Items []InferenceIngress `json:"items"` } ================================================ FILE: ingress-operator/pkg/apis/modelzetes/v1/zz_generated.deepcopy.go ================================================ //go:build !ignore_autogenerated // +build !ignore_autogenerated /* Copyright 2023 TensorChord Inc. Licensed under the MIT license. See LICENSE file in the project root for full license information. */ // Code generated by deepcopy-gen. DO NOT EDIT. package v1 import ( runtime "k8s.io/apimachinery/pkg/runtime" ) // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *InferenceIngress) DeepCopyInto(out *InferenceIngress) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) in.Spec.DeepCopyInto(&out.Spec) return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InferenceIngress. func (in *InferenceIngress) DeepCopy() *InferenceIngress { if in == nil { return nil } out := new(InferenceIngress) in.DeepCopyInto(out) return out } // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. func (in *InferenceIngress) DeepCopyObject() runtime.Object { if c := in.DeepCopy(); c != nil { return c } return nil } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *InferenceIngressList) DeepCopyInto(out *InferenceIngressList) { *out = *in out.TypeMeta = in.TypeMeta in.ListMeta.DeepCopyInto(&out.ListMeta) if in.Items != nil { in, out := &in.Items, &out.Items *out = make([]InferenceIngress, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } } return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InferenceIngressList. func (in *InferenceIngressList) DeepCopy() *InferenceIngressList { if in == nil { return nil } out := new(InferenceIngressList) in.DeepCopyInto(out) return out } // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. func (in *InferenceIngressList) DeepCopyObject() runtime.Object { if c := in.DeepCopy(); c != nil { return c } return nil } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *InferenceIngressSpec) DeepCopyInto(out *InferenceIngressSpec) { *out = *in if in.TLS != nil { in, out := &in.TLS, &out.TLS *out = new(InferenceIngressTLS) **out = **in } return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InferenceIngressSpec. func (in *InferenceIngressSpec) DeepCopy() *InferenceIngressSpec { if in == nil { return nil } out := new(InferenceIngressSpec) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *InferenceIngressTLS) DeepCopyInto(out *InferenceIngressTLS) { *out = *in out.IssuerRef = in.IssuerRef return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InferenceIngressTLS. func (in *InferenceIngressTLS) DeepCopy() *InferenceIngressTLS { if in == nil { return nil } out := new(InferenceIngressTLS) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ObjectReference) DeepCopyInto(out *ObjectReference) { *out = *in return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ObjectReference. func (in *ObjectReference) DeepCopy() *ObjectReference { if in == nil { return nil } out := new(ObjectReference) in.DeepCopyInto(out) return out } ================================================ FILE: ingress-operator/pkg/app/config.go ================================================ package app import ( cli "github.com/urfave/cli/v2" "github.com/tensorchord/openmodelz/ingress-operator/pkg/config" ) func configFromCLI(c *cli.Context) config.Config { cfg := config.Config{} // kubernetes cfg.KubeConfig.Kubeconfig = c.String(flagKubeConfig) cfg.KubeConfig.MasterURL = c.String(flagMasterURL) cfg.KubeConfig.QPS = c.Int(flagQPS) cfg.KubeConfig.Burst = c.Int(flagBurst) cfg.KubeConfig.ResyncPeriod = c.Duration(flagResyncPeriod) // controller cfg.Controller.ThreadCount = c.Int(flagControllerThreads) cfg.Controller.Namespace = c.String(flagNamespace) cfg.Controller.Host = c.String(flagHost) return cfg } ================================================ FILE: ingress-operator/pkg/app/root.go ================================================ // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. package app import ( "time" "github.com/gin-gonic/gin" "github.com/pkg/errors" "github.com/sirupsen/logrus" cli "github.com/urfave/cli/v2" controller "github.com/tensorchord/openmodelz/ingress-operator/pkg/controller/v1" "github.com/tensorchord/openmodelz/ingress-operator/pkg/signals" "github.com/tensorchord/openmodelz/ingress-operator/pkg/version" ) const ( flagDebug = "debug" // kubernetes flagMasterURL = "master-url" flagKubeConfig = "kube-config" flagQPS = "kube-qps" flagBurst = "kube-burst" flagResyncPeriod = "kube-resync-period" // controller flagControllerThreads = "controller-thread-count" flagNamespace = "namespace" flagHost = "host" ) type App struct { *cli.App } func New() App { internalApp := cli.NewApp() internalApp.EnableBashCompletion = true internalApp.Name = "ingress-operator" internalApp.Usage = "kubernetes operator for inference ingress" internalApp.HideHelpCommand = true internalApp.HideVersion = false internalApp.Version = version.GetVersion().String() internalApp.Flags = []cli.Flag{ &cli.BoolFlag{ Name: flagDebug, Usage: "enable debug output in logs", EnvVars: []string{"DEBUG"}, }, &cli.StringFlag{ Name: flagMasterURL, Usage: "URL to master for kubernetes cluster", EnvVars: []string{"MODELZ_MASTER_URL"}, Aliases: []string{"mu"}, }, &cli.StringFlag{ Name: flagKubeConfig, Usage: "Path to kubeconfig file. If not provided, will use in-cluster config", EnvVars: []string{"MODELZ_KUBE_CONFIG"}, Aliases: []string{"kc"}, }, &cli.IntFlag{ Name: flagQPS, Usage: "QPS for kubernetes client", Value: 100, EnvVars: []string{"MODELZ_KUBE_QPS"}, Aliases: []string{"kq"}, }, &cli.IntFlag{ Name: flagBurst, Value: 250, Usage: "Burst for kubernetes client", EnvVars: []string{"MODELZ_KUBE_BURST"}, Aliases: []string{"kb"}, }, &cli.DurationFlag{ Name: flagResyncPeriod, Value: time.Minute * 5, Usage: "Resync period for kubernetes client", EnvVars: []string{"MODELZ_KUBE_RESYNC_PERIOD"}, Aliases: []string{"kr"}, }, &cli.IntFlag{ Name: flagControllerThreads, Value: 1, Usage: "Number of threads to use for controller", EnvVars: []string{"MODELZ_CONTROLLER_THREAD_COUNT"}, Aliases: []string{"ct"}, }, &cli.StringFlag{ Name: flagNamespace, Value: "default", Usage: "Namespace to create the ingress in. (We need to keep the same namespace as the inference ingress, because kubernetes does not allow cross namespace owner references)", EnvVars: []string{"MODELZ_NAMESPACE"}, Aliases: []string{"ns"}, }, &cli.StringFlag{ Name: flagHost, Value: "apiserver", Usage: "Host to redirect the request to. (apiserver, agent)", EnvVars: []string{"MODELZ_HOST"}, }, } internalApp.Action = runServer // Deal with debug flag. var debugEnabled bool internalApp.Before = func(context *cli.Context) error { debugEnabled = context.Bool(flagDebug) if debugEnabled { logrus.SetReportCaller(true) logrus.SetFormatter(&logrus.TextFormatter{FullTimestamp: true}) logrus.SetLevel(logrus.DebugLevel) gin.SetMode(gin.DebugMode) } else { logrus.SetFormatter(&logrus.JSONFormatter{}) } return nil } return App{ App: internalApp, } } func runServer(clicontext *cli.Context) error { c := configFromCLI(clicontext) cfgString, _ := c.GetString() logrus.WithField("config", c).Info("starting ingress operator") if err := c.Validate(); err != nil { if clicontext.Bool(flagDebug) { return errors.Wrap(err, "invalid config: "+cfgString) } else { return errors.Wrap(err, "invalid config") } } // set up signals so we handle the first shutdown signal gracefully stopCh := signals.SetupSignalHandler() s, err := controller.New(c, stopCh) if err != nil { return errors.Wrap(err, "failed to create server") } return s.Run(c.Controller.ThreadCount, stopCh) } ================================================ FILE: ingress-operator/pkg/client/clientset/versioned/clientset.go ================================================ /* Copyright 2023 TensorChord Inc. Licensed under the MIT license. See LICENSE file in the project root for full license information. */ // Code generated by client-gen. DO NOT EDIT. package versioned import ( "fmt" tensorchordv1 "github.com/tensorchord/openmodelz/ingress-operator/pkg/client/clientset/versioned/typed/modelzetes/v1" discovery "k8s.io/client-go/discovery" rest "k8s.io/client-go/rest" flowcontrol "k8s.io/client-go/util/flowcontrol" ) type Interface interface { Discovery() discovery.DiscoveryInterface TensorchordV1() tensorchordv1.TensorchordV1Interface } // Clientset contains the clients for groups. Each group has exactly one // version included in a Clientset. type Clientset struct { *discovery.DiscoveryClient tensorchordV1 *tensorchordv1.TensorchordV1Client } // TensorchordV1 retrieves the TensorchordV1Client func (c *Clientset) TensorchordV1() tensorchordv1.TensorchordV1Interface { return c.tensorchordV1 } // Discovery retrieves the DiscoveryClient func (c *Clientset) Discovery() discovery.DiscoveryInterface { if c == nil { return nil } return c.DiscoveryClient } // NewForConfig creates a new Clientset for the given config. // If config's RateLimiter is not set and QPS and Burst are acceptable, // NewForConfig will generate a rate-limiter in configShallowCopy. func NewForConfig(c *rest.Config) (*Clientset, error) { configShallowCopy := *c if configShallowCopy.RateLimiter == nil && configShallowCopy.QPS > 0 { if configShallowCopy.Burst <= 0 { return nil, fmt.Errorf("burst is required to be greater than 0 when RateLimiter is not set and QPS is set to greater than 0") } configShallowCopy.RateLimiter = flowcontrol.NewTokenBucketRateLimiter(configShallowCopy.QPS, configShallowCopy.Burst) } var cs Clientset var err error cs.tensorchordV1, err = tensorchordv1.NewForConfig(&configShallowCopy) if err != nil { return nil, err } cs.DiscoveryClient, err = discovery.NewDiscoveryClientForConfig(&configShallowCopy) if err != nil { return nil, err } return &cs, nil } // NewForConfigOrDie creates a new Clientset for the given config and // panics if there is an error in the config. func NewForConfigOrDie(c *rest.Config) *Clientset { var cs Clientset cs.tensorchordV1 = tensorchordv1.NewForConfigOrDie(c) cs.DiscoveryClient = discovery.NewDiscoveryClientForConfigOrDie(c) return &cs } // New creates a new Clientset for the given RESTClient. func New(c rest.Interface) *Clientset { var cs Clientset cs.tensorchordV1 = tensorchordv1.New(c) cs.DiscoveryClient = discovery.NewDiscoveryClient(c) return &cs } ================================================ FILE: ingress-operator/pkg/client/clientset/versioned/doc.go ================================================ /* Copyright 2023 TensorChord Inc. Licensed under the MIT license. See LICENSE file in the project root for full license information. */ // Code generated by client-gen. DO NOT EDIT. // This package has the automatically generated clientset. package versioned ================================================ FILE: ingress-operator/pkg/client/clientset/versioned/fake/clientset_generated.go ================================================ /* Copyright 2023 TensorChord Inc. Licensed under the MIT license. See LICENSE file in the project root for full license information. */ // Code generated by client-gen. DO NOT EDIT. package fake import ( clientset "github.com/tensorchord/openmodelz/ingress-operator/pkg/client/clientset/versioned" tensorchordv1 "github.com/tensorchord/openmodelz/ingress-operator/pkg/client/clientset/versioned/typed/modelzetes/v1" faketensorchordv1 "github.com/tensorchord/openmodelz/ingress-operator/pkg/client/clientset/versioned/typed/modelzetes/v1/fake" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/discovery" fakediscovery "k8s.io/client-go/discovery/fake" "k8s.io/client-go/testing" ) // NewSimpleClientset returns a clientset that will respond with the provided objects. // It's backed by a very simple object tracker that processes creates, updates and deletions as-is, // without applying any validations and/or defaults. It shouldn't be considered a replacement // for a real clientset and is mostly useful in simple unit tests. func NewSimpleClientset(objects ...runtime.Object) *Clientset { o := testing.NewObjectTracker(scheme, codecs.UniversalDecoder()) for _, obj := range objects { if err := o.Add(obj); err != nil { panic(err) } } cs := &Clientset{tracker: o} cs.discovery = &fakediscovery.FakeDiscovery{Fake: &cs.Fake} cs.AddReactor("*", "*", testing.ObjectReaction(o)) cs.AddWatchReactor("*", func(action testing.Action) (handled bool, ret watch.Interface, err error) { gvr := action.GetResource() ns := action.GetNamespace() watch, err := o.Watch(gvr, ns) if err != nil { return false, nil, err } return true, watch, nil }) return cs } // Clientset implements clientset.Interface. Meant to be embedded into a // struct to get a default implementation. This makes faking out just the method // you want to test easier. type Clientset struct { testing.Fake discovery *fakediscovery.FakeDiscovery tracker testing.ObjectTracker } func (c *Clientset) Discovery() discovery.DiscoveryInterface { return c.discovery } func (c *Clientset) Tracker() testing.ObjectTracker { return c.tracker } var ( _ clientset.Interface = &Clientset{} _ testing.FakeClient = &Clientset{} ) // TensorchordV1 retrieves the TensorchordV1Client func (c *Clientset) TensorchordV1() tensorchordv1.TensorchordV1Interface { return &faketensorchordv1.FakeTensorchordV1{Fake: &c.Fake} } ================================================ FILE: ingress-operator/pkg/client/clientset/versioned/fake/doc.go ================================================ /* Copyright 2023 TensorChord Inc. Licensed under the MIT license. See LICENSE file in the project root for full license information. */ // Code generated by client-gen. DO NOT EDIT. // This package has the automatically generated fake clientset. package fake ================================================ FILE: ingress-operator/pkg/client/clientset/versioned/fake/register.go ================================================ /* Copyright 2023 TensorChord Inc. Licensed under the MIT license. See LICENSE file in the project root for full license information. */ // Code generated by client-gen. DO NOT EDIT. package fake import ( tensorchordv1 "github.com/tensorchord/openmodelz/ingress-operator/pkg/apis/modelzetes/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" schema "k8s.io/apimachinery/pkg/runtime/schema" serializer "k8s.io/apimachinery/pkg/runtime/serializer" utilruntime "k8s.io/apimachinery/pkg/util/runtime" ) var scheme = runtime.NewScheme() var codecs = serializer.NewCodecFactory(scheme) var localSchemeBuilder = runtime.SchemeBuilder{ tensorchordv1.AddToScheme, } // AddToScheme adds all types of this clientset into the given scheme. This allows composition // of clientsets, like in: // // import ( // "k8s.io/client-go/kubernetes" // clientsetscheme "k8s.io/client-go/kubernetes/scheme" // aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme" // ) // // kclientset, _ := kubernetes.NewForConfig(c) // _ = aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme) // // After this, RawExtensions in Kubernetes types will serialize kube-aggregator types // correctly. var AddToScheme = localSchemeBuilder.AddToScheme func init() { v1.AddToGroupVersion(scheme, schema.GroupVersion{Version: "v1"}) utilruntime.Must(AddToScheme(scheme)) } ================================================ FILE: ingress-operator/pkg/client/clientset/versioned/scheme/doc.go ================================================ /* Copyright 2023 TensorChord Inc. Licensed under the MIT license. See LICENSE file in the project root for full license information. */ // Code generated by client-gen. DO NOT EDIT. // This package contains the scheme of the automatically generated clientset. package scheme ================================================ FILE: ingress-operator/pkg/client/clientset/versioned/scheme/register.go ================================================ /* Copyright 2023 TensorChord Inc. Licensed under the MIT license. See LICENSE file in the project root for full license information. */ // Code generated by client-gen. DO NOT EDIT. package scheme import ( tensorchordv1 "github.com/tensorchord/openmodelz/ingress-operator/pkg/apis/modelzetes/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" schema "k8s.io/apimachinery/pkg/runtime/schema" serializer "k8s.io/apimachinery/pkg/runtime/serializer" utilruntime "k8s.io/apimachinery/pkg/util/runtime" ) var Scheme = runtime.NewScheme() var Codecs = serializer.NewCodecFactory(Scheme) var ParameterCodec = runtime.NewParameterCodec(Scheme) var localSchemeBuilder = runtime.SchemeBuilder{ tensorchordv1.AddToScheme, } // AddToScheme adds all types of this clientset into the given scheme. This allows composition // of clientsets, like in: // // import ( // "k8s.io/client-go/kubernetes" // clientsetscheme "k8s.io/client-go/kubernetes/scheme" // aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme" // ) // // kclientset, _ := kubernetes.NewForConfig(c) // _ = aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme) // // After this, RawExtensions in Kubernetes types will serialize kube-aggregator types // correctly. var AddToScheme = localSchemeBuilder.AddToScheme func init() { v1.AddToGroupVersion(Scheme, schema.GroupVersion{Version: "v1"}) utilruntime.Must(AddToScheme(Scheme)) } ================================================ FILE: ingress-operator/pkg/client/clientset/versioned/typed/modelzetes/v1/doc.go ================================================ /* Copyright 2023 TensorChord Inc. Licensed under the MIT license. See LICENSE file in the project root for full license information. */ // Code generated by client-gen. DO NOT EDIT. // This package has the automatically generated typed clients. package v1 ================================================ FILE: ingress-operator/pkg/client/clientset/versioned/typed/modelzetes/v1/fake/doc.go ================================================ /* Copyright 2023 TensorChord Inc. Licensed under the MIT license. See LICENSE file in the project root for full license information. */ // Code generated by client-gen. DO NOT EDIT. // Package fake has the automatically generated clients. package fake ================================================ FILE: ingress-operator/pkg/client/clientset/versioned/typed/modelzetes/v1/fake/fake_inferenceingress.go ================================================ /* Copyright 2023 TensorChord Inc. Licensed under the MIT license. See LICENSE file in the project root for full license information. */ // Code generated by client-gen. DO NOT EDIT. package fake import ( "context" modelzetesv1 "github.com/tensorchord/openmodelz/ingress-operator/pkg/apis/modelzetes/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" labels "k8s.io/apimachinery/pkg/labels" schema "k8s.io/apimachinery/pkg/runtime/schema" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" testing "k8s.io/client-go/testing" ) // FakeInferenceIngresses implements InferenceIngressInterface type FakeInferenceIngresses struct { Fake *FakeTensorchordV1 ns string } var inferenceingressesResource = schema.GroupVersionResource{Group: "tensorchord.ai", Version: "v1", Resource: "inferenceingresses"} var inferenceingressesKind = schema.GroupVersionKind{Group: "tensorchord.ai", Version: "v1", Kind: "InferenceIngress"} // Get takes name of the inferenceIngress, and returns the corresponding inferenceIngress object, and an error if there is any. func (c *FakeInferenceIngresses) Get(ctx context.Context, name string, options v1.GetOptions) (result *modelzetesv1.InferenceIngress, err error) { obj, err := c.Fake. Invokes(testing.NewGetAction(inferenceingressesResource, c.ns, name), &modelzetesv1.InferenceIngress{}) if obj == nil { return nil, err } return obj.(*modelzetesv1.InferenceIngress), err } // List takes label and field selectors, and returns the list of InferenceIngresses that match those selectors. func (c *FakeInferenceIngresses) List(ctx context.Context, opts v1.ListOptions) (result *modelzetesv1.InferenceIngressList, err error) { obj, err := c.Fake. Invokes(testing.NewListAction(inferenceingressesResource, inferenceingressesKind, c.ns, opts), &modelzetesv1.InferenceIngressList{}) if obj == nil { return nil, err } label, _, _ := testing.ExtractFromListOptions(opts) if label == nil { label = labels.Everything() } list := &modelzetesv1.InferenceIngressList{ListMeta: obj.(*modelzetesv1.InferenceIngressList).ListMeta} for _, item := range obj.(*modelzetesv1.InferenceIngressList).Items { if label.Matches(labels.Set(item.Labels)) { list.Items = append(list.Items, item) } } return list, err } // Watch returns a watch.Interface that watches the requested inferenceIngresses. func (c *FakeInferenceIngresses) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { return c.Fake. InvokesWatch(testing.NewWatchAction(inferenceingressesResource, c.ns, opts)) } // Create takes the representation of a inferenceIngress and creates it. Returns the server's representation of the inferenceIngress, and an error, if there is any. func (c *FakeInferenceIngresses) Create(ctx context.Context, inferenceIngress *modelzetesv1.InferenceIngress, opts v1.CreateOptions) (result *modelzetesv1.InferenceIngress, err error) { obj, err := c.Fake. Invokes(testing.NewCreateAction(inferenceingressesResource, c.ns, inferenceIngress), &modelzetesv1.InferenceIngress{}) if obj == nil { return nil, err } return obj.(*modelzetesv1.InferenceIngress), err } // Update takes the representation of a inferenceIngress and updates it. Returns the server's representation of the inferenceIngress, and an error, if there is any. func (c *FakeInferenceIngresses) Update(ctx context.Context, inferenceIngress *modelzetesv1.InferenceIngress, opts v1.UpdateOptions) (result *modelzetesv1.InferenceIngress, err error) { obj, err := c.Fake. Invokes(testing.NewUpdateAction(inferenceingressesResource, c.ns, inferenceIngress), &modelzetesv1.InferenceIngress{}) if obj == nil { return nil, err } return obj.(*modelzetesv1.InferenceIngress), err } // Delete takes name of the inferenceIngress and deletes it. Returns an error if one occurs. func (c *FakeInferenceIngresses) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { _, err := c.Fake. Invokes(testing.NewDeleteAction(inferenceingressesResource, c.ns, name), &modelzetesv1.InferenceIngress{}) return err } // DeleteCollection deletes a collection of objects. func (c *FakeInferenceIngresses) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { action := testing.NewDeleteCollectionAction(inferenceingressesResource, c.ns, listOpts) _, err := c.Fake.Invokes(action, &modelzetesv1.InferenceIngressList{}) return err } // Patch applies the patch and returns the patched inferenceIngress. func (c *FakeInferenceIngresses) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *modelzetesv1.InferenceIngress, err error) { obj, err := c.Fake. Invokes(testing.NewPatchSubresourceAction(inferenceingressesResource, c.ns, name, pt, data, subresources...), &modelzetesv1.InferenceIngress{}) if obj == nil { return nil, err } return obj.(*modelzetesv1.InferenceIngress), err } ================================================ FILE: ingress-operator/pkg/client/clientset/versioned/typed/modelzetes/v1/fake/fake_modelzetes_client.go ================================================ /* Copyright 2023 TensorChord Inc. Licensed under the MIT license. See LICENSE file in the project root for full license information. */ // Code generated by client-gen. DO NOT EDIT. package fake import ( v1 "github.com/tensorchord/openmodelz/ingress-operator/pkg/client/clientset/versioned/typed/modelzetes/v1" rest "k8s.io/client-go/rest" testing "k8s.io/client-go/testing" ) type FakeTensorchordV1 struct { *testing.Fake } func (c *FakeTensorchordV1) InferenceIngresses(namespace string) v1.InferenceIngressInterface { return &FakeInferenceIngresses{c, namespace} } // RESTClient returns a RESTClient that is used to communicate // with API server by this client implementation. func (c *FakeTensorchordV1) RESTClient() rest.Interface { var ret *rest.RESTClient return ret } ================================================ FILE: ingress-operator/pkg/client/clientset/versioned/typed/modelzetes/v1/generated_expansion.go ================================================ /* Copyright 2023 TensorChord Inc. Licensed under the MIT license. See LICENSE file in the project root for full license information. */ // Code generated by client-gen. DO NOT EDIT. package v1 type InferenceIngressExpansion interface{} ================================================ FILE: ingress-operator/pkg/client/clientset/versioned/typed/modelzetes/v1/inferenceingress.go ================================================ /* Copyright 2023 TensorChord Inc. Licensed under the MIT license. See LICENSE file in the project root for full license information. */ // Code generated by client-gen. DO NOT EDIT. package v1 import ( "context" "time" v1 "github.com/tensorchord/openmodelz/ingress-operator/pkg/apis/modelzetes/v1" scheme "github.com/tensorchord/openmodelz/ingress-operator/pkg/client/clientset/versioned/scheme" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" rest "k8s.io/client-go/rest" ) // InferenceIngressesGetter has a method to return a InferenceIngressInterface. // A group's client should implement this interface. type InferenceIngressesGetter interface { InferenceIngresses(namespace string) InferenceIngressInterface } // InferenceIngressInterface has methods to work with InferenceIngress resources. type InferenceIngressInterface interface { Create(ctx context.Context, inferenceIngress *v1.InferenceIngress, opts metav1.CreateOptions) (*v1.InferenceIngress, error) Update(ctx context.Context, inferenceIngress *v1.InferenceIngress, opts metav1.UpdateOptions) (*v1.InferenceIngress, error) Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error DeleteCollection(ctx context.Context, opts metav1.DeleteOptions, listOpts metav1.ListOptions) error Get(ctx context.Context, name string, opts metav1.GetOptions) (*v1.InferenceIngress, error) List(ctx context.Context, opts metav1.ListOptions) (*v1.InferenceIngressList, error) Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (result *v1.InferenceIngress, err error) InferenceIngressExpansion } // inferenceIngresses implements InferenceIngressInterface type inferenceIngresses struct { client rest.Interface ns string } // newInferenceIngresses returns a InferenceIngresses func newInferenceIngresses(c *TensorchordV1Client, namespace string) *inferenceIngresses { return &inferenceIngresses{ client: c.RESTClient(), ns: namespace, } } // Get takes name of the inferenceIngress, and returns the corresponding inferenceIngress object, and an error if there is any. func (c *inferenceIngresses) Get(ctx context.Context, name string, options metav1.GetOptions) (result *v1.InferenceIngress, err error) { result = &v1.InferenceIngress{} err = c.client.Get(). Namespace(c.ns). Resource("inferenceingresses"). Name(name). VersionedParams(&options, scheme.ParameterCodec). Do(ctx). Into(result) return } // List takes label and field selectors, and returns the list of InferenceIngresses that match those selectors. func (c *inferenceIngresses) List(ctx context.Context, opts metav1.ListOptions) (result *v1.InferenceIngressList, err error) { var timeout time.Duration if opts.TimeoutSeconds != nil { timeout = time.Duration(*opts.TimeoutSeconds) * time.Second } result = &v1.InferenceIngressList{} err = c.client.Get(). Namespace(c.ns). Resource("inferenceingresses"). VersionedParams(&opts, scheme.ParameterCodec). Timeout(timeout). Do(ctx). Into(result) return } // Watch returns a watch.Interface that watches the requested inferenceIngresses. func (c *inferenceIngresses) Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) { var timeout time.Duration if opts.TimeoutSeconds != nil { timeout = time.Duration(*opts.TimeoutSeconds) * time.Second } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("inferenceingresses"). VersionedParams(&opts, scheme.ParameterCodec). Timeout(timeout). Watch(ctx) } // Create takes the representation of a inferenceIngress and creates it. Returns the server's representation of the inferenceIngress, and an error, if there is any. func (c *inferenceIngresses) Create(ctx context.Context, inferenceIngress *v1.InferenceIngress, opts metav1.CreateOptions) (result *v1.InferenceIngress, err error) { result = &v1.InferenceIngress{} err = c.client.Post(). Namespace(c.ns). Resource("inferenceingresses"). VersionedParams(&opts, scheme.ParameterCodec). Body(inferenceIngress). Do(ctx). Into(result) return } // Update takes the representation of a inferenceIngress and updates it. Returns the server's representation of the inferenceIngress, and an error, if there is any. func (c *inferenceIngresses) Update(ctx context.Context, inferenceIngress *v1.InferenceIngress, opts metav1.UpdateOptions) (result *v1.InferenceIngress, err error) { result = &v1.InferenceIngress{} err = c.client.Put(). Namespace(c.ns). Resource("inferenceingresses"). Name(inferenceIngress.Name). VersionedParams(&opts, scheme.ParameterCodec). Body(inferenceIngress). Do(ctx). Into(result) return } // Delete takes name of the inferenceIngress and deletes it. Returns an error if one occurs. func (c *inferenceIngresses) Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error { return c.client.Delete(). Namespace(c.ns). Resource("inferenceingresses"). Name(name). Body(&opts). Do(ctx). Error() } // DeleteCollection deletes a collection of objects. func (c *inferenceIngresses) DeleteCollection(ctx context.Context, opts metav1.DeleteOptions, listOpts metav1.ListOptions) error { var timeout time.Duration if listOpts.TimeoutSeconds != nil { timeout = time.Duration(*listOpts.TimeoutSeconds) * time.Second } return c.client.Delete(). Namespace(c.ns). Resource("inferenceingresses"). VersionedParams(&listOpts, scheme.ParameterCodec). Timeout(timeout). Body(&opts). Do(ctx). Error() } // Patch applies the patch and returns the patched inferenceIngress. func (c *inferenceIngresses) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (result *v1.InferenceIngress, err error) { result = &v1.InferenceIngress{} err = c.client.Patch(pt). Namespace(c.ns). Resource("inferenceingresses"). Name(name). SubResource(subresources...). VersionedParams(&opts, scheme.ParameterCodec). Body(data). Do(ctx). Into(result) return } ================================================ FILE: ingress-operator/pkg/client/clientset/versioned/typed/modelzetes/v1/modelzetes_client.go ================================================ /* Copyright 2023 TensorChord Inc. Licensed under the MIT license. See LICENSE file in the project root for full license information. */ // Code generated by client-gen. DO NOT EDIT. package v1 import ( v1 "github.com/tensorchord/openmodelz/ingress-operator/pkg/apis/modelzetes/v1" "github.com/tensorchord/openmodelz/ingress-operator/pkg/client/clientset/versioned/scheme" rest "k8s.io/client-go/rest" ) type TensorchordV1Interface interface { RESTClient() rest.Interface InferenceIngressesGetter } // TensorchordV1Client is used to interact with features provided by the tensorchord.ai group. type TensorchordV1Client struct { restClient rest.Interface } func (c *TensorchordV1Client) InferenceIngresses(namespace string) InferenceIngressInterface { return newInferenceIngresses(c, namespace) } // NewForConfig creates a new TensorchordV1Client for the given config. func NewForConfig(c *rest.Config) (*TensorchordV1Client, error) { config := *c if err := setConfigDefaults(&config); err != nil { return nil, err } client, err := rest.RESTClientFor(&config) if err != nil { return nil, err } return &TensorchordV1Client{client}, nil } // NewForConfigOrDie creates a new TensorchordV1Client for the given config and // panics if there is an error in the config. func NewForConfigOrDie(c *rest.Config) *TensorchordV1Client { client, err := NewForConfig(c) if err != nil { panic(err) } return client } // New creates a new TensorchordV1Client for the given RESTClient. func New(c rest.Interface) *TensorchordV1Client { return &TensorchordV1Client{c} } func setConfigDefaults(config *rest.Config) error { gv := v1.SchemeGroupVersion config.GroupVersion = &gv config.APIPath = "/apis" config.NegotiatedSerializer = scheme.Codecs.WithoutConversion() if config.UserAgent == "" { config.UserAgent = rest.DefaultKubernetesUserAgent() } return nil } // RESTClient returns a RESTClient that is used to communicate // with API server by this client implementation. func (c *TensorchordV1Client) RESTClient() rest.Interface { if c == nil { return nil } return c.restClient } ================================================ FILE: ingress-operator/pkg/client/informers/externalversions/factory.go ================================================ /* Copyright 2023 TensorChord Inc. Licensed under the MIT license. See LICENSE file in the project root for full license information. */ // Code generated by informer-gen. DO NOT EDIT. package externalversions import ( reflect "reflect" sync "sync" time "time" versioned "github.com/tensorchord/openmodelz/ingress-operator/pkg/client/clientset/versioned" internalinterfaces "github.com/tensorchord/openmodelz/ingress-operator/pkg/client/informers/externalversions/internalinterfaces" modelzetes "github.com/tensorchord/openmodelz/ingress-operator/pkg/client/informers/externalversions/modelzetes" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" schema "k8s.io/apimachinery/pkg/runtime/schema" cache "k8s.io/client-go/tools/cache" ) // SharedInformerOption defines the functional option type for SharedInformerFactory. type SharedInformerOption func(*sharedInformerFactory) *sharedInformerFactory type sharedInformerFactory struct { client versioned.Interface namespace string tweakListOptions internalinterfaces.TweakListOptionsFunc lock sync.Mutex defaultResync time.Duration customResync map[reflect.Type]time.Duration informers map[reflect.Type]cache.SharedIndexInformer // startedInformers is used for tracking which informers have been started. // This allows Start() to be called multiple times safely. startedInformers map[reflect.Type]bool } // WithCustomResyncConfig sets a custom resync period for the specified informer types. func WithCustomResyncConfig(resyncConfig map[v1.Object]time.Duration) SharedInformerOption { return func(factory *sharedInformerFactory) *sharedInformerFactory { for k, v := range resyncConfig { factory.customResync[reflect.TypeOf(k)] = v } return factory } } // WithTweakListOptions sets a custom filter on all listers of the configured SharedInformerFactory. func WithTweakListOptions(tweakListOptions internalinterfaces.TweakListOptionsFunc) SharedInformerOption { return func(factory *sharedInformerFactory) *sharedInformerFactory { factory.tweakListOptions = tweakListOptions return factory } } // WithNamespace limits the SharedInformerFactory to the specified namespace. func WithNamespace(namespace string) SharedInformerOption { return func(factory *sharedInformerFactory) *sharedInformerFactory { factory.namespace = namespace return factory } } // NewSharedInformerFactory constructs a new instance of sharedInformerFactory for all namespaces. func NewSharedInformerFactory(client versioned.Interface, defaultResync time.Duration) SharedInformerFactory { return NewSharedInformerFactoryWithOptions(client, defaultResync) } // NewFilteredSharedInformerFactory constructs a new instance of sharedInformerFactory. // Listers obtained via this SharedInformerFactory will be subject to the same filters // as specified here. // Deprecated: Please use NewSharedInformerFactoryWithOptions instead func NewFilteredSharedInformerFactory(client versioned.Interface, defaultResync time.Duration, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) SharedInformerFactory { return NewSharedInformerFactoryWithOptions(client, defaultResync, WithNamespace(namespace), WithTweakListOptions(tweakListOptions)) } // NewSharedInformerFactoryWithOptions constructs a new instance of a SharedInformerFactory with additional options. func NewSharedInformerFactoryWithOptions(client versioned.Interface, defaultResync time.Duration, options ...SharedInformerOption) SharedInformerFactory { factory := &sharedInformerFactory{ client: client, namespace: v1.NamespaceAll, defaultResync: defaultResync, informers: make(map[reflect.Type]cache.SharedIndexInformer), startedInformers: make(map[reflect.Type]bool), customResync: make(map[reflect.Type]time.Duration), } // Apply all options for _, opt := range options { factory = opt(factory) } return factory } // Start initializes all requested informers. func (f *sharedInformerFactory) Start(stopCh <-chan struct{}) { f.lock.Lock() defer f.lock.Unlock() for informerType, informer := range f.informers { if !f.startedInformers[informerType] { go informer.Run(stopCh) f.startedInformers[informerType] = true } } } // WaitForCacheSync waits for all started informers' cache were synced. func (f *sharedInformerFactory) WaitForCacheSync(stopCh <-chan struct{}) map[reflect.Type]bool { informers := func() map[reflect.Type]cache.SharedIndexInformer { f.lock.Lock() defer f.lock.Unlock() informers := map[reflect.Type]cache.SharedIndexInformer{} for informerType, informer := range f.informers { if f.startedInformers[informerType] { informers[informerType] = informer } } return informers }() res := map[reflect.Type]bool{} for informType, informer := range informers { res[informType] = cache.WaitForCacheSync(stopCh, informer.HasSynced) } return res } // InternalInformerFor returns the SharedIndexInformer for obj using an internal // client. func (f *sharedInformerFactory) InformerFor(obj runtime.Object, newFunc internalinterfaces.NewInformerFunc) cache.SharedIndexInformer { f.lock.Lock() defer f.lock.Unlock() informerType := reflect.TypeOf(obj) informer, exists := f.informers[informerType] if exists { return informer } resyncPeriod, exists := f.customResync[informerType] if !exists { resyncPeriod = f.defaultResync } informer = newFunc(f.client, resyncPeriod) f.informers[informerType] = informer return informer } // SharedInformerFactory provides shared informers for resources in all known // API group versions. type SharedInformerFactory interface { internalinterfaces.SharedInformerFactory ForResource(resource schema.GroupVersionResource) (GenericInformer, error) WaitForCacheSync(stopCh <-chan struct{}) map[reflect.Type]bool Tensorchord() modelzetes.Interface } func (f *sharedInformerFactory) Tensorchord() modelzetes.Interface { return modelzetes.New(f, f.namespace, f.tweakListOptions) } ================================================ FILE: ingress-operator/pkg/client/informers/externalversions/generic.go ================================================ /* Copyright 2023 TensorChord Inc. Licensed under the MIT license. See LICENSE file in the project root for full license information. */ // Code generated by informer-gen. DO NOT EDIT. package externalversions import ( "fmt" v1 "github.com/tensorchord/openmodelz/ingress-operator/pkg/apis/modelzetes/v1" schema "k8s.io/apimachinery/pkg/runtime/schema" cache "k8s.io/client-go/tools/cache" ) // GenericInformer is type of SharedIndexInformer which will locate and delegate to other // sharedInformers based on type type GenericInformer interface { Informer() cache.SharedIndexInformer Lister() cache.GenericLister } type genericInformer struct { informer cache.SharedIndexInformer resource schema.GroupResource } // Informer returns the SharedIndexInformer. func (f *genericInformer) Informer() cache.SharedIndexInformer { return f.informer } // Lister returns the GenericLister. func (f *genericInformer) Lister() cache.GenericLister { return cache.NewGenericLister(f.Informer().GetIndexer(), f.resource) } // ForResource gives generic access to a shared informer of the matching type // TODO extend this to unknown resources with a client pool func (f *sharedInformerFactory) ForResource(resource schema.GroupVersionResource) (GenericInformer, error) { switch resource { // Group=tensorchord.ai, Version=v1 case v1.SchemeGroupVersion.WithResource("inferenceingresses"): return &genericInformer{resource: resource.GroupResource(), informer: f.Tensorchord().V1().InferenceIngresses().Informer()}, nil } return nil, fmt.Errorf("no informer found for %v", resource) } ================================================ FILE: ingress-operator/pkg/client/informers/externalversions/internalinterfaces/factory_interfaces.go ================================================ /* Copyright 2023 TensorChord Inc. Licensed under the MIT license. See LICENSE file in the project root for full license information. */ // Code generated by informer-gen. DO NOT EDIT. package internalinterfaces import ( time "time" versioned "github.com/tensorchord/openmodelz/ingress-operator/pkg/client/clientset/versioned" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" cache "k8s.io/client-go/tools/cache" ) // NewInformerFunc takes versioned.Interface and time.Duration to return a SharedIndexInformer. type NewInformerFunc func(versioned.Interface, time.Duration) cache.SharedIndexInformer // SharedInformerFactory a small interface to allow for adding an informer without an import cycle type SharedInformerFactory interface { Start(stopCh <-chan struct{}) InformerFor(obj runtime.Object, newFunc NewInformerFunc) cache.SharedIndexInformer } // TweakListOptionsFunc is a function that transforms a v1.ListOptions. type TweakListOptionsFunc func(*v1.ListOptions) ================================================ FILE: ingress-operator/pkg/client/informers/externalversions/modelzetes/interface.go ================================================ /* Copyright 2023 TensorChord Inc. Licensed under the MIT license. See LICENSE file in the project root for full license information. */ // Code generated by informer-gen. DO NOT EDIT. package modelzetes import ( internalinterfaces "github.com/tensorchord/openmodelz/ingress-operator/pkg/client/informers/externalversions/internalinterfaces" v1 "github.com/tensorchord/openmodelz/ingress-operator/pkg/client/informers/externalversions/modelzetes/v1" ) // Interface provides access to each of this group's versions. type Interface interface { // V1 provides access to shared informers for resources in V1. V1() v1.Interface } type group struct { factory internalinterfaces.SharedInformerFactory namespace string tweakListOptions internalinterfaces.TweakListOptionsFunc } // New returns a new Interface. func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) Interface { return &group{factory: f, namespace: namespace, tweakListOptions: tweakListOptions} } // V1 returns a new v1.Interface. func (g *group) V1() v1.Interface { return v1.New(g.factory, g.namespace, g.tweakListOptions) } ================================================ FILE: ingress-operator/pkg/client/informers/externalversions/modelzetes/v1/inferenceingress.go ================================================ /* Copyright 2023 TensorChord Inc. Licensed under the MIT license. See LICENSE file in the project root for full license information. */ // Code generated by informer-gen. DO NOT EDIT. package v1 import ( "context" time "time" modelzetesv1 "github.com/tensorchord/openmodelz/ingress-operator/pkg/apis/modelzetes/v1" versioned "github.com/tensorchord/openmodelz/ingress-operator/pkg/client/clientset/versioned" internalinterfaces "github.com/tensorchord/openmodelz/ingress-operator/pkg/client/informers/externalversions/internalinterfaces" v1 "github.com/tensorchord/openmodelz/ingress-operator/pkg/client/listers/modelzetes/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" watch "k8s.io/apimachinery/pkg/watch" cache "k8s.io/client-go/tools/cache" ) // InferenceIngressInformer provides access to a shared informer and lister for // InferenceIngresses. type InferenceIngressInformer interface { Informer() cache.SharedIndexInformer Lister() v1.InferenceIngressLister } type inferenceIngressInformer struct { factory internalinterfaces.SharedInformerFactory tweakListOptions internalinterfaces.TweakListOptionsFunc namespace string } // NewInferenceIngressInformer constructs a new informer for InferenceIngress type. // Always prefer using an informer factory to get a shared informer instead of getting an independent // one. This reduces memory footprint and number of connections to the server. func NewInferenceIngressInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { return NewFilteredInferenceIngressInformer(client, namespace, resyncPeriod, indexers, nil) } // NewFilteredInferenceIngressInformer constructs a new informer for InferenceIngress type. // Always prefer using an informer factory to get a shared informer instead of getting an independent // one. This reduces memory footprint and number of connections to the server. func NewFilteredInferenceIngressInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { return cache.NewSharedIndexInformer( &cache.ListWatch{ ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { if tweakListOptions != nil { tweakListOptions(&options) } return client.TensorchordV1().InferenceIngresses(namespace).List(context.TODO(), options) }, WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { if tweakListOptions != nil { tweakListOptions(&options) } return client.TensorchordV1().InferenceIngresses(namespace).Watch(context.TODO(), options) }, }, &modelzetesv1.InferenceIngress{}, resyncPeriod, indexers, ) } func (f *inferenceIngressInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { return NewFilteredInferenceIngressInformer(client, f.namespace, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions) } func (f *inferenceIngressInformer) Informer() cache.SharedIndexInformer { return f.factory.InformerFor(&modelzetesv1.InferenceIngress{}, f.defaultInformer) } func (f *inferenceIngressInformer) Lister() v1.InferenceIngressLister { return v1.NewInferenceIngressLister(f.Informer().GetIndexer()) } ================================================ FILE: ingress-operator/pkg/client/informers/externalversions/modelzetes/v1/interface.go ================================================ /* Copyright 2023 TensorChord Inc. Licensed under the MIT license. See LICENSE file in the project root for full license information. */ // Code generated by informer-gen. DO NOT EDIT. package v1 import ( internalinterfaces "github.com/tensorchord/openmodelz/ingress-operator/pkg/client/informers/externalversions/internalinterfaces" ) // Interface provides access to all the informers in this group version. type Interface interface { // InferenceIngresses returns a InferenceIngressInformer. InferenceIngresses() InferenceIngressInformer } type version struct { factory internalinterfaces.SharedInformerFactory namespace string tweakListOptions internalinterfaces.TweakListOptionsFunc } // New returns a new Interface. func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) Interface { return &version{factory: f, namespace: namespace, tweakListOptions: tweakListOptions} } // InferenceIngresses returns a InferenceIngressInformer. func (v *version) InferenceIngresses() InferenceIngressInformer { return &inferenceIngressInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} } ================================================ FILE: ingress-operator/pkg/client/listers/modelzetes/v1/expansion_generated.go ================================================ /* Copyright 2023 TensorChord Inc. Licensed under the MIT license. See LICENSE file in the project root for full license information. */ // Code generated by lister-gen. DO NOT EDIT. package v1 // InferenceIngressListerExpansion allows custom methods to be added to // InferenceIngressLister. type InferenceIngressListerExpansion interface{} // InferenceIngressNamespaceListerExpansion allows custom methods to be added to // InferenceIngressNamespaceLister. type InferenceIngressNamespaceListerExpansion interface{} ================================================ FILE: ingress-operator/pkg/client/listers/modelzetes/v1/inferenceingress.go ================================================ /* Copyright 2023 TensorChord Inc. Licensed under the MIT license. See LICENSE file in the project root for full license information. */ // Code generated by lister-gen. DO NOT EDIT. package v1 import ( v1 "github.com/tensorchord/openmodelz/ingress-operator/pkg/apis/modelzetes/v1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/tools/cache" ) // InferenceIngressLister helps list InferenceIngresses. // All objects returned here must be treated as read-only. type InferenceIngressLister interface { // List lists all InferenceIngresses in the indexer. // Objects returned here must be treated as read-only. List(selector labels.Selector) (ret []*v1.InferenceIngress, err error) // InferenceIngresses returns an object that can list and get InferenceIngresses. InferenceIngresses(namespace string) InferenceIngressNamespaceLister InferenceIngressListerExpansion } // inferenceIngressLister implements the InferenceIngressLister interface. type inferenceIngressLister struct { indexer cache.Indexer } // NewInferenceIngressLister returns a new InferenceIngressLister. func NewInferenceIngressLister(indexer cache.Indexer) InferenceIngressLister { return &inferenceIngressLister{indexer: indexer} } // List lists all InferenceIngresses in the indexer. func (s *inferenceIngressLister) List(selector labels.Selector) (ret []*v1.InferenceIngress, err error) { err = cache.ListAll(s.indexer, selector, func(m interface{}) { ret = append(ret, m.(*v1.InferenceIngress)) }) return ret, err } // InferenceIngresses returns an object that can list and get InferenceIngresses. func (s *inferenceIngressLister) InferenceIngresses(namespace string) InferenceIngressNamespaceLister { return inferenceIngressNamespaceLister{indexer: s.indexer, namespace: namespace} } // InferenceIngressNamespaceLister helps list and get InferenceIngresses. // All objects returned here must be treated as read-only. type InferenceIngressNamespaceLister interface { // List lists all InferenceIngresses in the indexer for a given namespace. // Objects returned here must be treated as read-only. List(selector labels.Selector) (ret []*v1.InferenceIngress, err error) // Get retrieves the InferenceIngress from the indexer for a given namespace and name. // Objects returned here must be treated as read-only. Get(name string) (*v1.InferenceIngress, error) InferenceIngressNamespaceListerExpansion } // inferenceIngressNamespaceLister implements the InferenceIngressNamespaceLister // interface. type inferenceIngressNamespaceLister struct { indexer cache.Indexer namespace string } // List lists all InferenceIngresses in the indexer for a given namespace. func (s inferenceIngressNamespaceLister) List(selector labels.Selector) (ret []*v1.InferenceIngress, err error) { err = cache.ListAllByNamespace(s.indexer, s.namespace, selector, func(m interface{}) { ret = append(ret, m.(*v1.InferenceIngress)) }) return ret, err } // Get retrieves the InferenceIngress from the indexer for a given namespace and name. func (s inferenceIngressNamespaceLister) Get(name string) (*v1.InferenceIngress, error) { obj, exists, err := s.indexer.GetByKey(s.namespace + "/" + name) if err != nil { return nil, err } if !exists { return nil, errors.NewNotFound(v1.Resource("inferenceingress"), name) } return obj.(*v1.InferenceIngress), nil } ================================================ FILE: ingress-operator/pkg/config/config.go ================================================ package config import ( "encoding/json" "errors" "time" ) type Config struct { KubeConfig KubeConfig `json:"kube_config,omitempty"` Controller ControllerConfig `json:"controller,omitempty"` } type ControllerConfig struct { ThreadCount int `json:"thread_count,omitempty"` Namespace string `json:"namespace,omitempty"` Host string `json:"host,omitempty"` } type KubeConfig struct { Kubeconfig string `json:"kubeconfig,omitempty"` MasterURL string `json:"master_url,omitempty"` QPS int `json:"qps,omitempty"` Burst int `json:"burst,omitempty"` ResyncPeriod time.Duration `json:"resync_period,omitempty"` } func New() Config { return Config{} } func (c Config) GetString() (string, error) { bytes, err := json.Marshal(c) return string(bytes), err } func (c Config) Validate() error { if c.KubeConfig.QPS == 0 || c.KubeConfig.Burst == 0 || c.KubeConfig.ResyncPeriod == 0 { return errors.New("invalid kubeconfig") } if c.Controller.ThreadCount == 0 || c.Controller.Namespace == "" || c.Controller.Host == "" { return errors.New("invalid controller config") } return nil } ================================================ FILE: ingress-operator/pkg/consts/consts.go ================================================ package consts const ( KeyCert = "cert" EnvironmentPrefix = "MODELZ" ) ================================================ FILE: ingress-operator/pkg/controller/core.go ================================================ package controller import ( "context" "encoding/json" "fmt" "strings" "time" "github.com/google/go-cmp/cmp" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" kubeinformers "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/record" "k8s.io/client-go/util/workqueue" klog "k8s.io/klog" faasv1 "github.com/tensorchord/openmodelz/ingress-operator/pkg/apis/modelzetes/v1" "github.com/tensorchord/openmodelz/ingress-operator/pkg/client/clientset/versioned/scheme" faasscheme "github.com/tensorchord/openmodelz/ingress-operator/pkg/client/clientset/versioned/scheme" v1 "github.com/tensorchord/openmodelz/ingress-operator/pkg/client/informers/externalversions/modelzetes/v1" listers "github.com/tensorchord/openmodelz/ingress-operator/pkg/client/listers/modelzetes/v1" "github.com/tensorchord/openmodelz/modelzetes/pkg/consts" ) const AgentName = "ingress-operator" const FaasIngressKind = "InferenceIngress" const OpenfaasWorkloadPort = 8080 const ( // SuccessSynced is used as part of the Event 'reason' when a Function is synced SuccessSynced = "Synced" // ErrResourceExists is used as part of the Event 'reason' when a Function fails // to sync due to a Deployment of the same name already existing. ErrResourceExists = "ErrResourceExists" // MessageResourceExists is the message used for Events when a resource // fails to sync due to a Deployment already existing MessageResourceExists = "Resource %q already exists and is not managed by controller" // MessageResourceSynced is the message used for an Event fired when a Function // is synced successfully MessageResourceSynced = "FunctionIngress synced successfully" ) // BaseController is the controller contains the common function ingress // implementation that is shared between the various versions of k8s. type BaseController struct { FunctionsLister listers.InferenceIngressLister FunctionsSynced cache.InformerSynced // Workqueue is a rate limited work queue. This is used to queue work to be // processed instead of performing it as soon as a change happens. This // means we can ensure we only process a fixed amount of resources at a // time, and makes it easy to ensure we are never processing the same item // simultaneously in two different workers. Workqueue workqueue.RateLimitingInterface SyncHandler func(ctx context.Context, key string) error } func (c BaseController) Run(threadiness int, stopCh <-chan struct{}) error { ctx, cancel := context.WithCancel(context.Background()) defer runtime.HandleCrash() defer c.Workqueue.ShutDown() defer cancel() // Start the informer factories to begin populating the informer caches // Wait for the caches to be synced before starting workers klog.Info("Waiting for informer caches to sync") if ok := cache.WaitForCacheSync(stopCh, c.FunctionsSynced); !ok { return fmt.Errorf("failed to wait for caches to sync") } klog.Info("Starting workers") // Launch two workers to process Function resources for i := 0; i < threadiness; i++ { go wait.Until(c.runWorker(ctx), time.Second, stopCh) } klog.Info("Started workers") <-stopCh klog.Info("Shutting down workers") return nil } // runWorker is a long-running function that will continually call the // processNextWorkItem function in order to read and process a message on the workqueue. func (c BaseController) runWorker(ctx context.Context) func() { return func() { for c.processNextWorkItem(ctx) { } } } // processNextWorkItem will read a single work item off the workqueue and // attempt to process it, by calling the syncHandler. func (c BaseController) processNextWorkItem(ctx context.Context) bool { obj, shutdown := c.Workqueue.Get() if shutdown { return false } err := func(obj interface{}) error { defer c.Workqueue.Done(obj) var key string var ok bool if key, ok = obj.(string); !ok { c.Workqueue.Forget(obj) runtime.HandleError(fmt.Errorf("expected string in workqueue but got %#v", obj)) return nil } if err := c.SyncHandler(ctx, key); err != nil { return fmt.Errorf("error syncing '%s': %s", key, err.Error()) } c.Workqueue.Forget(obj) return nil }(obj) if err != nil { runtime.HandleError(err) return true } return true } // enqueueFunction takes a fni resource and converts it into a namespace/name // string which is then put onto the work queue. This method should *not* be // passed resources of any type other than fni. func (c *BaseController) EnqueueFunction(obj interface{}) { var key string var err error if key, err = cache.MetaNamespaceKeyFunc(obj); err != nil { runtime.HandleError(err) return } c.Workqueue.AddRateLimited(key) } // handleObject will take any resource implementing metav1.Object and attempt // to find the fni resource that 'owns' it. It does this by looking at the // objects metadata.ownerReferences field for an appropriate OwnerReference. // It then enqueues that fni resource to be processed. If the object does not // have an appropriate OwnerReference, it will simply be skipped. func (c BaseController) HandleObject(obj interface{}) { var object metav1.Object var ok bool if object, ok = obj.(metav1.Object); !ok { tombstone, ok := obj.(cache.DeletedFinalStateUnknown) if !ok { runtime.HandleError(fmt.Errorf("error decoding object, invalid type")) return } object, ok = tombstone.Obj.(metav1.Object) if !ok { runtime.HandleError(fmt.Errorf("error decoding object tombstone, invalid type")) return } klog.V(4).Infof("Recovered deleted object '%s' from tombstone", object.GetName()) } klog.V(4).Infof("Processing object: %s", object.GetName()) if ownerRef := metav1.GetControllerOf(object); ownerRef != nil { // If this object is not owned by a fni, we should not do anything more // with it. if ownerRef.Kind != FaasIngressKind { return } fni, err := c.FunctionsLister.InferenceIngresses(object.GetNamespace()).Get(ownerRef.Name) if err != nil { klog.Infof("FunctionIngress '%s' deleted. Ignoring orphaned object '%s': %v", ownerRef.Name, object.GetSelfLink(), err) return } c.EnqueueFunction(fni) return } } func (c BaseController) SetupEventHandlers( functionIngress v1.InferenceIngressInformer, kubeInformerFactory kubeinformers.SharedInformerFactory, ) { functionIngress.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: c.EnqueueFunction, UpdateFunc: func(old, new interface{}) { oldFn, ok := CheckCustomResourceType(old) if !ok { return } newFn, ok := CheckCustomResourceType(new) if !ok { return } diffSpec := cmp.Diff(oldFn.Spec, newFn.Spec) diffAnnotations := cmp.Diff(oldFn.ObjectMeta.Annotations, newFn.ObjectMeta.Annotations) if diffSpec != "" || diffAnnotations != "" { c.EnqueueFunction(new) } }, }) // Set up an event handler for when functions related resources like pods, deployments, replica sets // can't be materialized. This logs abnormal events like ImagePullBackOff, back-off restarting failed container, // failed to start container, oci runtime errors, etc // Enable this with -v=3 kubeInformerFactory.Core().V1().Events().Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: func(obj interface{}) { key, err := cache.MetaNamespaceKeyFunc(obj) if err == nil { event := obj.(*corev1.Event) since := time.Since(event.LastTimestamp.Time) // log abnormal events occurred in the last minute if since.Seconds() < 61 && strings.Contains(event.Type, "Warning") { klog.V(3).Infof("Abnormal event detected on %s %s: %s", event.LastTimestamp, key, event.Message) } } }, }) } func GetClass(ingressType string) string { switch ingressType { case "": case "nginx": return "nginx" default: return ingressType } return "nginx" } func GetIssuerKind(issuerType string) string { switch issuerType { case "ClusterIssuer": return "cert-manager.io/cluster-issuer" default: return "cert-manager.io/issuer" } } func MakeAnnotations(fni *faasv1.InferenceIngress, host string) map[string]string { controlPlane, exist := fni.Annotations[consts.AnnotationControlPlaneKey] class := GetClass(fni.Spec.IngressType) specJSON, _ := json.Marshal(fni) annotations := make(map[string]string) annotations["ai.tensorchord.spec"] = string(specJSON) inferenceNamespace := fni.Labels[consts.LabelInferenceNamespace] if !fni.Spec.BypassGateway { switch class { case "nginx": switch host { // TODO: make this configurable case "apiserver": annotations["nginx.ingress.kubernetes.io/rewrite-target"] = "/api/v1/" + fni.Spec.Framework + "/" + fni.Spec.Function + "/$1" annotations["nginx.ingress.kubernetes.io/use-regex"] = "true" default: // for inference created by modelz apiserver if exist && controlPlane == consts.ModelzAnnotationValue { annotations["nginx.ingress.kubernetes.io/rewrite-target"] = "/api/v1/" + fni.Spec.Framework + "/" + fni.Spec.Function + "/$1" annotations["nginx.ingress.kubernetes.io/use-regex"] = "true" annotations["nginx.ingress.kubernetes.io/ssl-redirect"] = "false" } else { annotations["nginx.ingress.kubernetes.io/rewrite-target"] = "/inference/" + fni.Name + "." + inferenceNamespace + "/$1" annotations["nginx.ingress.kubernetes.io/ssl-redirect"] = "false" annotations["nginx.ingress.kubernetes.io/use-regex"] = "true" } } } } annotations["nginx.ingress.kubernetes.io/proxy-send-timeout"] = "300" annotations["nginx.ingress.kubernetes.io/proxy-read-timeout"] = "300" annotations["nginx.ingress.kubernetes.io/proxy-body-size"] = "16m" // We use the default certificate for now. // if fni.Spec.UseTLS() { // issuerType := GetIssuerKind(fni.Spec.TLS.IssuerRef.Kind) // annotations[issuerType] = fni.Spec.TLS.IssuerRef.Name // } // Set annotations with overrides from FunctionIngress // annotations for k, v := range fni.ObjectMeta.Annotations { annotations[k] = v } return annotations } func MakeOwnerRef(fni *faasv1.InferenceIngress) []metav1.OwnerReference { ref := []metav1.OwnerReference{ *metav1.NewControllerRef(fni, schema.GroupVersionKind{ Group: faasv1.SchemeGroupVersion.Group, Version: faasv1.SchemeGroupVersion.Version, Kind: FaasIngressKind, }), } return ref } func CheckCustomResourceType(obj interface{}) (faasv1.InferenceIngress, bool) { var fn *faasv1.InferenceIngress var ok bool if fn, ok = obj.(*faasv1.InferenceIngress); !ok { klog.Errorf("Event Watch received an invalid object: %#v", obj) return faasv1.InferenceIngress{}, false } return *fn, true } func IngressNeedsUpdate(old, fni *faasv1.InferenceIngress) bool { return !cmp.Equal(old.Spec, fni.Spec) || !cmp.Equal(old.ObjectMeta.Annotations, fni.ObjectMeta.Annotations) } func EventRecorder(client kubernetes.Interface) record.EventRecorder { // Create event broadcaster // Add o6s types to the default Kubernetes Scheme so Events can be // logged for faas-controller types. faasscheme.AddToScheme(scheme.Scheme) klog.V(4).Info("Creating event broadcaster") eventBroadcaster := record.NewBroadcaster() eventBroadcaster.StartLogging(klog.V(4).Infof) eventBroadcaster.StartRecordingToSink(&typedcorev1.EventSinkImpl{Interface: client.CoreV1().Events("")}) return eventBroadcaster.NewRecorder(scheme.Scheme, corev1.EventSource{Component: AgentName}) } ================================================ FILE: ingress-operator/pkg/controller/core_test.go ================================================ package controller import ( "testing" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" faasv1 "github.com/tensorchord/openmodelz/ingress-operator/pkg/apis/modelzetes/v1" ) func TestMakeAnnotations(t *testing.T) { cases := []struct { name string ingress faasv1.InferenceIngress expected map[string]string excluded []string }{ { name: "can override ingress class value", ingress: faasv1.InferenceIngress{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ "kubernetes.io/ingress.class": "awesome-nginx", }, }, Spec: faasv1.InferenceIngressSpec{ IngressType: "awesome-nginx", }, }, expected: map[string]string{ "kubernetes.io/ingress.class": "awesome-nginx", }, }, { name: "bypass removes rewrite target", ingress: faasv1.InferenceIngress{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ "kubernetes.io/ingress.class": "nginx", }, }, Spec: faasv1.InferenceIngressSpec{ IngressType: "nginx", Function: "nodeinfo", BypassGateway: true, Domain: "nodeinfo.example.com", }, }, excluded: []string{"nginx.ingress.kubernetes.io/rewrite-target"}, }, { name: "default annotations includes a rewrite-target", ingress: faasv1.InferenceIngress{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{}, }, Spec: faasv1.InferenceIngressSpec{ IngressType: "nginx", }, }, expected: map[string]string{ "nginx.ingress.kubernetes.io/rewrite-target": "/api/v1///$1", }, }, { name: "creates required nginx annotations", ingress: faasv1.InferenceIngress{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ "kubernetes.io/ingress.class": "nginx", }, }, Spec: faasv1.InferenceIngressSpec{ IngressType: "nginx", Framework: "mosec", Function: "main", }, }, expected: map[string]string{ "nginx.ingress.kubernetes.io/rewrite-target": "/api/v1/mosec/main/$1", }, }, { name: "creates required skipper annotations", ingress: faasv1.InferenceIngress{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ "kubernetes.io/ingress.class": "skipper", "zalando.org/skipper-filter": `setPath("/function/nodeinfo")`, }, }, Spec: faasv1.InferenceIngressSpec{ IngressType: "skipper", Function: "nodeinfo", BypassGateway: false, Domain: "nodeinfo.example.com", }, }, expected: map[string]string{ "kubernetes.io/ingress.class": "skipper", "zalando.org/skipper-filter": `setPath("/function/nodeinfo")`, }, }, // { // name: "creates tls issuer annotation", // ingress: faasv1.InferenceIngress{ // ObjectMeta: metav1.ObjectMeta{ // Annotations: map[string]string{ // "kubernetes.io/ingress.class": "nginx", // }, // }, // Spec: faasv1.InferenceIngressSpec{ // IngressType: "nginx", // Function: "nodeinfo", // BypassGateway: false, // Domain: "nodeinfo.example.com", // TLS: &faasv1.InferenceIngressTLS{ // IssuerRef: faasv1.ObjectReference{ // Name: "clusterFoo", // Kind: "ClusterIssuer", // }, // Enabled: true, // }, // }, // }, // expected: map[string]string{ // "cert-manager.io/cluster-issuer": "clusterFoo", // }, // }, // { // name: "default tls issuer is local", // ingress: faasv1.InferenceIngress{ // ObjectMeta: metav1.ObjectMeta{ // Annotations: map[string]string{ // "kubernetes.io/ingress.class": "nginx", // }, // }, // Spec: faasv1.InferenceIngressSpec{ // IngressType: "nginx", // Function: "nodeinfo", // BypassGateway: false, // Domain: "nodeinfo.example.com", // TLS: &faasv1.InferenceIngressTLS{ // IssuerRef: faasv1.ObjectReference{ // Name: "clusterFoo", // }, // Enabled: true, // }, // }, // }, // expected: map[string]string{ // "cert-manager.io/issuer": "clusterFoo", // }, // }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { result := MakeAnnotations(&tc.ingress, "apiserver") for key, value := range tc.expected { found, ok := result[key] if !ok { t.Fatalf("Failed to find expected annotation: %q", key) } if found != value { t.Fatalf("expected annotation value %q, got %q", value, found) } } for _, key := range tc.excluded { value, ok := result[key] if ok { t.Fatalf("annotations should not include %q, but it was found with value %q", key, value) } } }) } } ================================================ FILE: ingress-operator/pkg/controller/v1/controller.go ================================================ package v1 import ( "context" "encoding/json" "fmt" "strings" pkgerrors "github.com/pkg/errors" "github.com/sirupsen/logrus" faasv1 "github.com/tensorchord/openmodelz/ingress-operator/pkg/apis/modelzetes/v1" clientset "github.com/tensorchord/openmodelz/ingress-operator/pkg/client/clientset/versioned" informers "github.com/tensorchord/openmodelz/ingress-operator/pkg/client/informers/externalversions" listers "github.com/tensorchord/openmodelz/ingress-operator/pkg/client/listers/modelzetes/v1" "github.com/tensorchord/openmodelz/ingress-operator/pkg/config" "github.com/tensorchord/openmodelz/ingress-operator/pkg/controller" corev1 "k8s.io/api/core/v1" netv1 "k8s.io/api/networking/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/runtime" kubeinformers "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" networkingv1 "k8s.io/client-go/listers/networking/v1" "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/record" "k8s.io/client-go/util/workqueue" ) // SyncHandler is the controller implementation for Function resources type SyncHandler struct { config config.Config // kubeclientset is a standard kubernetes clientset kubeclientset kubernetes.Interface functionsLister listers.InferenceIngressLister ingressLister networkingv1.IngressLister // recorder is an event recorder for recording Event resources to the // Kubernetes API. recorder record.EventRecorder } // NewController returns a new OpenFaaS controller func NewController( cfg config.Config, kubeclientset kubernetes.Interface, faasclientset clientset.Interface, kubeInformerFactory kubeinformers.SharedInformerFactory, functionIngressFactory informers.SharedInformerFactory, ) controller.BaseController { recorder := controller.EventRecorder(kubeclientset) functionIngress := functionIngressFactory.Tensorchord().V1().InferenceIngresses() ingressInformer := kubeInformerFactory.Networking().V1().Ingresses() ingressLister := ingressInformer.Lister() syncer := SyncHandler{ config: cfg, kubeclientset: kubeclientset, functionsLister: functionIngress.Lister(), ingressLister: ingressLister, recorder: recorder, } ctrl := controller.BaseController{ FunctionsLister: functionIngress.Lister(), FunctionsSynced: functionIngress.Informer().HasSynced, Workqueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "FunctionIngresses"), SyncHandler: syncer.handler, } logrus.Info("Setting up event handlers") ctrl.SetupEventHandlers(functionIngress, kubeInformerFactory) ingressInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ DeleteFunc: ctrl.HandleObject, }) return ctrl } // handler compares the actual state with the desired, and attempts to // converge the two. It then updates the Status block of the fni resource // with the current status of the resource. func (h SyncHandler) handler(ctx context.Context, key string) error { // Convert the namespace/name string into a distinct namespace and name namespace, name, err := cache.SplitMetaNamespaceKey(key) if err != nil { runtime.HandleError(fmt.Errorf("invalid resource key: %s", key)) return nil } // Get the fni resource with this namespace/name fni, err := h.functionsLister.InferenceIngresses(namespace).Get(name) if err != nil { // The fni resource may no longer exist, in which case we stop processing. if errors.IsNotFound(err) { runtime.HandleError(fmt.Errorf("function ingress '%s' in work queue no longer exists", key)) return nil } return err } logger := logrus.WithFields(logrus.Fields{ "inference": fni.Name, "namespace": fni.Namespace, }) ingresses := h.ingressLister.Ingresses(namespace) ingress, getIngressErr := ingresses.Get(fni.Name) createIngress := errors.IsNotFound(getIngressErr) if !createIngress && ingress == nil { logrus.Errorf("cannot get ingress: %s in %s, error: %s", fni.Name, namespace, getIngressErr.Error()) } logger.Debugf("createIngress: %v", createIngress) if createIngress { host := h.config.Controller.Host rules := makeRules(fni, host) tls := makeTLS(fni) ns := h.config.Controller.Namespace newIngress := netv1.Ingress{ ObjectMeta: metav1.ObjectMeta{ Name: name, Namespace: ns, Annotations: controller.MakeAnnotations(fni, host), OwnerReferences: controller.MakeOwnerRef(fni), }, Spec: netv1.IngressSpec{ Rules: rules, IngressClassName: &fni.Spec.IngressType, TLS: tls, }, } _, createErr := h.kubeclientset.NetworkingV1().Ingresses(ns).Create(ctx, &newIngress, metav1.CreateOptions{}) if createErr != nil { logger.Errorf("cannot create ingress: %v in %v, error: %v", name, namespace, createErr.Error()) } h.recorder.Event(fni, corev1.EventTypeNormal, controller.SuccessSynced, controller.MessageResourceSynced) return nil } old := faasv1.InferenceIngress{} if val, ok := ingress.Annotations["ai.tensorchord.spec"]; ok && len(val) > 0 { unmarshalErr := json.Unmarshal([]byte(val), &old) if unmarshalErr != nil { return pkgerrors.Wrap(unmarshalErr, "unable to unmarshal from field inference") } } // Update the Deployment resource if the fni definition differs if controller.IngressNeedsUpdate(&old, fni) { logger.Debugf("updating ingress: %s in %s", fni.Name, namespace) if old.ObjectMeta.Name != fni.ObjectMeta.Name { return fmt.Errorf("cannot rename object") } updated := ingress.DeepCopy() rules := makeRules(fni, h.config.Controller.Host) annotations := controller.MakeAnnotations(fni, h.config.Controller.Host) for k, v := range annotations { updated.Annotations[k] = v } updated.Spec.Rules = rules updated.Spec.TLS = makeTLS(fni) _, updateErr := h.kubeclientset.NetworkingV1().Ingresses(namespace).Update(ctx, updated, metav1.UpdateOptions{}) if updateErr != nil { logrus.Errorf("error updating ingress: %v", updateErr) return updateErr } } // If an error occurs during Get/Create, we'll requeue the item so we can // attempt processing again later. This could have been caused by a // temporary network failure, or any other transient reason. if err != nil { return fmt.Errorf("transient error: %v", err) } h.recorder.Event(fni, corev1.EventTypeNormal, controller.SuccessSynced, controller.MessageResourceSynced) return nil } func makeRules(fni *faasv1.InferenceIngress, host string) []netv1.IngressRule { path := "/(.*)" if fni.Spec.BypassGateway { path = "/" } if len(fni.Spec.Path) > 0 { path = fni.Spec.Path } if controller.GetClass(fni.Spec.IngressType) == "traefik" { // We have to trim the regex and the trailing slash for Traefik, // otherwise routing won't work path = strings.TrimRight(path, "/(.*)") if len(path) == 0 { path = "/" } } pathType := netv1.PathTypeImplementationSpecific return []netv1.IngressRule{ { Host: fni.Spec.Domain, IngressRuleValue: netv1.IngressRuleValue{ HTTP: &netv1.HTTPIngressRuleValue{ Paths: []netv1.HTTPIngressPath{ { Path: path, PathType: &pathType, Backend: netv1.IngressBackend{ Service: &netv1.IngressServiceBackend{ Name: host, Port: netv1.ServiceBackendPort{ Number: controller.OpenfaasWorkloadPort, }, }, }, }, }, }, }, }, } } func makeTLS(fni *faasv1.InferenceIngress) []netv1.IngressTLS { if !fni.Spec.UseTLS() { return []netv1.IngressTLS{} } return []netv1.IngressTLS{ { // Use default secret name, thus no need to specify SecretName. Hosts: []string{ fni.Spec.Domain, }, }, } } ================================================ FILE: ingress-operator/pkg/controller/v1/controller_factory.go ================================================ package v1 import ( "errors" "fmt" "strings" "github.com/sirupsen/logrus" kubeinformers "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/clientcmd" clientset "github.com/tensorchord/openmodelz/ingress-operator/pkg/client/clientset/versioned" informers "github.com/tensorchord/openmodelz/ingress-operator/pkg/client/informers/externalversions" "github.com/tensorchord/openmodelz/ingress-operator/pkg/config" "github.com/tensorchord/openmodelz/ingress-operator/pkg/controller" ) func New(c config.Config, stopCh <-chan struct{}) (*controller.BaseController, error) { clientCmdConfig, err := clientcmd.BuildConfigFromFlags( c.KubeConfig.MasterURL, c.KubeConfig.Kubeconfig) if err != nil { return nil, fmt.Errorf("error building kubeconfig: %s", err.Error()) } clientCmdConfig.QPS = float32(c.KubeConfig.QPS) clientCmdConfig.Burst = c.KubeConfig.Burst kubeClient, err := kubernetes.NewForConfig(clientCmdConfig) if err != nil { return nil, fmt.Errorf("error building Kubernetes clientset: %s", err.Error()) } ingressClient, err := clientset.NewForConfig(clientCmdConfig) if err != nil { return nil, fmt.Errorf("error building Inference clientset: %s", err.Error()) } kubeInformerFactory := kubeinformers.NewSharedInformerFactoryWithOptions(kubeClient, c.KubeConfig.ResyncPeriod) ingressInformerFactory := informers.NewSharedInformerFactoryWithOptions(ingressClient, c.KubeConfig.ResyncPeriod) capabilities, err := getPreferredAvailableAPIs(kubeClient, "Ingress") if err != nil { return nil, fmt.Errorf("error retrieving Kubernetes cluster capabilities: %s", err.Error()) } logrus.Infof("cluster supports ingress in: %s", capabilities) if !capabilities.Has("networking.k8s.io/v1") { return nil, errors.New("networking.k8s.io/v1 is not available") } inferenceIngresses := ingressInformerFactory.Tensorchord().V1().InferenceIngresses() go inferenceIngresses.Informer().Run(stopCh) if ok := cache.WaitForNamedCacheSync( fmt.Sprintf("%s:inferenceingresses", "tensorchord"), stopCh, inferenceIngresses.Informer().HasSynced); !ok { return nil, errors.New("failed to wait for inferenceingresses caches to sync") } ingresses := kubeInformerFactory.Networking().V1().Ingresses() go ingresses.Informer().Run(stopCh) if ok := cache.WaitForNamedCacheSync( fmt.Sprintf("%s:ingresses", "networking"), stopCh, ingresses.Informer().HasSynced); !ok { return nil, errors.New("failed to wait for ingresses caches to sync") } ctr := NewController(c, kubeClient, ingressClient, kubeInformerFactory, ingressInformerFactory) return &ctr, nil } // getPreferredAvailableAPIs queries the cluster for the preferred resources information and returns a Capabilities // instance containing those api groups that support the specified kind. // // kind should be the title case singular name of the kind. For example, "Ingress" is the kind for a resource "ingress". func getPreferredAvailableAPIs(client kubernetes.Interface, kind string) (Capabilities, error) { discoveryclient := client.Discovery() lists, err := discoveryclient.ServerPreferredResources() if err != nil { return nil, err } caps := Capabilities{} for _, list := range lists { if len(list.APIResources) == 0 { continue } for _, resource := range list.APIResources { if len(resource.Verbs) == 0 { continue } if resource.Kind == kind { caps[list.GroupVersion] = true } } } return caps, nil } type Capabilities map[string]bool func (c Capabilities) Has(wanted string) bool { return c[wanted] } func (c Capabilities) String() string { keys := make([]string, 0, len(c)) for k := range c { keys = append(keys, k) } return strings.Join(keys, ", ") } ================================================ FILE: ingress-operator/pkg/controller/v1/controller_test.go ================================================ package v1 import ( "reflect" "testing" netv1 "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" faasv1 "github.com/tensorchord/openmodelz/ingress-operator/pkg/apis/modelzetes/v1" "github.com/tensorchord/openmodelz/ingress-operator/pkg/controller" ) func Test_makeRules_Nginx_RootPath_HasRegex(t *testing.T) { ingress := faasv1.InferenceIngress{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{}, }, Spec: faasv1.InferenceIngressSpec{ IngressType: "nginx", }, } rules := makeRules(&ingress, "apiserver") if len(rules) == 0 { t.Errorf("Ingress should give at least one rule") t.Fail() } wantPath := "/(.*)" gotPath := rules[0].HTTP.Paths[0].Path if gotPath != wantPath { t.Errorf("want path %s, but got %s", wantPath, gotPath) } gotPort := rules[0].HTTP.Paths[0].Backend.Service.Port.Number if gotPort != controller.OpenfaasWorkloadPort { t.Errorf("want port %d, but got %d", controller.OpenfaasWorkloadPort, gotPort) } } func Test_makeRules_Nginx_RootPath_IsRootWithBypassMode(t *testing.T) { wantFunction := "apiserver" ingress := faasv1.InferenceIngress{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{}, }, Spec: faasv1.InferenceIngressSpec{ BypassGateway: true, IngressType: "nginx", Function: "nodeinfo", // Path: "/", }, } rules := makeRules(&ingress, "apiserver") if len(rules) == 0 { t.Errorf("Ingress should give at least one rule") t.Fail() } wantPath := "/" gotPath := rules[0].HTTP.Paths[0].Path if gotPath != wantPath { t.Errorf("want path %s, but got %s", wantPath, gotPath) } gotHost := rules[0].HTTP.Paths[0].Backend.Service.Name if gotHost != wantFunction { t.Errorf("want host to be function: %s, but got %s", wantFunction, gotHost) } } func Test_makeRules_Nginx_PathOverride(t *testing.T) { ingress := faasv1.InferenceIngress{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{}, }, Spec: faasv1.InferenceIngressSpec{ IngressType: "nginx", Path: "/v1/profiles/view/(.*)", }, } rules := makeRules(&ingress, "apiserver") if len(rules) == 0 { t.Errorf("Ingress should give at least one rule") t.Fail() } wantPath := ingress.Spec.Path gotPath := rules[0].HTTP.Paths[0].Path if gotPath != wantPath { t.Errorf("want path %s, but got %s", wantPath, gotPath) } } func Test_makeRules_Traefik_RootPath_TrimsRegex(t *testing.T) { ingress := faasv1.InferenceIngress{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{}, }, Spec: faasv1.InferenceIngressSpec{ IngressType: "traefik", }, } rules := makeRules(&ingress, "apiserver") if len(rules) == 0 { t.Errorf("Ingress should give at least one rule") t.Fail() } wantPath := "/" gotPath := rules[0].HTTP.Paths[0].Path if gotPath != wantPath { t.Errorf("want path %s, but got %s", wantPath, gotPath) } } func Test_makeRules_Traefik_NestedPath_TrimsRegex_And_TrailingSlash(t *testing.T) { ingress := faasv1.InferenceIngress{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{}, }, Spec: faasv1.InferenceIngressSpec{ IngressType: "traefik", Path: "/v1/profiles/view/(.*)", }, } rules := makeRules(&ingress, "apiserver") if len(rules) == 0 { t.Errorf("Ingress should give at least one rule") t.Fail() } wantPath := "/v1/profiles/view" gotPath := rules[0].HTTP.Paths[0].Path if gotPath != wantPath { t.Errorf("want path %s, but got %s", wantPath, gotPath) } } func Test_makeTLS(t *testing.T) { cases := []struct { name string fni *faasv1.InferenceIngress expected []netv1.IngressTLS }{ { name: "tls disabled results in empty tls config", fni: &faasv1.InferenceIngress{ Spec: faasv1.InferenceIngressSpec{ TLS: &faasv1.InferenceIngressTLS{ Enabled: false, }, }, }, expected: []netv1.IngressTLS{}, }, { name: "tls enabled creates TLS object with correct host and secret with matching the host", fni: &faasv1.InferenceIngress{ Spec: faasv1.InferenceIngressSpec{ Domain: "foo.example.com", TLS: &faasv1.InferenceIngressTLS{ Enabled: true, IssuerRef: faasv1.ObjectReference{ Name: "test-issuer", Kind: "ClusterIssuer", }, }, }, }, expected: []netv1.IngressTLS{ { Hosts: []string{ "foo.example.com", }, }, }, }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { got := makeTLS(tc.fni) if !reflect.DeepEqual(tc.expected, got) { t.Fatalf("want tls config %v, got %v", tc.expected, got) } }) } } ================================================ FILE: ingress-operator/pkg/controller/v1/docs.go ================================================ package v1 /* v1 package provides the original ingress controller implementation. This provides support for ingress operator on k8s >= 1.19 networking/v1 api group */ ================================================ FILE: ingress-operator/pkg/signals/signal.go ================================================ package signals import ( "os" "os/signal" ) var onlyOneSignalHandler = make(chan struct{}) // SetupSignalHandler registered for SIGTERM and SIGINT. A stop channel is returned // which is closed on one of these signals. If a second signal is caught, the program // is terminated with exit code 1. func SetupSignalHandler() (stopCh <-chan struct{}) { close(onlyOneSignalHandler) // panics when called twice stop := make(chan struct{}) c := make(chan os.Signal, 2) signal.Notify(c, shutdownSignals...) go func() { <-c close(stop) <-c os.Exit(1) // second signal. Exit directly. }() return stop } ================================================ FILE: ingress-operator/pkg/signals/signal_posix.go ================================================ //go:build !windows // +build !windows package signals import ( "os" "syscall" ) var shutdownSignals = []os.Signal{os.Interrupt, syscall.SIGTERM} ================================================ FILE: ingress-operator/pkg/signals/signal_windows.go ================================================ package signals import ( "os" ) var shutdownSignals = []os.Signal{os.Interrupt} ================================================ FILE: ingress-operator/pkg/version/version.go ================================================ /* Copyright The TensorChord Inc. Copyright The BuildKit Authors. Copyright The containerd Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package version import ( "fmt" "regexp" "runtime" "strings" "sync" ) var ( // Package is filled at linking time Package = "github.com/tensorchord/openmodelz/modelzetes" // Revision is filled with the VCS (e.g. git) revision being used to build // the program at linking time. Revision = "" version = "0.0.0+unknown" buildDate = "1970-01-01T00:00:00Z" // output from `date -u +'%Y-%m-%dT%H:%M:%SZ'` gitCommit = "" // output from `git rev-parse HEAD` gitTag = "" // output from `git describe --exact-match --tags HEAD` (if clean tree state) gitTreeState = "" // determined from `git status --porcelain`. either 'clean' or 'dirty' developmentFlag = "false" ) // Version contains envd version information type Version struct { Version string BuildDate string GitCommit string GitTag string GitTreeState string GoVersion string Compiler string Platform string } func (v Version) String() string { return v.Version } // SetGitTagForE2ETest sets the gitTag for test purpose. func SetGitTagForE2ETest(tag string) { gitTag = tag } // GetEnvdVersion gets Envd version information func GetEnvdVersion() string { var versionStr string if gitCommit != "" && gitTag != "" && gitTreeState == "clean" && developmentFlag == "false" { // if we have a clean tree state and the current commit is tagged, // this is an official release. versionStr = gitTag } else { // otherwise formulate a version string based on as much metadata // information we have available. if strings.HasPrefix(version, "v") { versionStr = version } else { versionStr = "v" + version } if len(gitCommit) >= 7 { versionStr += "+" + gitCommit[0:7] if gitTreeState != "clean" { versionStr += ".dirty" } } else { versionStr += "+unknown" } } return versionStr } // GetVersion returns the version information func GetVersion() Version { return Version{ Version: GetEnvdVersion(), BuildDate: buildDate, GitCommit: gitCommit, GitTag: gitTag, GitTreeState: gitTreeState, GoVersion: runtime.Version(), Compiler: runtime.Compiler, Platform: fmt.Sprintf("%s/%s", runtime.GOOS, runtime.GOARCH), } } var ( reRelease *regexp.Regexp reDev *regexp.Regexp reOnce sync.Once ) func UserAgent() string { version := GetVersion().String() reOnce.Do(func() { reRelease = regexp.MustCompile(`^(v[0-9]+\.[0-9]+)\.[0-9]+$`) reDev = regexp.MustCompile(`^(v[0-9]+\.[0-9]+)\.[0-9]+`) }) if matches := reRelease.FindAllStringSubmatch(version, 1); len(matches) > 0 { version = matches[0][1] } else if matches := reDev.FindAllStringSubmatch(version, 1); len(matches) > 0 { version = matches[0][1] + "-dev" } return "envd/" + version } ================================================ FILE: ingress-operator/vendor.go ================================================ //go:build vendor package main // This file exists to trick "go mod vendor" to include "main" packages. // It is not expected to build, the build tag above is only to prevent this // file from being included in builds. import ( _ "k8s.io/code-generator/cmd/client-gen" _ "k8s.io/code-generator/cmd/deepcopy-gen" _ "k8s.io/code-generator/cmd/defaulter-gen" _ "k8s.io/code-generator/cmd/informer-gen" _ "k8s.io/code-generator/cmd/lister-gen" _ "k8s.io/code-generator/cmd/openapi-gen" ) func main() {} ================================================ FILE: mdz/.gitignore ================================================ # Binaries for programs and plugins *.exe *.dll *.so *.dylib # Test binary, build with `go test -c` *.test # Output of the go coverage tool, specifically when used with LiteIDE *.out # Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736 .glide/ .idea bin/ debug-bin/ **/password.txt **/gateway-password.txt .vscode of_kind_portforward.pid /kind* /kubectl /yaml_armhf /yaml_arm64 /broker-* /chart/pro-builder/out /chart/pro-builder/payload.txt /pgconnector.yaml jwt_key jwt_key.pub /*.pid .tools/ ================================================ FILE: mdz/Makefile ================================================ # Copyright 2022 TensorChord Inc. # # The old school Makefile, following are required targets. The Makefile is written # to allow building multiple binaries. You are free to add more targets or change # existing implementations, as long as the semantics are preserved. # # make - default to 'build' target # make lint - code analysis # make test - run unit test (or plus integration test) # make build - alias to build-local target # make build-local - build local binary targets # make build-linux - build linux binary targets # make container - build containers # $ docker login registry -u username -p xxxxx # make push - push containers # make clean - clean up targets # # Not included but recommended targets: # make e2e-test # # The makefile is also responsible to populate project version information. # # # Tweak the variables based on your project. # # This repo's root import path (under GOPATH). ROOT := github.com/tensorchord/openmodelz/mdz # Target binaries. You can build multiple binaries for a single project. TARGETS := mdz # Container image prefix and suffix added to targets. # The final built images are: # $[REGISTRY]/$[IMAGE_PREFIX]$[TARGET]$[IMAGE_SUFFIX]:$[VERSION] # $[REGISTRY] is an item from $[REGISTRIES], $[TARGET] is an item from $[TARGETS]. IMAGE_PREFIX ?= $(strip ) IMAGE_SUFFIX ?= $(strip ) # Container registries. REGISTRY ?= ghcr.io/tensorchord # Container registry for base images. BASE_REGISTRY ?= docker.io BASE_REGISTRY_USER ?= modelzai # Disable CGO by default. CGO_ENABLED ?= 0 GOOS ?= $(shell go env GOOS) GOARCH ?= $(shell go env GOARCH) # # These variables should not need tweaking. # # It's necessary to set this because some environments don't link sh -> bash. export SHELL := bash # It's necessary to set the errexit flags for the bash shell. export SHELLOPTS := errexit PACKAGE_NAME := github.com/tensorchord/openmodelz/mdz GOLANG_CROSS_VERSION ?= v1.17.6 # Project main package location (can be multiple ones). CMD_DIR := ./cmd # Project output directory. OUTPUT_DIR := ./bin DEBUG_DIR := ./debug-bin # Build directory. BUILD_DIR := ./build # Current version of the project. VERSION ?= $(shell git describe --match 'v[0-9]*' --always --tags --abbrev=0) BUILD_DATE=$(shell date -u +'%Y-%m-%dT%H:%M:%SZ') GIT_COMMIT=$(shell git rev-parse HEAD) GIT_TAG=$(shell if [ -z "`git status --porcelain`" ]; then git describe --exact-match --tags HEAD 2>/dev/null; fi) GIT_TREE_STATE=$(shell if [ -z "`git status --porcelain`" ]; then echo "clean" ; else echo "dirty"; fi) GITSHA ?= $(shell git rev-parse --short HEAD) BUILD_FLAGS ?= -s -w \ -X $(ROOT)/pkg/version.version=$(VERSION) \ -X $(ROOT)/pkg/version.buildDate=$(BUILD_DATE) \ -X $(ROOT)/pkg/version.gitCommit=$(GIT_COMMIT) \ -X $(ROOT)/pkg/version.gitTreeState=$(GIT_TREE_STATE) # Track code version with Docker Label. DOCKER_LABELS ?= git-describe="$(shell date -u +v%Y%m%d)-$(shell git describe --tags --always --dirty)" # Golang standard bin directory. GOPATH ?= $(shell go env GOPATH) GOROOT ?= $(shell go env GOROOT) BIN_DIR := $(GOPATH)/bin GOLANGCI_LINT := $(BIN_DIR)/golangci-lint # check if we need embed the dashboard DASHBOARD_BUILD ?= debug # Default golang flags used in build and test # -mod=vendor: force go to use the vendor files instead of using the `$GOPATH/pkg/mod` # -p: the number of programs that can be run in parallel # -count: run each test and benchmark 1 times. Set this flag to disable test cache export GOFLAGS ?= -count=1 # # Define all targets. At least the following commands are required: # # All targets. .PHONY: help lint test build container push addlicense debug debug-local build-local generate clean test-local addlicense-install release build-image .DEFAULT_GOAL:=build build: build-release ## Build the release version help: ## Display this help @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z0-9_-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) debug: debug-local ## Build the debug version # more info about `GOGC` env: https://github.com/golangci/golangci-lint#memory-usage-of-golangci-lint lint: $(GOLANGCI_LINT) ## Lint GO code @$(GOLANGCI_LINT) run $(GOLANGCI_LINT): curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $$(go env GOPATH)/bin mockgen-install: go install github.com/golang/mock/mockgen@v1.6.0 addlicense-install: go install github.com/google/addlicense@latest sqlc-install: go install github.com/kyleconroy/sqlc/cmd/sqlc@latest # https://github.com/swaggo/swag/pull/1322, we should use master instead of latest for now. swag-install: go install github.com/swaggo/swag/cmd/swag@v1.8.7 build-local: @for target in $(TARGETS); do \ CGO_ENABLED=$(CGO_ENABLED) GOOS=$(GOOS) GOARCH=$(GOARCH) go build \ -trimpath -v -o $(OUTPUT_DIR)/$${target} \ -ldflags "$(BUILD_FLAGS)" \ $(CMD_DIR)/$${target}; \ done build-release: BUILD_FLAGS += -X $(ROOT)/pkg/version.gitTag=$(GIT_TAG) build-release: @for target in $(TARGETS); do \ CGO_ENABLED=$(CGO_ENABLED) go build -trimpath -o $(OUTPUT_DIR)/$${target} \ -ldflags "$(BUILD_FLAGS)" \ $(CMD_DIR)/$${target}; \ done # It is used by vscode to attach into the process. debug-local: @for target in $(TARGETS); do \ CGO_ENABLED=$(CGO_ENABLED) go build -tags $(DASHBOARD_BUILD) -trimpath \ -v -o $(DEBUG_DIR)/$${target} \ -ldflags "$(BUILD_FLAGS)" \ -gcflags='all=-N -l' \ $(CMD_DIR)/$${target}; \ done addlicense: addlicense-install ## Add license to GO code files addlicense -l mpl -c "TensorChord Inc." $$(find . -type f -name '*.go' | grep -v pkg/docs/docs.go) test-local: @go test -tags=$(DASHBOARD_BUILD) -v -race -coverprofile=coverage.out ./... test: ## Run the tests @go test -tags=$(DASHBOARD_BUILD) -race -coverpkg=./pkg/... -coverprofile=coverage.out ./... @go tool cover -func coverage.out | tail -n 1 | awk '{ print "Total coverage: " $$3 }' clean: ## Clean the outputs and artifacts @-rm -vrf ${OUTPUT_DIR} @-rm -vrf ${DEBUG_DIR} @-rm -vrf build dist .eggs *.egg-info fmt: swag-install ## Run go fmt against code. go fmt ./... swag fmt vet: ## Run go vet against code. go vet ./... build-image: build-local docker build -t ${BASE_REGISTRY}/${BASE_REGISTRY_USER}/modelz-autoscaler:dev -f Dockerfile ./bin docker push ${BASE_REGISTRY}/${BASE_REGISTRY_USER}/autoscaler:dev release: @if [ ! -f ".release-env" ]; then \ echo "\033[91m.release-env is required for release\033[0m";\ exit 1;\ fi docker run \ --rm \ --privileged \ -e CGO_ENABLED=1 \ --env-file .release-env \ -v /var/run/docker.sock:/var/run/docker.sock \ -v `pwd`:/go/src/$(PACKAGE_NAME) \ -v `pwd`/sysroot:/sysroot \ -w /go/src/$(PACKAGE_NAME) \ goreleaser/goreleaser-cross:${GOLANG_CROSS_VERSION} \ release --rm-dist tsschema: swag @cd dashboard; pnpm tsschema generate: mockgen-install sqlc-install swag tsschema @mockgen -source pkg/query/querier.go -destination pkg/query/mock/mock.go -package mock @sqlc generate dashboard-build: @cd dashboard; pnpm build ================================================ FILE: mdz/README.md ================================================
# mdz CLI for OpenModelZ.

discord invitation link trackgit-views

## Installation ``` pip install openmodelz ``` ## CLI Reference Please check out our [CLI Documentation](./docs/cli/mdz.md) ================================================ FILE: mdz/cmd/mdz/main.go ================================================ /* Copyright © 2023 NAME HERE */ package main import "github.com/tensorchord/openmodelz/mdz/pkg/cmd" func main() { cmd.Execute() } ================================================ FILE: mdz/docs/cli/mdz.md ================================================ ## mdz mdz manages your deployments ### Synopsis mdz helps you deploy applications, manage servers, and troubleshoot issues. ### Examples ``` mdz server start mdz deploy --image modelzai/llm-bloomz-560m:23.06.13 --name llm mdz list mdz logs llm mdz port-forward llm 7860 mdz exec llm ps mdz exec llm --tty bash mdz delete llm ``` ### Options ``` --debug Enable debug logging --disable-telemetry Disable anonymous telemetry -h, --help help for mdz -u, --url string URL to use for the server (MDZ_URL) (default http://localhost:80) ``` ### SEE ALSO * [mdz delete](mdz_delete.md) - Delete OpenModelz inferences * [mdz deploy](mdz_deploy.md) - Deploy a new deployment * [mdz exec](mdz_exec.md) - Execute a command in a deployment * [mdz list](mdz_list.md) - List the deployments * [mdz logs](mdz_logs.md) - Print the logs for a deployment * [mdz port-forward](mdz_port-forward.md) - Forward one local port to a deployment * [mdz scale](mdz_scale.md) - Scale a deployment * [mdz server](mdz_server.md) - Manage the servers * [mdz version](mdz_version.md) - Print the client and agent version information ###### Auto generated by spf13/cobra on 11-Aug-2023 ================================================ FILE: mdz/docs/cli/mdz_delete.md ================================================ ## mdz delete Delete OpenModelz inferences ### Synopsis Deletes OpenModelZ inferences ``` mdz delete [flags] ``` ### Examples ``` mdz delete blomdz-560m ``` ### Options ``` -h, --help help for delete ``` ### Options inherited from parent commands ``` --debug Enable debug logging --disable-telemetry Disable anonymous telemetry -u, --url string URL to use for the server (MDZ_URL) (default http://localhost:80) ``` ### SEE ALSO * [mdz](mdz.md) - mdz manages your deployments ###### Auto generated by spf13/cobra on 11-Aug-2023 ================================================ FILE: mdz/docs/cli/mdz_deploy.md ================================================ ## mdz deploy Deploy a new deployment ### Synopsis Deploys a new deployment directly via flags. ``` mdz deploy [flags] ``` ### Examples ``` mdz deploy --image=modelzai/llm-blomdz-560m:23.06.13 mdz deploy --image=modelzai/llm-blomdz-560m:23.06.13 --name blomdz-560m --node-labels gpu=true,name=node-name ``` ### Options ``` --command string Command to run --gpu int Number of GPUs -h, --help help for deploy --image string Image to deploy --max-replicas int32 Maximum number of replicas (default 1) --min-replicas int32 Minimum number of replicas (can be 0) (default 1) --name string Name of inference -l, --node-labels strings Node labels --port int32 Port to deploy on (default 8080) --probe-path string HTTP Health probe path ``` ### Options inherited from parent commands ``` --debug Enable debug logging --disable-telemetry Disable anonymous telemetry -u, --url string URL to use for the server (MDZ_URL) (default http://localhost:80) ``` ### SEE ALSO * [mdz](mdz.md) - mdz manages your deployments ###### Auto generated by spf13/cobra on 11-Aug-2023 ================================================ FILE: mdz/docs/cli/mdz_exec.md ================================================ ## mdz exec Execute a command in a deployment ### Synopsis Execute a command in a deployment. If no instance is specified, the first instance is used. ``` mdz exec [flags] ``` ### Examples ``` mdz exec bloomz-560m ps mdz exec bloomz-560m --instance bloomz-560m-abcde-abcde ps mdz exec bllomz-560m -ti bash mdz exec bloomz-560m --instance bloomz-560m-abcde-abcde -ti bash ``` ### Options ``` -h, --help help for exec -s, --instance string Instance name -i, --interactive Keep stdin open even if not attached -t, --tty Allocate a TTY for the container ``` ### Options inherited from parent commands ``` --debug Enable debug logging --disable-telemetry Disable anonymous telemetry -u, --url string URL to use for the server (MDZ_URL) (default http://localhost:80) ``` ### SEE ALSO * [mdz](mdz.md) - mdz manages your deployments ###### Auto generated by spf13/cobra on 11-Aug-2023 ================================================ FILE: mdz/docs/cli/mdz_list.md ================================================ ## mdz list List the deployments ### Synopsis List the deployments ``` mdz list [flags] ``` ### Examples ``` mdz list mdz list -v mdz list -q ``` ### Options ``` -h, --help help for list -q, --quiet Quiet mode - print out only the inference names -v, --verbose Verbose mode - print out all inference details ``` ### Options inherited from parent commands ``` --debug Enable debug logging --disable-telemetry Disable anonymous telemetry -u, --url string URL to use for the server (MDZ_URL) (default http://localhost:80) ``` ### SEE ALSO * [mdz](mdz.md) - mdz manages your deployments * [mdz list instance](mdz_list_instance.md) - List all instances for the given deployment ###### Auto generated by spf13/cobra on 11-Aug-2023 ================================================ FILE: mdz/docs/cli/mdz_list_instance.md ================================================ ## mdz list instance List all instances for the given deployment ### Synopsis List all instances for the given deployment ``` mdz list instance [flags] ``` ### Examples ``` mdz list instance bloomz-560m mdz list instance bloomz-560m -v mdz list instance bloomz-560m -q ``` ### Options ``` -h, --help help for instance -q, --quiet Quiet mode - print out only the instance names -v, --verbose Verbose mode - print out all instance details ``` ### Options inherited from parent commands ``` --debug Enable debug logging --disable-telemetry Disable anonymous telemetry -u, --url string URL to use for the server (MDZ_URL) (default http://localhost:80) ``` ### SEE ALSO * [mdz list](mdz_list.md) - List the deployments ###### Auto generated by spf13/cobra on 11-Aug-2023 ================================================ FILE: mdz/docs/cli/mdz_logs.md ================================================ ## mdz logs Print the logs for a deployment ### Synopsis Print the logs for a deployment ``` mdz logs [flags] ``` ### Examples ``` mdz logs blomdz-560m ``` ### Options ``` -e, --end string Only return logs before this timestamp (e.g. 2013-01-02T13:23:37Z) or relative (e.g. 42m for 42 minutes) -f, --follow Follow log output -h, --help help for logs -s, --since string Show logs since timestamp (e.g. 2013-01-02T13:23:37Z) or relative (e.g. 42m for 42 minutes) (default "2006-01-02T15:04:05Z") -t, --tail int Number of lines to show from the end of the logs ``` ### Options inherited from parent commands ``` --debug Enable debug logging --disable-telemetry Disable anonymous telemetry -u, --url string URL to use for the server (MDZ_URL) (default http://localhost:80) ``` ### SEE ALSO * [mdz](mdz.md) - mdz manages your deployments ###### Auto generated by spf13/cobra on 11-Aug-2023 ================================================ FILE: mdz/docs/cli/mdz_port-forward.md ================================================ ## mdz port-forward Forward one local port to a deployment ### Synopsis Forward one local port to a deployment ``` mdz port-forward [flags] ``` ### Examples ``` mdz port-forward blomdz-560m 7860 ``` ### Options ``` -h, --help help for port-forward ``` ### Options inherited from parent commands ``` --debug Enable debug logging --disable-telemetry Disable anonymous telemetry -u, --url string URL to use for the server (MDZ_URL) (default http://localhost:80) ``` ### SEE ALSO * [mdz](mdz.md) - mdz manages your deployments ###### Auto generated by spf13/cobra on 11-Aug-2023 ================================================ FILE: mdz/docs/cli/mdz_scale.md ================================================ ## mdz scale Scale a deployment ### Synopsis Scale a deployment ``` mdz scale [flags] ``` ### Examples ``` mdz scale bloomz-560m --replicas 3 ``` ### Options ``` -h, --help help for scale -r, --replicas int32 Number of replicas to scale to ``` ### Options inherited from parent commands ``` --debug Enable debug logging --disable-telemetry Disable anonymous telemetry -u, --url string URL to use for the server (MDZ_URL) (default http://localhost:80) ``` ### SEE ALSO * [mdz](mdz.md) - mdz manages your deployments ###### Auto generated by spf13/cobra on 11-Aug-2023 ================================================ FILE: mdz/docs/cli/mdz_server.md ================================================ ## mdz server Manage the servers ### Synopsis Manage the servers ### Examples ``` mdz server start ``` ### Options ``` -h, --help help for server -v, --verbose Verbose output ``` ### Options inherited from parent commands ``` --debug Enable debug logging --disable-telemetry Disable anonymous telemetry -u, --url string URL to use for the server (MDZ_URL) (default http://localhost:80) ``` ### SEE ALSO * [mdz](mdz.md) - mdz manages your deployments * [mdz server delete](mdz_server_delete.md) - Delete a node from the cluster * [mdz server destroy](mdz_server_destroy.md) - Destroy the cluster * [mdz server join](mdz_server_join.md) - Join to the cluster * [mdz server label](mdz_server_label.md) - Update the labels on a server * [mdz server list](mdz_server_list.md) - List all servers in the cluster * [mdz server start](mdz_server_start.md) - Start the server * [mdz server stop](mdz_server_stop.md) - Stop the server ###### Auto generated by spf13/cobra on 11-Aug-2023 ================================================ FILE: mdz/docs/cli/mdz_server_delete.md ================================================ ## mdz server delete Delete a node from the cluster ### Synopsis Delete a node from the cluster ``` mdz server delete [flags] ``` ### Examples ``` mdz server delete gpu-node-1 ``` ### Options ``` -h, --help help for delete ``` ### Options inherited from parent commands ``` --debug Enable debug logging --disable-telemetry Disable anonymous telemetry -u, --url string URL to use for the server (MDZ_URL) (default http://localhost:80) -v, --verbose Verbose output ``` ### SEE ALSO * [mdz server](mdz_server.md) - Manage the servers ###### Auto generated by spf13/cobra on 11-Aug-2023 ================================================ FILE: mdz/docs/cli/mdz_server_destroy.md ================================================ ## mdz server destroy Destroy the cluster ### Synopsis Destroy the cluster ``` mdz server destroy [flags] ``` ### Examples ``` mdz server destroy ``` ### Options ``` -h, --help help for destroy ``` ### Options inherited from parent commands ``` --debug Enable debug logging --disable-telemetry Disable anonymous telemetry -u, --url string URL to use for the server (MDZ_URL) (default http://localhost:80) -v, --verbose Verbose output ``` ### SEE ALSO * [mdz server](mdz_server.md) - Manage the servers ###### Auto generated by spf13/cobra on 11-Aug-2023 ================================================ FILE: mdz/docs/cli/mdz_server_join.md ================================================ ## mdz server join Join to the cluster ### Synopsis Join to the cluster ``` mdz server join [flags] ``` ### Examples ``` mdz server join 192.168.31.192 ``` ### Options ``` -h, --help help for join --mirror-endpoints https://quay.io Mirror URL endpoints of the registry like https://quay.io --mirror-name string Mirror domain name of the registry (default "docker.io") ``` ### Options inherited from parent commands ``` --debug Enable debug logging --disable-telemetry Disable anonymous telemetry -u, --url string URL to use for the server (MDZ_URL) (default http://localhost:80) -v, --verbose Verbose output ``` ### SEE ALSO * [mdz server](mdz_server.md) - Manage the servers ###### Auto generated by spf13/cobra on 11-Aug-2023 ================================================ FILE: mdz/docs/cli/mdz_server_label.md ================================================ ## mdz server label Update the labels on a server ### Synopsis Update the labels on a server * A label key and value must begin with a letter or number, and may contain letters, numbers, hyphens, dots, and underscores, up to 63 characters each. * Optionally, the key can begin with a DNS subdomain prefix and a single '/', like example.com/my-app. ``` mdz server label [flags] ``` ### Examples ``` mdz server label node-name key=value [key=value...] ``` ### Options ``` -h, --help help for label ``` ### Options inherited from parent commands ``` --debug Enable debug logging --disable-telemetry Disable anonymous telemetry -u, --url string URL to use for the server (MDZ_URL) (default http://localhost:80) -v, --verbose Verbose output ``` ### SEE ALSO * [mdz server](mdz_server.md) - Manage the servers ###### Auto generated by spf13/cobra on 11-Aug-2023 ================================================ FILE: mdz/docs/cli/mdz_server_list.md ================================================ ## mdz server list List all servers in the cluster ### Synopsis List all servers in the cluster ``` mdz server list [flags] ``` ### Examples ``` mdz server list ``` ### Options ``` -h, --help help for list -q, --quiet Quiet mode - print out only the server names -v, --verbose Verbose mode - print out all server details ``` ### Options inherited from parent commands ``` --debug Enable debug logging --disable-telemetry Disable anonymous telemetry -u, --url string URL to use for the server (MDZ_URL) (default http://localhost:80) ``` ### SEE ALSO * [mdz server](mdz_server.md) - Manage the servers ###### Auto generated by spf13/cobra on 11-Aug-2023 ================================================ FILE: mdz/docs/cli/mdz_server_start.md ================================================ ## mdz server start Start the server ### Synopsis Start the server with the public IP of the machine. If not provided, the internal IP will be used automatically. ``` mdz server start [flags] ``` ### Examples ``` mdz server start mdz server start -v mdz server start 1.2.3.4 ``` ### Options ``` -g, --force-gpu Start the server with GPU support (ignore the GPU detection) -h, --help help for start --mirror-endpoints https://quay.io Mirror URL endpoints of the registry like https://quay.io --mirror-name string Mirror domain name of the registry (default "docker.io") ``` ### Options inherited from parent commands ``` --debug Enable debug logging --disable-telemetry Disable anonymous telemetry -u, --url string URL to use for the server (MDZ_URL) (default http://localhost:80) -v, --verbose Verbose output ``` ### SEE ALSO * [mdz server](mdz_server.md) - Manage the servers ###### Auto generated by spf13/cobra on 11-Aug-2023 ================================================ FILE: mdz/docs/cli/mdz_server_stop.md ================================================ ## mdz server stop Stop the server ### Synopsis Stop the server ``` mdz server stop [flags] ``` ### Examples ``` mdz server stop ``` ### Options ``` -h, --help help for stop ``` ### Options inherited from parent commands ``` --debug Enable debug logging --disable-telemetry Disable anonymous telemetry -u, --url string URL to use for the server (MDZ_URL) (default http://localhost:80) -v, --verbose Verbose output ``` ### SEE ALSO * [mdz server](mdz_server.md) - Manage the servers ###### Auto generated by spf13/cobra on 11-Aug-2023 ================================================ FILE: mdz/docs/cli/mdz_version.md ================================================ ## mdz version Print the client and agent version information ### Synopsis Print the client and server version information ``` mdz version [flags] ``` ### Examples ``` mdz version ``` ### Options ``` -h, --help help for version ``` ### Options inherited from parent commands ``` --debug Enable debug logging --disable-telemetry Disable anonymous telemetry -u, --url string URL to use for the server (MDZ_URL) (default http://localhost:80) ``` ### SEE ALSO * [mdz](mdz.md) - mdz manages your deployments ###### Auto generated by spf13/cobra on 11-Aug-2023 ================================================ FILE: mdz/docs/macOS-quickstart.md ================================================ # `mdz` Quick Start Guide for macOS `mdz` only runs on Linux. To use it on macOS, you'll need a Linux VM. This guide will show you how to set up a Linux VM on macOS, install and use `mdz` on it. - [`mdz` Quick Start Guide for macOS](#mdz-quick-start-guide-for-macos) - [VM Setup](#vm-setup) - [Install `mdz`](#install-mdz) - [Drop into the VM shell](#drop-into-the-vm-shell) - [Install build dependencies](#install-build-dependencies) - [Clone the repository and build](#clone-the-repository-and-build) - [`mdz` Usage](#mdz-usage) ## VM Setup I use [OrbStack](https://docs.orbstack.dev/machines/) to create and manage my VMs. ![](images/orbstack-vm-create.png) Open OrbStack and go to the `Linux Machines` tab, then click `Create` button to create a new VM if you don't already have one. This guide uses Archlinux, but you're free to use any distribution you like. ## Install `mdz` ### Drop into the VM shell ``` orb # now you're in the Linux VM's shell ``` ### Install build dependencies ``` # or use your favorite package manager sudo pacman -Sy go git ``` ### Clone the repository and build ``` git clone https://github.com/tensorchord/OpenModelZ.git cd OpenModelZ/mdz && make # `make` will build the `mdz` binary under `bin/` for you ``` ## `mdz` Usage ``` # replace with your `mdz` path, typically it's `./bin/mdz` mdz --help ``` ================================================ FILE: mdz/examples/bloomz-560m-openai/README.md ================================================ # Bloomz 560M OpenAI Compatible API This is a simple API that allows you to use the Bloomz 560M as a OpenAI Gym environment. ## Deploy ```bash $ mdz deploy --image modelzai/llm-bloomz-560m:23.06.13 --name llm ``` ### Get the deployment ```bash $ mdz list NAME ENDPOINT STATUS REPLICAS llm http://localhost:31112/inference/llm.default Ready 1/1 ``` ### Test the deployment ```python import openai openai.api_base="http://localhost:31112/inference/llm.default" openai.api_key="any" openai.debug = True # create a chat completion chat_completion = openai.ChatCompletion.create(model="", messages=[ {"role": "user", "content": "Who are you?"}, {"role": "assistant", "content": "I am a student"}, {"role": "user", "content": "What do you learn?"}, {"role": "assistant", "content": "I learn math"}, {"role": "user", "content": "Do you like english?"} ], max_tokens=100) ``` ================================================ FILE: mdz/hack/cli-doc-gen/main.go ================================================ package main import ( "fmt" "os" "path/filepath" cmd "github.com/tensorchord/openmodelz/mdz/pkg/cmd" ) func main() { path, err := os.Getwd() if err != nil { panic(err) } fmt.Println("Generating docs in", filepath.Join(path, "docs", "cli")) if err := cmd.GenMarkdownTree(filepath.Join(path, "docs", "cli")); err != nil { panic(err) } } ================================================ FILE: mdz/pkg/agentd/runtime/create.go ================================================ package runtime import ( "context" "fmt" "os" "time" dockertypes "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" "github.com/docker/go-connections/nat" "github.com/moby/moby/pkg/jsonmessage" "github.com/moby/term" "github.com/phayes/freeport" "github.com/tensorchord/openmodelz/agent/api/types" ) func (r *Runtime) InferenceCreate(ctx context.Context, req types.InferenceDeployment) error { cfg := &container.Config{ Image: req.Spec.Image, ExposedPorts: nat.PortSet{}, } var port int32 = 8080 if req.Spec.Port != nil { port = *req.Spec.Port } now := time.Now() req.Status = types.InferenceDeploymentStatus{ Phase: types.PhaseNotReady, Replicas: 1, CreatedAt: &now, } // Lock the mutex and set cache r.mutex.Lock() r.cache[req.Spec.Name] = req r.mutex.Unlock() go func() error { body, err := r.cli.ImagePull(context.TODO(), req.Spec.Image, dockertypes.ImagePullOptions{}) if err != nil { return err } defer body.Close() termFd, isTerm := term.GetFdInfo(os.Stdout) err = jsonmessage.DisplayJSONMessagesStream(body, os.Stdout, termFd, isTerm, nil) if err != nil { return err } hostCfg := &container.HostConfig{ RestartPolicy: container.RestartPolicy{ Name: "always", }, PortBindings: nat.PortMap{}, } natPort := nat.Port(fmt.Sprintf("%d/tcp", port)) hostPort, err := freeport.GetFreePort() if err != nil { return err } hostCfg.PortBindings[natPort] = []nat.PortBinding{ { HostIP: Localhost, HostPort: fmt.Sprintf("%d", hostPort), }, } cfg.ExposedPorts[natPort] = struct{}{} cfg.Labels = expectedLabels(req) ctr, err := r.cli.ContainerCreate(context.TODO(), cfg, hostCfg, nil, nil, req.Spec.Name) if err != nil { return err } if err := r.cli.ContainerStart(context.TODO(), ctr.ID, dockertypes.ContainerStartOptions{}); err != nil { return err } r.mutex.Lock() new := r.cache[req.Spec.Name] new.Status = types.InferenceDeploymentStatus{ Phase: types.PhaseReady, Replicas: 1, AvailableReplicas: 1, CreatedAt: &now, } r.mutex.Unlock() return nil }() return nil } ================================================ FILE: mdz/pkg/agentd/runtime/delete.go ================================================ package runtime import ( "context" dockertypes "github.com/docker/docker/api/types" "github.com/tensorchord/openmodelz/agent/client" ) func (r *Runtime) InferenceDelete(ctx context.Context, name string) error { defer func() { r.mutex.Lock() delete(r.cache, name) r.mutex.Unlock() }() ctr, err := r.cli.ContainerInspect(ctx, name) if err != nil { if !client.IsErrNotFound(err) { return nil } } if ctr.Config.Labels[labelVendor] != valueVendor { return nil } if err := r.cli.ContainerRemove(ctx, name, dockertypes.ContainerRemoveOptions{ Force: true, }); err != nil { return err } return nil } ================================================ FILE: mdz/pkg/agentd/runtime/label.go ================================================ package runtime import "github.com/tensorchord/openmodelz/agent/api/types" const ( labelVendor = "ai.modelz.open.vendor" valueVendor = "openmodelz" labelName = "ai.modelz.open.name" ) func expectedLabels(inf types.InferenceDeployment) map[string]string { return map[string]string{ labelVendor: valueVendor, labelName: inf.Spec.Name, } } ================================================ FILE: mdz/pkg/agentd/runtime/list.go ================================================ package runtime import ( "context" dockertypes "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/filters" "github.com/tensorchord/openmodelz/agent/api/types" ) func (r *Runtime) InferenceList(ns string) ([]types.InferenceDeployment, error) { r.mutex.Lock() res := r.cache r.mutex.Unlock() ctrs, err := r.cli.ContainerList(context.TODO(), dockertypes.ContainerListOptions{ Filters: filters.NewArgs(filters.Arg("label", labelVendor+"="+valueVendor)), }) if err != nil { return nil, err } for _, ctr := range ctrs { inf := types.InferenceDeployment{ Spec: types.InferenceDeploymentSpec{ Name: ctr.Labels[labelName], Image: ctr.Image, Namespace: "default", }, Status: types.InferenceDeploymentStatus{}, } if ctr.State == "running" { inf.Status.Phase = types.PhaseReady inf.Status.AvailableReplicas = 1 inf.Status.Replicas = 1 } else { inf.Status.Phase = types.PhaseNotReady inf.Status.Replicas = 1 } res[inf.Spec.Name] = inf } l := []types.InferenceDeployment{} for _, inf := range res { l = append(l, inf) } return l, nil } ================================================ FILE: mdz/pkg/agentd/runtime/proxy.go ================================================ package runtime import ( "errors" "net" "net/http" "net/http/httputil" "net/url" "time" "github.com/gin-gonic/gin" "github.com/tensorchord/openmodelz/agent/errdefs" ) func (r *Runtime) InfereceProxy(c *gin.Context, name string) error { ctr, err := r.cli.ContainerInspect(c.Request.Context(), name) if err != nil { return errdefs.System(err) } if ctr.Config.Labels[labelVendor] != valueVendor { return errdefs.NotFound(errors.New("container not found")) } port := "" for _, c := range ctr.HostConfig.PortBindings { if len(c) > 0 { port = c[0].HostPort break } } if port == "" { return errdefs.NotFound(errors.New("port not found")) } url, err := url.Parse("http://" + Localhost + ":" + port) if err != nil { return errdefs.System(err) } proxyServer := httputil.NewSingleHostReverseProxy(url) proxyServer.Transport = &http.Transport{ Proxy: http.ProxyFromEnvironment, DialContext: (&net.Dialer{ Timeout: time.Minute * 5, KeepAlive: time.Minute * 5, DualStack: true, }).DialContext, } proxyServer.Director = func(req *http.Request) { targetQuery := url.RawQuery req.URL.Scheme = url.Scheme req.URL.Host = url.Host // req.URL.Path, req.URL.RawPath = joinURLPath(target, req.URL) if targetQuery == "" || req.URL.RawQuery == "" { req.URL.RawQuery = targetQuery + req.URL.RawQuery } else { req.URL.RawQuery = targetQuery + "&" + req.URL.RawQuery } req.URL.Path = c.Param("proxyPath") if req.URL.Path == "" { req.URL.Path = "/" } } proxyServer.ServeHTTP(c.Writer, c.Request) return nil } ================================================ FILE: mdz/pkg/agentd/runtime/runtime.go ================================================ package runtime import ( "context" "sync" "github.com/docker/docker/client" "github.com/tensorchord/openmodelz/agent/api/types" ) type Runtime struct { cli client.APIClient cache map[string]types.InferenceDeployment mutex sync.Mutex } const ( Localhost = "127.0.0.1" ) func New() (*Runtime, error) { cli, err := client.NewClientWithOpts(client.FromEnv) if err != nil { return nil, err } cli.NegotiateAPIVersion(context.Background()) return &Runtime{ cli: cli, cache: map[string]types.InferenceDeployment{}, }, nil } ================================================ FILE: mdz/pkg/agentd/server/error.go ================================================ package server import ( "bytes" "fmt" "net/http" "github.com/tensorchord/openmodelz/agent/errdefs" ) // Error defines a standard application error. type Error struct { // Machine-readable error code. HTTPStatusCode int `json:"http_status_code,omitempty"` // Human-readable message. Message string `json:"message,omitempty"` Request string `json:"request,omitempty"` // Logical operation and nested error. Op string `json:"op,omitempty"` Err error `json:"error,omitempty"` } // Error returns the string representation of the error message. func (e *Error) Error() string { var buf bytes.Buffer // Print the current operation in our stack, if any. if e.Op != "" { fmt.Fprintf(&buf, "%s: ", e.Op) } // If wrapping an error, print its Error() message. // Otherwise print the error code & message. if e.Err != nil { buf.WriteString(e.Err.Error()) } else { if e.HTTPStatusCode != 0 { fmt.Fprintf(&buf, "<%s> ", http.StatusText(e.HTTPStatusCode)) } buf.WriteString(e.Message) } return buf.String() } func NewError(code int, err error, op string) error { return &Error{ HTTPStatusCode: code, Err: err, Message: err.Error(), Op: op, } } func errFromErrDefs(err error, op string) error { if errdefs.IsCancelled(err) { return NewError(http.StatusRequestTimeout, err, op) } else if errdefs.IsConflict(err) { return NewError(http.StatusConflict, err, op) } else if errdefs.IsDataLoss(err) { return NewError(http.StatusInternalServerError, err, op) } else if errdefs.IsDeadline(err) { return NewError(http.StatusRequestTimeout, err, op) } else if errdefs.IsForbidden(err) { return NewError(http.StatusForbidden, err, op) } else if errdefs.IsInvalidParameter(err) { return NewError(http.StatusBadRequest, err, op) } else if errdefs.IsNotFound(err) { return NewError(http.StatusNotFound, err, op) } else if errdefs.IsNotImplemented(err) { return NewError(http.StatusNotImplemented, err, op) } else if errdefs.IsNotModified(err) { return NewError(http.StatusNotModified, err, op) } else if errdefs.IsSystem(err) { return NewError(http.StatusInternalServerError, err, op) } else if errdefs.IsUnauthorized(err) { return NewError(http.StatusUnauthorized, err, op) } else if errdefs.IsUnavailable(err) { return NewError(http.StatusServiceUnavailable, err, op) } else if errdefs.IsUnknown(err) { return NewError(http.StatusInternalServerError, err, op) } return NewError(http.StatusInternalServerError, err, op) } ================================================ FILE: mdz/pkg/agentd/server/handler_healthz.go ================================================ package server import ( "net/http" "github.com/gin-gonic/gin" ) // @Summary Healthz // @Description Healthz // @Tags system // @Accept json // @Produce json // @Success 200 // @Router /healthz [get] func (s *Server) handleHealthz(c *gin.Context) error { c.Status(http.StatusOK) return nil } ================================================ FILE: mdz/pkg/agentd/server/handler_inference_create.go ================================================ package server import ( "net/http" "github.com/gin-gonic/gin" "github.com/tensorchord/openmodelz/agent/api/types" ) // @Summary Create the inferences. // @Description Create the inferences. // @Tags inference // @Accept json // @Produce json // @Param request body types.InferenceDeployment true "query params" // @Success 201 {object} types.InferenceDeployment // @Router /system/inferences [post] func (s *Server) handleInferenceCreate(c *gin.Context) error { event := types.DeploymentCreateEvent var req types.InferenceDeployment if err := c.ShouldBindJSON(&req); err != nil { return NewError(http.StatusBadRequest, err, event) } // Set the default values. s.validator.DefaultDeployRequest(&req) // Validate the request. if err := s.validator.ValidateDeployRequest(&req); err != nil { return NewError(http.StatusBadRequest, err, event) } // Create the inference. if err := s.runtime.InferenceCreate(c.Request.Context(), req); err != nil { return errFromErrDefs(err, event) } c.JSON(http.StatusCreated, req) return nil } ================================================ FILE: mdz/pkg/agentd/server/handler_inference_delete.go ================================================ package server import ( "errors" "net/http" "github.com/gin-gonic/gin" "github.com/tensorchord/openmodelz/agent/api/types" ) // @Summary Delete the inferences. // @Description Delete the inferences. // @Tags inference // @Accept json // @Produce json // @Param request body types.DeleteFunctionRequest true "query params" // @Param namespace query string true "Namespace" example("modelz-d3524a71-c17c-4c92-8faf-8603f02f4713") // @Success 202 {object} types.DeleteFunctionRequest // @Router /system/inferences [delete] func (s *Server) handleInferenceDelete(c *gin.Context) error { event := types.DeploymentDeleteEvent var req types.DeleteFunctionRequest if err := c.ShouldBindJSON(&req); err != nil { return NewError(http.StatusBadRequest, err, event) } if req.FunctionName == "" { return NewError( http.StatusBadRequest, errors.New("function name is required"), event) } if err := s.runtime.InferenceDelete(c.Request.Context(), req.FunctionName); err != nil { return errFromErrDefs(err, event) } c.JSON(http.StatusAccepted, req) return nil } ================================================ FILE: mdz/pkg/agentd/server/handler_inference_get.go ================================================ package server import ( "errors" "net/http" "github.com/gin-gonic/gin" _ "github.com/tensorchord/openmodelz/agent/api/types" ) // @Summary Get the inference by name. // @Description Get the inference by name. // @Tags inference // @Accept json // @Produce json // @Param namespace query string true "Namespace" example("modelz-d3524a71-c17c-4c92-8faf-8603f02f4713") // @Param name path string true "inference id" example("e50886f3-caa6-449f-9fa8-7849c6ba2e08") // @Success 200 {object} types.InferenceDeployment // @Router /system/inference/{name} [get] func (s *Server) handleInferenceGet(c *gin.Context) error { namespace := c.Query("namespace") if namespace == "" { return NewError( http.StatusBadRequest, errors.New("namespace is required"), "inference-get") } name := c.Param("name") if name == "" { return NewError( http.StatusBadRequest, errors.New("name is required"), "inference-get") } // function, err := s.runtime.InferenceGet(namespace, name) // if err != nil { // return errFromErrDefs(err, "inference-get") // } // c.JSON(http.StatusOK, function) return nil } ================================================ FILE: mdz/pkg/agentd/server/handler_inference_list.go ================================================ package server import ( "errors" "net/http" "github.com/gin-gonic/gin" _ "github.com/tensorchord/openmodelz/agent/api/types" ) // @Summary List the inferences. // @Description List the inferences. // @Tags inference // @Accept json // @Produce json // @Param namespace query string true "Namespace" example("modelz-d3524a71-c17c-4c92-8faf-8603f02f4713") // @Success 200 {object} []types.InferenceDeployment // @Router /system/inferences [get] func (s *Server) handleInferenceList(c *gin.Context) error { namespace := c.Query("namespace") if namespace == "" { return NewError( http.StatusBadRequest, errors.New("namespace is required"), "inference-list") } inferenes, err := s.runtime.InferenceList(namespace) if err != nil { return errFromErrDefs(err, "inference-list") } c.JSON(http.StatusOK, inferenes) return nil } ================================================ FILE: mdz/pkg/agentd/server/handler_inference_logs.go ================================================ package server import ( "github.com/gin-gonic/gin" ) // @Summary Get the inference logs. // @Description Get the inference logs. // @Tags log // @Accept json // @Produce json // @Param namespace query string true "Namespace" example("modelz-d3524a71-c17c-4c92-8faf-8603f02f4713") // @Param name query string true "Name" // @Param instance query string false "Instance" // @Param tail query int false "Tail" // @Param follow query bool false "Follow" // @Param since query string false "Since" example("2023-04-01T00:06:31+08:00") // @Param end query string false "End" example("2023-05-31T00:06:31+08:00") // @Success 200 {object} []types.Message // @Router /system/logs/inference [get] func (s *Server) handleInferenceLogs(c *gin.Context) error { return nil } ================================================ FILE: mdz/pkg/agentd/server/handler_inference_proxy.go ================================================ package server import ( "errors" "fmt" "net/http" "strings" "github.com/gin-gonic/gin" ) // @Summary Inference. // @Description Inference proxy. // @Tags inference-proxy // @Accept json // @Produce json // @Router /inference/{name} [post] // @Router /inference/{name} [get] // @Router /inference/{name} [put] // @Router /inference/{name} [delete] func (s *Server) handleInferenceProxy(c *gin.Context) error { namespacedName := c.Param("name") if namespacedName == "" { return NewError( http.StatusBadRequest, errors.New("name is required"), "inference-proxy") } _, name, err := getNamespaceAndName(namespacedName) if err != nil { return NewError( http.StatusBadRequest, err, "inference-proxy") } return s.runtime.InfereceProxy(c, name) } func getNamespaceAndName(name string) (string, string, error) { if !strings.Contains(name, ".") { return "", "", fmt.Errorf("name is not namespaced") } namespace := name[strings.LastIndexAny(name, ".")+1:] infName := strings.TrimSuffix(name, "."+namespace) if namespace == "" { return "", "", fmt.Errorf("namespace is empty") } if infName == "" { return "", "", fmt.Errorf("inference name is empty") } return namespace, infName, nil } ================================================ FILE: mdz/pkg/agentd/server/handler_info.go ================================================ package server import ( "net/http" "github.com/gin-gonic/gin" "github.com/tensorchord/openmodelz/agent/api/types" "github.com/tensorchord/openmodelz/mdz/pkg/version" ) // @Summary Get system info. // @Description Get system info. // @Tags system // @Accept json // @Produce json // @Success 200 {object} types.ProviderInfo // @Router /system/info [get] func (s *Server) handleInfo(c *gin.Context) error { v := version.GetVersion() c.JSON(http.StatusOK, types.ProviderInfo{ Name: "local agent", Orchestration: "docker", Version: &types.VersionInfo{ Version: v.Version, BuildDate: v.BuildDate, GitCommit: v.GitCommit, GitTag: v.GitTag, GitTreeState: v.GitTreeState, GoVersion: v.GoVersion, Compiler: v.Compiler, Platform: v.Platform, }, }) return nil } ================================================ FILE: mdz/pkg/agentd/server/middleware_callid.go ================================================ package server import ( "fmt" "time" "github.com/gin-gonic/gin" "github.com/google/uuid" ) func (s Server) middlewareCallID(c *gin.Context) error { start := time.Now() if len(c.Request.Header.Get("X-Call-Id")) == 0 { callID := uuid.New().String() c.Request.Header.Add("X-Call-Id", callID) c.Writer.Header().Add("X-Call-Id", callID) } c.Request.Header.Add("X-Start-Time", fmt.Sprintf("%d", start.UTC().UnixNano())) c.Writer.Header().Add("X-Start-Time", fmt.Sprintf("%d", start.UTC().UnixNano())) c.Next() return nil } ================================================ FILE: mdz/pkg/agentd/server/server_factory.go ================================================ package server import ( "github.com/gin-contrib/cors" "github.com/gin-gonic/gin" "github.com/sirupsen/logrus" "github.com/tensorchord/openmodelz/agent/pkg/server/validator" ginlogrus "github.com/toorop/gin-logrus" "github.com/tensorchord/openmodelz/mdz/pkg/agentd/runtime" ) type Server struct { router *gin.Engine metricsRouter *gin.Engine logger *logrus.Entry validator *validator.Validator runtime *runtime.Runtime } func New() (*Server, error) { router := gin.New() router.Use(ginlogrus.Logger(logrus.StandardLogger(), "/healthz")) router.Use(gin.Recovery()) // metrics server metricsRouter := gin.New() metricsRouter.Use(gin.Recovery()) if gin.Mode() == gin.DebugMode { logrus.SetLevel(logrus.DebugLevel) logrus.Debug("Allow CORS") router.Use(cors.New(cors.Config{ AllowOrigins: []string{"*"}, AllowMethods: []string{"GET", "POST", "PUT", "PATCH", "DELETE"}, AllowHeaders: []string{"*"}, })) } logger := logrus.WithField("component", "agentd") r, err := runtime.New() if err != nil { return nil, err } s := &Server{ router: router, metricsRouter: metricsRouter, logger: logger, validator: validator.New(), runtime: r, } s.registerRoutes() return s, nil } ================================================ FILE: mdz/pkg/agentd/server/server_handlerfunc.go ================================================ package server import ( "errors" "net/http" "github.com/gin-gonic/gin" "github.com/sirupsen/logrus" ) type HandlerFunc func(c *gin.Context) error func WrapHandler(handler HandlerFunc) gin.HandlerFunc { return func(c *gin.Context) { err := handler(c) if err != nil { var serverErr *Error if !errors.As(err, &serverErr) { serverErr = &Error{ HTTPStatusCode: http.StatusInternalServerError, Err: err, Message: err.Error(), } } serverErr.Request = c.Request.Method + " " + c.Request.URL.String() if gin.Mode() == "debug" { logrus.Debugf("error: %+v", err) } else { // Remove detailed info when in the release mode serverErr.Op = "" serverErr.Err = nil } c.JSON(serverErr.HTTPStatusCode, serverErr) c.Abort() return } } } ================================================ FILE: mdz/pkg/agentd/server/server_init_route.go ================================================ package server import ( swaggerfiles "github.com/swaggo/files" ginSwagger "github.com/swaggo/gin-swagger" ) const ( endpointInferencePlural = "/inferences" endpointInference = "/inference" endpointScaleInference = "/scale-inference" endpointInfo = "/info" endpointLogPlural = "/logs" endpointNamespacePlural = "/namespaces" endpointHealthz = "/healthz" endpointBuild = "/build" ) func (s *Server) registerRoutes() { root := s.router.Group("/") // swagger root.GET("/swagger/*any", ginSwagger.WrapHandler(swaggerfiles.Handler)) // dataplane root.Any("/inference/:name", WrapHandler(s.middlewareCallID), WrapHandler(s.handleInferenceProxy)) root.Any("/inference/:name/*proxyPath", WrapHandler(s.middlewareCallID), WrapHandler(s.handleInferenceProxy)) // healthz root.GET(endpointHealthz, WrapHandler(s.handleHealthz)) // control plane controlPlane := root.Group("/system") // inferences controlPlane.GET(endpointInferencePlural, WrapHandler(s.handleInferenceList)) controlPlane.POST(endpointInferencePlural, WrapHandler(s.handleInferenceCreate)) // controlPlane.PUT(endpointInferencePlural, // WrapHandler(s.handleInferenceUpdate)) controlPlane.DELETE(endpointInferencePlural, WrapHandler(s.handleInferenceDelete)) // controlPlane.POST(endpointScaleInference, // WrapHandler(s.handleInferenceScale)) controlPlane.GET(endpointInference+"/:name", WrapHandler(s.handleInferenceGet)) // instances // controlPlane.GET(endpointInference+"/:name/instances", // WrapHandler(s.handleInferenceInstance)) // info controlPlane.GET(endpointInfo, WrapHandler(s.handleInfo)) // logs controlPlane.GET(endpointLogPlural+endpointInference, WrapHandler(s.handleInferenceLogs)) } ================================================ FILE: mdz/pkg/agentd/server/server_run.go ================================================ package server import ( "context" "errors" "fmt" "net/http" "os" "os/signal" "syscall" "time" "github.com/sirupsen/logrus" ) func (s *Server) Run(port int) error { srv := &http.Server{ Addr: fmt.Sprintf(":%d", port), Handler: s.router, WriteTimeout: time.Hour * 24, ReadTimeout: time.Hour * 24, } go func() { if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { logrus.Errorf("listen on port %d error: %v", port, err) } }() logrus.WithField("port", port). Info("server is running...") quit := make(chan os.Signal, 1) signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM) <-quit logrus.Info("shutdown server") ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() return srv.Shutdown(ctx) } ================================================ FILE: mdz/pkg/cmd/delete.go ================================================ package cmd import ( "github.com/cockroachdb/errors" "github.com/spf13/cobra" ) // deleteCmd represents the delete command var deleteCmd = &cobra.Command{ Use: "delete", Short: "Delete OpenModelz inferences", Long: `Deletes OpenModelZ inferences`, Example: ` mdz delete blomdz-560m`, GroupID: "basic", PreRunE: commandInit, Args: cobra.ExactArgs(1), RunE: commandDelete, } func init() { rootCmd.AddCommand(deleteCmd) // Here you will define your flags and configuration settings. // Cobra supports Persistent Flags which will work for this command // and all subcommands, e.g.: // Cobra supports local flags which will only run when this command // is called directly, e.g.: } func commandDelete(cmd *cobra.Command, args []string) error { name := args[0] if err := agentClient.InferenceRemove( cmd.Context(), namespace, name); err != nil { cmd.PrintErrf("Failed to remove the inference: %s\n", errors.Cause(err)) return err } cmd.Printf("Inference %s is deleted\n", name) return nil } ================================================ FILE: mdz/pkg/cmd/deploy.go ================================================ package cmd import ( "math/rand" "strconv" "time" "github.com/cockroachdb/errors" petname "github.com/dustinkirkland/golang-petname" "github.com/spf13/cobra" "github.com/tensorchord/openmodelz/agent/api/types" "github.com/tensorchord/openmodelz/mdz/pkg/telemetry" ) var ( // Used for flags. deployImage string deployPort int32 deployMinReplicas int32 deployMaxReplicas int32 deployName string deployGPU int deployNodeLabel []string deployCommand string deployProbePath string ) // deployCmd represents the deploy command var deployCmd = &cobra.Command{ Use: "deploy", Short: "Deploy a new deployment", Long: `Deploys a new deployment directly via flags.`, Example: ` mdz deploy --image=modelzai/llm-blomdz-560m:23.06.13 mdz deploy --image=modelzai/llm-blomdz-560m:23.06.13 --name blomdz-560m --node-labels gpu=true,name=node-name`, GroupID: "basic", PreRunE: commandInit, RunE: commandDeploy, } func init() { rand.Seed(time.Now().UTC().UnixNano()) rootCmd.AddCommand(deployCmd) // Here you will define your flags and configuration settings. // Cobra supports Persistent Flags which will work for this command // and all subcommands, e.g.: // Cobra supports local flags which will only run when this command // is called directly, e.g.: // deployCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") deployCmd.Flags().StringVar(&deployImage, "image", "", "Image to deploy") deployCmd.Flags().Int32Var(&deployPort, "port", 8080, "Port to deploy on") deployCmd.Flags().Int32Var(&deployMinReplicas, "min-replicas", 1, "Minimum number of replicas (can be 0)") deployCmd.Flags().Int32Var(&deployMaxReplicas, "max-replicas", 1, "Maximum number of replicas") deployCmd.Flags().IntVar(&deployGPU, "gpu", 0, "Number of GPUs") deployCmd.Flags().StringVar(&deployName, "name", "", "Name of inference") deployCmd.Flags().StringSliceVarP(&deployNodeLabel, "node-labels", "l", []string{}, "Node labels") deployCmd.Flags().StringVar(&deployCommand, "command", "", "Command to run") deployCmd.Flags().StringVar(&deployProbePath, "probe-path", "", "HTTP Health probe path") } func commandDeploy(cmd *cobra.Command, args []string) error { if deployImage == "" { return cmd.Help() } name := deployName if name == "" { name = petname.Generate(2, "-") } var typ types.ScalingType = types.ScalingTypeCapacity inf := types.InferenceDeployment{ Spec: types.InferenceDeploymentSpec{ Image: deployImage, Namespace: namespace, Name: name, Labels: map[string]string{ "ai.tensorchord.name": name, }, Framework: types.FrameworkOther, Scaling: &types.ScalingConfig{ MinReplicas: int32Ptr(deployMinReplicas), MaxReplicas: int32Ptr(deployMaxReplicas), TargetLoad: int32Ptr(10), Type: &typ, StartupDuration: int32Ptr(600), ZeroDuration: int32Ptr(600), }, Port: int32Ptr(deployPort), }, } if deployCommand != "" { inf.Spec.Command = &deployCommand } if deployProbePath != "" { inf.Spec.HTTPProbePath = &deployProbePath } if len(deployNodeLabel) > 0 { inf.Spec.Constraints = []string{} for _, label := range deployNodeLabel { inf.Spec.Constraints = append(inf.Spec.Constraints, "tensorchord.ai/"+label) } } if deployGPU > 0 { GPUNum := types.Quantity(strconv.Itoa(deployGPU)) inf.Spec.Resources = &types.ResourceRequirements{ // no need to set Requests for GPU Limits: types.ResourceList{ types.ResourceGPU: GPUNum, }, } } telemetry.GetTelemetry().Record( "deploy", telemetry.AddField("GPU", deployGPU), telemetry.AddField("FromZero", deployMinReplicas == 0), ) if _, err := agentClient.InferenceCreate( cmd.Context(), namespace, inf); err != nil { cmd.PrintErrf("Failed to create the inference: %s\n", errors.Cause(err)) return err } cmd.Printf("Inference %s is created\n", inf.Spec.Name) return nil } func int32Ptr(i int32) *int32 { return &i } ================================================ FILE: mdz/pkg/cmd/exec.go ================================================ package cmd import ( "fmt" "io" "os" "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/tensorchord/openmodelz/agent/client" "github.com/tensorchord/openmodelz/mdz/pkg/cmd/streams" terminal "golang.org/x/term" "k8s.io/apimachinery/pkg/util/rand" ) var ( execInstance string execTTY bool execInteractive bool ) // execCommand represents the exec command var execCommand = &cobra.Command{ Use: "exec", Short: "Execute a command in a deployment", Long: `Execute a command in a deployment. If no instance is specified, the first instance is used.`, Example: ` mdz exec bloomz-560m ps mdz exec bloomz-560m --instance bloomz-560m-abcde-abcde ps mdz exec bllomz-560m -ti bash mdz exec bloomz-560m --instance bloomz-560m-abcde-abcde -ti bash`, GroupID: "debug", PreRunE: commandInit, Args: cobra.MinimumNArgs(1), RunE: commandExec, } func init() { rootCmd.AddCommand(execCommand) // Here you will define your flags and configuration settings. // Cobra supports Persistent Flags which will work for this command // and all subcommands, e.g.: // Cobra supports local flags which will only run when this command // is called directly, e.g.: execCommand.Flags().StringVarP(&execInstance, "instance", "s", "", "Instance name") execCommand.Flags().BoolVarP(&execTTY, "tty", "t", false, "Allocate a TTY for the container") execCommand.Flags().BoolVarP(&execInteractive, "interactive", "i", false, "Keep stdin open even if not attached") } func commandExec(cmd *cobra.Command, args []string) error { name := args[0] if execInstance == "" { instances, err := agentClient.InstanceList(cmd.Context(), namespace, name) if err != nil { cmd.PrintErrf("Failed to list instances: %s\n", errors.Cause(err)) return err } if len(instances) == 0 { cmd.PrintErrf("instance %s not found\n", name) return errors.Newf("instance %s not found", name) } else if len(instances) > 1 { cmd.PrintErrf("inference %s has multiple instances, please specify with -i\n", name) return errors.Newf("inference %s has multiple instances, please specify with -i", name) } execInstance = instances[0].Spec.Name } if execTTY { shell := "sh" if len(args) > 1 { shell = args[1] } else if len(args) > 2 { cmd.PrintErrf("too many arguments in tty mode, please use a shell program e.g. bash\n") return fmt.Errorf("too many arguments") } if !isAvailableShell(shell) { cmd.PrintErrf("shell %s is not available, try `sh` or `bash`\n", shell) return fmt.Errorf("shell %s is not available, try `sh` or `bash`", shell) } resp, err := agentClient.InstanceExecTTY(cmd.Context(), namespace, name, execInstance, []string{shell}) if err != nil { cmd.PrintErrf("Failed to execute the shell: %s\n", errors.Cause(err)) return err } defer resp.Conn.Close() c := resp.Conn if !terminal.IsTerminal(0) || !terminal.IsTerminal(1) { cmd.PrintErrf("stdin/stdout should be terminal\n") return fmt.Errorf("stdin/stdout should be terminal") } // oldState, err := terminal.MakeRaw(0) // if err != nil { // cmd.PrintErrf("Failed to make raw terminal: %s\n", errors.Cause(err)) // return err // } // oldOutState, err := terminal.MakeRaw(1) // if err != nil { // cmd.PrintErrf("Failed to make raw terminal: %s\n", errors.Cause(err)) // return err // } // defer func() { // terminal.Restore(0, oldState) // terminal.Restore(1, oldOutState) // }() // Send terminal size. w, h, err := terminal.GetSize(0) if err != nil { cmd.PrintErrf("Failed to get terminal size: %s\n", errors.Cause(err)) return err } msg := &client.TerminalMessage{ ID: rand.String(5), Op: "resize", Data: "", Rows: uint16(h), Cols: uint16(w), } if err := c.WriteJSON(msg); err != nil { cmd.PrintErrf("Failed to send terminal message: %s\n", errors.Cause(err)) return err } errCh := make(chan error, 1) cli := newMDZCLI() go func() { defer close(errCh) errCh <- func() error { streamer := hijackedIOStreamer{ streams: cli, inputStream: cli.In(), outputStream: cli.Out(), errorStream: cli.Err(), resp: resp, tty: true, detachKeys: "", } return streamer.stream(cmd.Context()) }() }() if err := <-errCh; err != nil { logrus.Debugf("Error hijack: %s", err) return err } return nil } else { res, err := agentClient.InstanceExec(cmd.Context(), namespace, name, execInstance, args[1:], false) if err != nil { cmd.PrintErrf("Failed to execute the command: %s\n", errors.Cause(err)) return err } cmd.Printf("%s", res) return nil } } func isAvailableShell(shell string) bool { switch shell { case "sh", "bash", "zsh", "fish": return true default: return false } } type mdzCli struct { in *streams.In out *streams.Out err io.Writer } func newMDZCLI() *mdzCli { return &mdzCli{ in: streams.NewIn(os.Stdin), out: streams.NewOut(os.Stdout), err: os.Stderr, } } func (c mdzCli) In() *streams.In { return c.in } func (c mdzCli) Out() *streams.Out { return c.out } func (c mdzCli) Err() io.Writer { return c.err } ================================================ FILE: mdz/pkg/cmd/exec_stream.go ================================================ package cmd import ( "context" "fmt" "io" "runtime" "sync" "github.com/moby/term" "github.com/sirupsen/logrus" "github.com/tensorchord/openmodelz/agent/client" "github.com/tensorchord/openmodelz/mdz/pkg/cmd/ioutils" "github.com/tensorchord/openmodelz/mdz/pkg/cmd/streams" ) // Streams is an interface which exposes the standard input and output streams type Streams interface { In() *streams.In Out() *streams.Out Err() io.Writer } // The default escape key sequence: ctrl-p, ctrl-q // TODO: This could be moved to `pkg/term`. var defaultEscapeKeys = []byte{16, 17} // A hijackedIOStreamer handles copying input to and output from streams to the // connection. type hijackedIOStreamer struct { streams Streams inputStream io.ReadCloser outputStream io.Writer errorStream io.Writer resp client.HijackedResponse tty bool detachKeys string } // stream handles setting up the IO and then begins streaming stdin/stdout // to/from the hijacked connection, blocking until it is either done reading // output, the user inputs the detach key sequence when in TTY mode, or when // the given context is cancelled. func (h *hijackedIOStreamer) stream(ctx context.Context) error { restoreInput, err := h.setupInput() if err != nil { return fmt.Errorf("unable to setup input stream: %s", err) } defer restoreInput() outputDone := h.beginOutputStream(restoreInput) inputDone, detached := h.beginInputStream(restoreInput) select { case err := <-outputDone: return err case <-inputDone: // Input stream has closed. if h.outputStream != nil || h.errorStream != nil { // Wait for output to complete streaming. select { case err := <-outputDone: return err case <-ctx.Done(): return ctx.Err() } } return nil case err := <-detached: // Got a detach key sequence. return err case <-ctx.Done(): return ctx.Err() } } func (h *hijackedIOStreamer) setupInput() (restore func(), err error) { if h.inputStream == nil || !h.tty { // No need to setup input TTY. // The restore func is a nop. return func() {}, nil } if err := setRawTerminal(h.streams); err != nil { return nil, fmt.Errorf("unable to set IO streams as raw terminal: %s", err) } // Use sync.Once so we may call restore multiple times but ensure we // only restore the terminal once. var restoreOnce sync.Once restore = func() { restoreOnce.Do(func() { _ = restoreTerminal(h.streams, h.inputStream) }) } // Wrap the input to detect detach escape sequence. // Use default escape keys if an invalid sequence is given. escapeKeys := defaultEscapeKeys if h.detachKeys != "" { customEscapeKeys, err := term.ToBytes(h.detachKeys) if err != nil { logrus.Warnf("invalid detach escape keys, using default: %s", err) } else { escapeKeys = customEscapeKeys } } h.inputStream = ioutils.NewReadCloserWrapper(term.NewEscapeProxy(h.inputStream, escapeKeys), h.inputStream.Close) return restore, nil } func (h *hijackedIOStreamer) beginOutputStream(restoreInput func()) <-chan error { if h.outputStream == nil && h.errorStream == nil { // There is no need to copy output. return nil } outputDone := make(chan error) go func() { var err error // When TTY is ON, use regular copy if h.outputStream != nil && h.tty { _, err = io.Copy(h.outputStream, h.resp) // We should restore the terminal as soon as possible // once the connection ends so any following print // messages will be in normal type. restoreInput() } logrus.Debug("[hijack] End of stdout") if err != nil { logrus.Debugf("Error receiveStdout: %s", err) } outputDone <- err }() return outputDone } func (h *hijackedIOStreamer) beginInputStream(restoreInput func()) (doneC <-chan struct{}, detachedC <-chan error) { inputDone := make(chan struct{}) detached := make(chan error) go func() { if h.inputStream != nil { _, err := io.Copy(h.resp, h.inputStream) // We should restore the terminal as soon as possible // once the connection ends so any following print // messages will be in normal type. restoreInput() logrus.Debug("[hijack] End of stdin") if _, ok := err.(term.EscapeError); ok { detached <- err return } if err != nil { // This error will also occur on the receive // side (from stdout) where it will be // propagated back to the caller. logrus.Debugf("Error sendStdin: %s", err) } } // if err := h.resp.CloseWrite(); err != nil { // logrus.Debugf("Couldn't send EOF: %s", err) // } close(inputDone) }() return inputDone, detached } func setRawTerminal(streams Streams) error { if err := streams.In().SetRawTerminal(); err != nil { return err } return streams.Out().SetRawTerminal() } func restoreTerminal(streams Streams, in io.Closer) error { streams.In().RestoreTerminal() streams.Out().RestoreTerminal() // WARNING: DO NOT REMOVE THE OS CHECKS !!! // For some reason this Close call blocks on darwin.. // As the client exits right after, simply discard the close // until we find a better solution. // // This can also cause the client on Windows to get stuck in Win32 CloseHandle() // in some cases. See https://github.com/docker/docker/issues/28267#issuecomment-288237442 // Tracked internally at Microsoft by VSO #11352156. In the // Windows case, you hit this if you are using the native/v2 console, // not the "legacy" console, and you start the client in a new window. eg // `start docker run --rm -it microsoft/nanoserver cmd /s /c echo foobar` // will hang. Remove start, and it won't repro. if in != nil && runtime.GOOS != "darwin" && runtime.GOOS != "windows" { return in.Close() } return nil } ================================================ FILE: mdz/pkg/cmd/ioutils/reader.go ================================================ package ioutils // import "github.com/docker/docker/pkg/ioutils" import ( "context" "io" // make sure crypto.SHA256, crypto.sha512 and crypto.SHA384 are registered // TODO remove once https://github.com/opencontainers/go-digest/pull/64 is merged. _ "crypto/sha256" _ "crypto/sha512" ) // ReadCloserWrapper wraps an io.Reader, and implements an io.ReadCloser // It calls the given callback function when closed. It should be constructed // with NewReadCloserWrapper type ReadCloserWrapper struct { io.Reader closer func() error } // Close calls back the passed closer function func (r *ReadCloserWrapper) Close() error { return r.closer() } // NewReadCloserWrapper returns a new io.ReadCloser. func NewReadCloserWrapper(r io.Reader, closer func() error) io.ReadCloser { return &ReadCloserWrapper{ Reader: r, closer: closer, } } type readerErrWrapper struct { reader io.Reader closer func() } func (r *readerErrWrapper) Read(p []byte) (int, error) { n, err := r.reader.Read(p) if err != nil { r.closer() } return n, err } // NewReaderErrWrapper returns a new io.Reader. func NewReaderErrWrapper(r io.Reader, closer func()) io.Reader { return &readerErrWrapper{ reader: r, closer: closer, } } // OnEOFReader wraps an io.ReadCloser and a function // the function will run at the end of file or close the file. type OnEOFReader struct { Rc io.ReadCloser Fn func() } func (r *OnEOFReader) Read(p []byte) (n int, err error) { n, err = r.Rc.Read(p) if err == io.EOF { r.runFunc() } return } // Close closes the file and run the function. func (r *OnEOFReader) Close() error { err := r.Rc.Close() r.runFunc() return err } func (r *OnEOFReader) runFunc() { if fn := r.Fn; fn != nil { fn() r.Fn = nil } } // cancelReadCloser wraps an io.ReadCloser with a context for cancelling read // operations. type cancelReadCloser struct { cancel func() pR *io.PipeReader // Stream to read from pW *io.PipeWriter } // NewCancelReadCloser creates a wrapper that closes the ReadCloser when the // context is cancelled. The returned io.ReadCloser must be closed when it is // no longer needed. func NewCancelReadCloser(ctx context.Context, in io.ReadCloser) io.ReadCloser { pR, pW := io.Pipe() // Create a context used to signal when the pipe is closed doneCtx, cancel := context.WithCancel(context.Background()) p := &cancelReadCloser{ cancel: cancel, pR: pR, pW: pW, } go func() { _, err := io.Copy(pW, in) select { case <-ctx.Done(): // If the context was closed, p.closeWithError // was already called. Calling it again would // change the error that Read returns. default: p.closeWithError(err) } in.Close() }() go func() { for { select { case <-ctx.Done(): p.closeWithError(ctx.Err()) case <-doneCtx.Done(): return } } }() return p } // Read wraps the Read method of the pipe that provides data from the wrapped // ReadCloser. func (p *cancelReadCloser) Read(buf []byte) (n int, err error) { return p.pR.Read(buf) } // closeWithError closes the wrapper and its underlying reader. It will // cause future calls to Read to return err. func (p *cancelReadCloser) closeWithError(err error) { p.pW.CloseWithError(err) p.cancel() } // Close closes the wrapper its underlying reader. It will cause // future calls to Read to return io.EOF. func (p *cancelReadCloser) Close() error { p.closeWithError(io.EOF) return nil } ================================================ FILE: mdz/pkg/cmd/list.go ================================================ package cmd import ( "fmt" "sort" "strings" "github.com/jedib0t/go-pretty/v6/table" "github.com/spf13/cobra" "github.com/tensorchord/openmodelz/agent/api/types" "github.com/tensorchord/openmodelz/mdz/pkg/telemetry" ) const ( annotationDomain = "ai.tensorchord.domain" ) var ( // Used for flags. listQuiet bool listVerbose bool ) // listCommand represents the list command var listCommand = &cobra.Command{ Use: "list", Short: "List the deployments", Long: `List the deployments`, Example: ` mdz list mdz list -v mdz list -q`, GroupID: "basic", PreRunE: commandInit, RunE: commandList, } func init() { rootCmd.AddCommand(listCommand) // Here you will define your flags and configuration settings. // Cobra supports Persistent Flags which will work for this command // and all subcommands, e.g.: // Cobra supports local flags which will only run when this command // is called directly, e.g.: listCommand.Flags().BoolVarP(&listQuiet, "quiet", "q", false, "Quiet mode - print out only the inference names") listCommand.Flags().BoolVarP(&listVerbose, "verbose", "v", false, "Verbose mode - print out all inference details") } func commandList(cmd *cobra.Command, args []string) error { telemetry.GetTelemetry().Record("list") infs, err := agentClient.InferenceList(cmd.Context(), namespace) if err != nil { cmd.PrintErrf("Failed to list inferences: %v\n", err) return err } sort.Sort(byName(infs)) if listQuiet { for _, inf := range infs { cmd.Printf("%s\n", inf.Spec.Name) } return nil } else if listVerbose { t := table.NewWriter() t.SetStyle(table.Style{ Box: table.StyleBoxDefault, Color: table.ColorOptionsDefault, Format: table.FormatOptionsDefault, HTML: table.DefaultHTMLOptions, Options: table.OptionsNoBordersAndSeparators, Title: table.TitleOptionsDefault, }) t.AppendHeader(table.Row{"Name", "Endpoint", "Image", "Status", "Invocations", "Replicas", "CreatedAt"}) for _, inf := range infs { functionImage := inf.Spec.Image createdAt := "" if inf.Status.CreatedAt != nil { createdAt = inf.Status.CreatedAt.String() } t.AppendRow(table.Row{ inf.Spec.Name, getEndpoint(inf), functionImage, inf.Status.Phase, int64(inf.Status.InvocationCount), fmt.Sprintf("%d/%d", inf.Status.AvailableReplicas, inf.Status.Replicas), createdAt, }) } cmd.Println(t.Render()) } else { t := table.NewWriter() t.SetStyle(table.Style{ Box: table.StyleBoxDefault, Color: table.ColorOptionsDefault, Format: table.FormatOptionsDefault, HTML: table.DefaultHTMLOptions, Options: table.OptionsNoBordersAndSeparators, Title: table.TitleOptionsDefault, }) t.AppendHeader(table.Row{"Name", "Endpoint", "Status", "Invocations", "Replicas"}) for _, inf := range infs { t.AppendRow(table.Row{ inf.Spec.Name, getEndpoint(inf), inf.Status.Phase, int64(inf.Status.InvocationCount), fmt.Sprintf("%d/%d", inf.Status.AvailableReplicas, inf.Status.Replicas), }) } cmd.Println(t.Render()) } return nil } type byName []types.InferenceDeployment func (a byName) Len() int { return len(a) } func (a byName) Swap(i, j int) { a[i], a[j] = a[j], a[i] } func (a byName) Less(i, j int) bool { return a[i].Spec.Name < a[j].Spec.Name } func getEndpoint(inf types.InferenceDeployment) string { endpoint := fmt.Sprintf("%s/inference/%s.%s", mdzURL, inf.Spec.Name, inf.Spec.Namespace) if d, ok := inf.Spec.Annotations[annotationDomain]; ok { // Replace https with http now. rawHTTPDomain := strings.Replace(d, "https://", "http://", 1) endpoint = fmt.Sprintf("%s\n%s", rawHTTPDomain, endpoint) } return endpoint } ================================================ FILE: mdz/pkg/cmd/list_instance.go ================================================ package cmd import ( "sort" "github.com/jedib0t/go-pretty/v6/table" "github.com/spf13/cobra" "github.com/tensorchord/openmodelz/agent/api/types" ) var ( // Used for flags. listInstanceQuiet bool listInstanceVerbose bool ) // listInstanceCmd represents the list instance command var listInstanceCmd = &cobra.Command{ Use: "instance", Short: "List all instances for the given deployment", Long: `List all instances for the given deployment`, Example: ` mdz list instance bloomz-560m mdz list instance bloomz-560m -v mdz list instance bloomz-560m -q`, Args: cobra.ExactArgs(1), PreRunE: commandInit, RunE: commandListInstance, } func init() { listCommand.AddCommand(listInstanceCmd) // Here you will define your flags and configuration settings. // Cobra supports Persistent Flags which will work for this command // and all subcommands, e.g.: // Cobra supports local flags which will only run when this command // is called directly, e.g.: listInstanceCmd.Flags().BoolVarP(&listInstanceQuiet, "quiet", "q", false, "Quiet mode - print out only the instance names") listInstanceCmd.Flags().BoolVarP(&listInstanceVerbose, "verbose", "v", false, "Verbose mode - print out all instance details") } func commandListInstance(cmd *cobra.Command, args []string) error { instances, err := agentClient.InstanceList(cmd.Context(), namespace, args[0]) if err != nil { cmd.PrintErrf("Failed to list inference instances: %v\n", err) return err } sort.Sort(byInstanceName(instances)) if listInstanceQuiet { for _, i := range instances { cmd.Printf("%s\n", i.Spec.Name) } return nil } else if listInstanceVerbose { t := table.NewWriter() t.SetStyle(table.Style{ Box: table.StyleBoxDefault, Color: table.ColorOptionsDefault, Format: table.FormatOptionsDefault, HTML: table.DefaultHTMLOptions, Options: table.OptionsNoBordersAndSeparators, Title: table.TitleOptionsDefault, }) t.AppendHeader(table.Row{"Name", "Status", "Reason", "Message", "CreatedAt"}) for _, i := range instances { t.AppendRow(table.Row{i.Spec.Name, i.Status.Phase, i.Status.Reason, i.Status.Message, i.Status.StartTime}) } cmd.Println(t.Render()) return nil } else { t := table.NewWriter() t.SetStyle(table.Style{ Box: table.StyleBoxDefault, Color: table.ColorOptionsDefault, Format: table.FormatOptionsDefault, HTML: table.DefaultHTMLOptions, Options: table.OptionsNoBordersAndSeparators, Title: table.TitleOptionsDefault, }) t.AppendHeader(table.Row{"Name", "Status", "CreatedAt"}) for _, i := range instances { t.AppendRow(table.Row{i.Spec.Name, i.Status.Phase, i.Status.StartTime}) } cmd.Println(t.Render()) return nil } } type byInstanceName []types.InferenceDeploymentInstance func (a byInstanceName) Len() int { return len(a) } func (a byInstanceName) Swap(i, j int) { a[i], a[j] = a[j], a[i] } func (a byInstanceName) Less(i, j int) bool { return a[i].Spec.Name < a[j].Spec.Name } ================================================ FILE: mdz/pkg/cmd/localagent.go ================================================ package cmd import ( "github.com/spf13/cobra" "github.com/tensorchord/openmodelz/mdz/pkg/agentd/server" ) var ( localAgentPort int ) // localAgentCmd represents the local-agent command var localAgentCmd = &cobra.Command{ Use: "local-agent", Short: "Start agent with local docker runtime", Long: `Start agent with local docker runtime`, Example: ` mdz local-agent`, GroupID: "basic", PreRunE: commandInit, RunE: commandLocalAgent, Hidden: true, } func init() { rootCmd.AddCommand(localAgentCmd) // Here you will define your flags and configuration settings. // Cobra supports Persistent Flags which will work for this command // and all subcommands, e.g.: // Cobra supports local flags which will only run when this command // is called directly, e.g.: localAgentCmd.Flags().IntVarP(&localAgentPort, "port", "p", 31112, "Port to listen on") } func commandLocalAgent(cmd *cobra.Command, args []string) error { server, err := server.New() if err != nil { return err } return server.Run(localAgentPort) } ================================================ FILE: mdz/pkg/cmd/logs.go ================================================ package cmd import ( "github.com/spf13/cobra" ) var ( // Used for flags. tail int since string end string follow bool ) // logCmd represents the log command var logsCmd = &cobra.Command{ Use: "logs", Short: "Print the logs for a deployment", Long: `Print the logs for a deployment`, Example: ` mdz logs blomdz-560m`, GroupID: "debug", PreRunE: commandInit, Args: cobra.ExactArgs(1), RunE: commandLogs, } func init() { rootCmd.AddCommand(logsCmd) // Here you will define your flags and configuration settings. // Cobra supports Persistent Flags which will work for this command // and all subcommands, e.g.: // Cobra supports local flags which will only run when this command // is called directly, e.g.: logsCmd.Flags().IntVarP(&tail, "tail", "t", 0, "Number of lines to show from the end of the logs") logsCmd.Flags().StringVarP(&since, "since", "s", "2006-01-02T15:04:05Z", "Show logs since timestamp (e.g. 2013-01-02T13:23:37Z) or relative (e.g. 42m for 42 minutes)") logsCmd.Flags().StringVarP(&end, "end", "e", "", "Only return logs before this timestamp (e.g. 2013-01-02T13:23:37Z) or relative (e.g. 42m for 42 minutes)") logsCmd.Flags().BoolVarP(&follow, "follow", "f", false, "Follow log output") } func commandLogs(cmd *cobra.Command, args []string) error { logStream, err := agentClient.DeploymentLogGet(cmd.Context(), namespace, args[0], since, tail, end, follow) if err != nil { cmd.PrintErrf("Failed to get logs: %s\n", err) return err } for log := range logStream { cmd.Printf("%s: %s\n", log.Instance, log.Text) } return nil } ================================================ FILE: mdz/pkg/cmd/portforward.go ================================================ package cmd import ( "fmt" "net/http" "net/http/httputil" "net/url" "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "github.com/spf13/cobra" ) // portForwardCmd represents the port-forward command var portForwardCmd = &cobra.Command{ Use: "port-forward", Short: "Forward one local port to a deployment", Long: `Forward one local port to a deployment`, Example: ` mdz port-forward blomdz-560m 7860`, GroupID: "debug", PreRunE: commandInit, Args: cobra.ExactArgs(2), RunE: commandForward, } func init() { rootCmd.AddCommand(portForwardCmd) // Here you will define your flags and configuration settings. // Cobra supports Persistent Flags which will work for this command // and all subcommands, e.g.: // Cobra supports local flags which will only run when this command // is called directly, e.g.: } func commandForward(cmd *cobra.Command, args []string) error { name := args[0] port := args[1] if _, err := agentClient.InferenceGet(cmd.Context(), namespace, name); err != nil { cmd.PrintErrf("Failed to get inference: %s\n", errors.Cause(err)) return err } url, err := url.Parse(fmt.Sprintf("%s/inference/%s.%s", mdzURL, name, namespace)) if err != nil { cmd.PrintErrf("Failed to parse URL: %s\n", errors.Cause(err)) return errors.Newf("failed to parse URL: %s\n", errors.Cause(err)) } rp := httputil.NewSingleHostReverseProxy(url) cmd.Printf("Forwarding inference %s to local port %s\n", name, port) logrus.WithField("url", url).Debugf( "Forwarding inference %s to local port %s\n", name, port) handler := func(p *httputil.ReverseProxy) func(http.ResponseWriter, *http.Request) { return func(w http.ResponseWriter, r *http.Request) { cmd.Printf("Handling connection for %s\n", port) p.ServeHTTP(w, r) } } http.HandleFunc("/", handler(rp)) err = http.ListenAndServe(fmt.Sprintf(":%s", port), nil) if err != nil { cmd.PrintErrf("Failed to listen and serve: %s\n", errors.Cause(err)) return errors.Newf("failed to listen and serve: %s", errors.Cause(err)) } return nil } ================================================ FILE: mdz/pkg/cmd/root.go ================================================ package cmd import ( "os" "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/spf13/cobra/doc" "github.com/tensorchord/openmodelz/agent/client" "github.com/tensorchord/openmodelz/mdz/pkg/telemetry" ) var ( // Used for flags. mdzURL string namespace string debug bool disableTelemetry bool agentClient *client.Client ) // rootCmd represents the base command when called without any subcommands var rootCmd = &cobra.Command{ Use: "mdz", Short: "mdz manages your deployments", Long: `mdz helps you deploy applications, manage servers, and troubleshoot issues.`, Example: ` mdz server start mdz deploy --image modelzai/llm-bloomz-560m:23.06.13 --name llm mdz list mdz logs llm mdz port-forward llm 7860 mdz exec llm ps mdz exec llm --tty bash mdz delete llm `, SilenceUsage: true, // Uncomment the following line if your bare application // has an action associated with it: // Run: func(cmd *cobra.Command, args []string) { }, } // Execute adds all child commands to the root command and sets flags appropriately. // This is called by main.main(). It only needs to happen once to the rootCmd. func Execute() { err := rootCmd.Execute() if err != nil { os.Exit(1) } } func init() { // Here you will define your flags and configuration settings. // Cobra supports persistent flags, which, if defined here, // will be global for your application. // rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.mdz.yaml)") rootCmd.PersistentFlags().StringVarP(&mdzURL, "url", "u", "", "URL to use for the server (MDZ_URL) (default http://localhost:80)") rootCmd.PersistentFlags().StringVarP(&namespace, "namespace", "n", "default", "Namespace to use for OpenModelZ inferences") rootCmd.PersistentFlags().MarkHidden("namespace") rootCmd.PersistentFlags().BoolVarP(&debug, "debug", "", false, "Enable debug logging") rootCmd.PersistentFlags().BoolVarP(&disableTelemetry, "disable-telemetry", "", false, "Disable anonymous telemetry") // Cobra also supports local flags, which will only run // when this action is called directly. rootCmd.AddGroup(&cobra.Group{ID: "basic", Title: "Basic Commands:"}) rootCmd.AddGroup(&cobra.Group{ID: "debug", Title: "Troubleshooting and Debugging Commands:"}) rootCmd.AddGroup(&cobra.Group{ID: "management", Title: "Management Commands:"}) // telemetry if err := telemetry.Initialize(!disableTelemetry); err != nil { logrus.WithError(err).Debug("Failed to initialize telemetry") } } func commandInit(cmd *cobra.Command, args []string) error { if err := commandInitLog(cmd, args); err != nil { return err } if agentClient == nil { if mdzURL == "" { // Checkout environment variable MDZ_URL. mdzURL = os.Getenv("MDZ_URL") } if mdzURL == "" { mdzURL = "http://localhost:80" } var err error agentClient, err = client.NewClientWithOpts(client.WithHost(mdzURL)) if err != nil { cmd.PrintErrf("Failed to connect to agent: %s\n", errors.Cause(err)) return err } } return nil } func commandInitLog(cmd *cobra.Command, args []string) error { if debug { logrus.SetLevel(logrus.DebugLevel) logrus.Debug("Debug logging enabled") logrus.SetFormatter(&logrus.TextFormatter{FullTimestamp: true}) } return nil } func GenMarkdownTree(dir string) error { return doc.GenMarkdownTree(rootCmd, dir) } ================================================ FILE: mdz/pkg/cmd/scale.go ================================================ package cmd import ( "github.com/spf13/cobra" "github.com/tensorchord/openmodelz/mdz/pkg/telemetry" ) var ( // Used for flags. replicas int32 min int32 max int32 targetInflightRequests int32 ) // scaleCmd represents the scale command var scaleCmd = &cobra.Command{ Use: "scale", Short: "Scale a deployment", Long: `Scale a deployment`, Example: ` mdz scale bloomz-560m --replicas 3`, GroupID: "basic", PreRunE: commandInit, Args: cobra.ExactArgs(1), RunE: commandScale, } func init() { rootCmd.AddCommand(scaleCmd) // Here you will define your flags and configuration settings. // Cobra supports Persistent Flags which will work for this command // and all subcommands, e.g.: // Cobra supports local flags which will only run when this command // is called directly, e.g.: scaleCmd.Flags().Int32VarP(&replicas, "replicas", "r", 0, "Number of replicas to scale to") scaleCmd.Flags().Int32VarP(&min, "min", "m", 0, "Minimum number of replicas to scale to") scaleCmd.Flags().Int32VarP(&max, "max", "x", 0, "Maximum number of replicas to scale to") scaleCmd.Flags().Int32VarP(&targetInflightRequests, "target-inflight-requests", "t", 0, "Target number of inflight requests per replica") scaleCmd.MarkFlagRequired("replicas") scaleCmd.Flags().MarkHidden("min") scaleCmd.Flags().MarkHidden("max") scaleCmd.Flags().MarkHidden("target-inflight-requests") } func commandScale(cmd *cobra.Command, args []string) error { name := args[0] deployment, err := agentClient.InferenceGet(cmd.Context(), namespace, name) if err != nil { cmd.PrintErrf("Failed to get deployment: %s\n", err) return err } if replicas != 0 { deployment.Spec.Scaling.MinReplicas = int32Ptr(replicas) deployment.Spec.Scaling.MaxReplicas = int32Ptr(replicas) if _, err := agentClient.DeploymentUpdate(cmd.Context(), namespace, deployment); err != nil { cmd.PrintErrf("Failed to update deployment: %s\n", err) return err } return nil } if min != 0 { deployment.Spec.Scaling.MinReplicas = int32Ptr(min) } if max != 0 { deployment.Spec.Scaling.MaxReplicas = int32Ptr(max) } if targetInflightRequests != 0 { deployment.Spec.Scaling.TargetLoad = int32Ptr(targetInflightRequests) } telemetry.GetTelemetry().Record("scale") if _, err := agentClient.DeploymentUpdate(cmd.Context(), namespace, deployment); err != nil { cmd.PrintErrf("Failed to update deployment: %s\n", err) return err } return nil } ================================================ FILE: mdz/pkg/cmd/server.go ================================================ package cmd import ( "time" "github.com/spf13/cobra" ) var ( serverVerbose bool serverPollingInterval time.Duration = 3 * time.Second serverRegistryMirrorName string serverRegistryMirrorEndpoints []string ) // serverCmd represents the server command var serverCmd = &cobra.Command{ Use: "server", Short: "Manage the servers", Long: `Manage the servers`, Example: ` mdz server start`, GroupID: "management", PreRunE: commandInitLog, } func init() { rootCmd.AddCommand(serverCmd) // Here you will define your flags and configuration settings. // Cobra supports Persistent Flags which will work for this command // and all subcommands, e.g.: serverCmd.PersistentFlags().BoolVarP(&serverVerbose, "verbose", "v", false, "Verbose output") // Cobra supports local flags which will only run when this command // is called directly, e.g.: } ================================================ FILE: mdz/pkg/cmd/server_delete.go ================================================ package cmd import "github.com/spf13/cobra" var serverDeleteCmd = &cobra.Command{ Use: "delete", Short: "Delete a node from the cluster", Long: `Delete a node from the cluster`, Example: ` mdz server delete gpu-node-1`, PreRunE: commandInit, Args: cobra.MinimumNArgs(1), RunE: commandServerDelete, } func init() { serverCmd.AddCommand(serverDeleteCmd) } func commandServerDelete(cmd *cobra.Command, args []string) error { nodeName := args[0] if err := agentClient.ServerNodeDelete(cmd.Context(), nodeName); err != nil { return err } return nil } ================================================ FILE: mdz/pkg/cmd/server_destroy.go ================================================ package cmd import ( "github.com/cockroachdb/errors" "github.com/spf13/cobra" "github.com/tensorchord/openmodelz/mdz/pkg/server" ) // serverDestroyCmd represents the server destroy command var serverDestroyCmd = &cobra.Command{ Use: "destroy", Short: "Destroy the cluster", Long: `Destroy the cluster`, Example: ` mdz server destroy`, PreRunE: commandInitLog, RunE: commandServerDestroy, } func init() { serverCmd.AddCommand(serverDestroyCmd) // Here you will define your flags and configuration settings. // Cobra supports Persistent Flags which will work for this command // and all subcommands, e.g.: // Cobra supports local flags which will only run when this command // is called directly, e.g.: } func commandServerDestroy(cmd *cobra.Command, args []string) error { engine, err := server.NewDestroy(server.Options{ Verbose: serverVerbose, OutputStream: cmd.ErrOrStderr(), RetryInternal: serverPollingInterval, }) if err != nil { cmd.PrintErrf("Failed to destroy the server: %s\n", errors.Cause(err)) return err } _, err = engine.Run() if err != nil { cmd.PrintErrf("Failed to destroy the server: %s\n", errors.Cause(err)) return err } cmd.Printf("✅ Server destroyed\n") return nil } ================================================ FILE: mdz/pkg/cmd/server_join.go ================================================ package cmd import ( "github.com/cockroachdb/errors" "github.com/spf13/cobra" "github.com/tensorchord/openmodelz/mdz/pkg/server" "github.com/tensorchord/openmodelz/mdz/pkg/telemetry" ) // serverJoinCmd represents the server join command var serverJoinCmd = &cobra.Command{ Use: "join", Short: "Join to the cluster", Long: `Join to the cluster`, Example: ` mdz server join 192.168.31.192`, PreRunE: commandInitLog, Args: cobra.ExactArgs(1), RunE: commandServerJoin, } func init() { serverCmd.AddCommand(serverJoinCmd) // Here you will define your flags and configuration settings. // Cobra supports Persistent Flags which will work for this command // and all subcommands, e.g.: // Cobra supports local flags which will only run when this command // is called directly, e.g.: serverJoinCmd.Flags().StringVarP(&serverRegistryMirrorName, "mirror-name", "", "docker.io", "Mirror domain name of the registry") serverJoinCmd.Flags().StringArrayVarP(&serverRegistryMirrorEndpoints, "mirror-endpoints", "", []string{}, "Mirror URL endpoints of the registry like `https://quay.io`") } func commandServerJoin(cmd *cobra.Command, args []string) error { engine, err := server.NewJoin(server.Options{ Verbose: serverVerbose, OutputStream: cmd.ErrOrStderr(), RetryInternal: serverPollingInterval, ServerIP: args[0], Mirror: server.Mirror{ Name: serverRegistryMirrorName, Endpoints: serverRegistryMirrorEndpoints, }, }) if err != nil { cmd.PrintErrf("Failed to configure before join: %s\n", errors.Cause(err)) return err } telemetry.GetTelemetry().Record("server join") _, err = engine.Run() if err != nil { cmd.PrintErrf("Failed to join the cluster: %s\n", errors.Cause(err)) return err } cmd.Printf("✅ Server joined\n") return nil } ================================================ FILE: mdz/pkg/cmd/server_label.go ================================================ package cmd import ( "fmt" "strings" "github.com/spf13/cobra" ) // serverLabelCmd represents the server label command var serverLabelCmd = &cobra.Command{ Use: "label", Short: "Update the labels on a server", Long: `Update the labels on a server * A label key and value must begin with a letter or number, and may contain letters, numbers, hyphens, dots, and underscores, up to 63 characters each. * Optionally, the key can begin with a DNS subdomain prefix and a single '/', like example.com/my-app. `, Example: ` mdz server label node-name key=value [key=value...]`, PreRunE: commandInit, Args: cobra.MinimumNArgs(1), RunE: commandServerLabel, } func init() { serverCmd.AddCommand(serverLabelCmd) // Here you will define your flags and configuration settings. // Cobra supports Persistent Flags which will work for this command // and all subcommands, e.g.: // Cobra supports local flags which will only run when this command // is called directly, e.g.: } func commandServerLabel(cmd *cobra.Command, args []string) error { nodeName := args[0] labels := args[1:] nodeLabels, err := parseNodeLabels(labels) if err != nil { return err } if err := agentClient.ServerLabelCreate(cmd.Context(), nodeName, nodeLabels); err != nil { return err } return nil } func parseNodeLabels(labels []string) (map[string]string, error) { res := make(map[string]string) for _, label := range labels { if !strings.Contains(label, "=") { return nil, fmt.Errorf("label must be in the form of key=value") } // Split the label into key and value parts := strings.SplitN(label, "=", 2) key := parts[0] value := parts[1] if len(key) == 0 { return nil, fmt.Errorf("label key cannot be empty") } res[key] = value } return res, nil } ================================================ FILE: mdz/pkg/cmd/server_list.go ================================================ package cmd import ( "fmt" "math" "github.com/cockroachdb/errors" "github.com/jedib0t/go-pretty/v6/table" "github.com/sirupsen/logrus" "github.com/spf13/cobra" "k8s.io/apimachinery/pkg/api/resource" "github.com/tensorchord/openmodelz/agent/api/types" "github.com/tensorchord/openmodelz/mdz/pkg/telemetry" ) var ( // Used for flags. serverListQuiet bool serverListVerbose bool ) // serverListCmd represents the server list command var serverListCmd = &cobra.Command{ Use: "list", Short: "List all servers in the cluster", Long: `List all servers in the cluster`, Example: ` mdz server list`, PreRunE: commandInit, RunE: commandServerList, } func init() { serverCmd.AddCommand(serverListCmd) // Here you will define your flags and configuration settings. // Cobra supports Persistent Flags which will work for this command // and all subcommands, e.g.: // Cobra supports local flags which will only run when this command // is called directly, e.g.: serverListCmd.Flags().BoolVarP(&serverListQuiet, "quiet", "q", false, "Quiet mode - print out only the server names") serverListCmd.Flags().BoolVarP(&serverListVerbose, "verbose", "v", false, "Verbose mode - print out all server details") } func commandServerList(cmd *cobra.Command, args []string) error { telemetry.GetTelemetry().Record("server list") servers, err := agentClient.ServerList(cmd.Context()) if err != nil { cmd.PrintErrf("Failed to list servers: %s\n", errors.Cause(err)) return err } if serverListQuiet { for _, server := range servers { cmd.Printf("%s\n", server.Spec.Name) } } else if serverListVerbose { t := table.NewWriter() t.SetStyle(table.Style{ Box: table.StyleBoxDefault, Color: table.ColorOptionsDefault, Format: table.FormatOptionsDefault, HTML: table.DefaultHTMLOptions, Options: table.OptionsNoBordersAndSeparators, Title: table.TitleOptionsDefault, }) t.AppendHeader(table.Row{"Name", "Phase", "Allocatable", "Capacity", "Distribution", "OS", "Kernel", "Labels"}) for _, server := range servers { t.AppendRow(table.Row{server.Spec.Name, server.Status.Phase, resourceListString(server.Status.Allocatable), resourceListString(server.Status.Capacity), server.Status.System.OSImage, server.Status.System.OperatingSystem, server.Status.System.KernelVersion, labelsString(server.Spec.Labels), }) } cmd.Println(t.Render()) } else { t := table.NewWriter() t.SetStyle(table.Style{ Box: table.StyleBoxDefault, Color: table.ColorOptionsDefault, Format: table.FormatOptionsDefault, HTML: table.DefaultHTMLOptions, Options: table.OptionsNoBordersAndSeparators, Title: table.TitleOptionsDefault, }) t.AppendHeader(table.Row{"Name", "Phase", "Allocatable", "Capacity"}) for _, server := range servers { t.AppendRow(table.Row{server.Spec.Name, server.Status.Phase, resourceListString(server.Status.Allocatable), resourceListString(server.Status.Capacity)}) } cmd.Println(t.Render()) } return nil } func labelsString(labels map[string]string) string { res := "" for k, v := range labels { res += fmt.Sprintf("%s=%s\n", k, v) } if len(res) == 0 { return res } return res[:len(res)-1] } func prettyByteSize(quantity string) (string, error) { r, err := resource.ParseQuantity(quantity) if err != nil { return "", err } bf := float64(r.Value()) for _, unit := range []string{"", "Ki", "Mi", "Gi", "Ti"} { if math.Abs(bf) < 1024.0 { return fmt.Sprintf("%3.1f%sB", bf, unit), nil } bf /= 1024.0 } return fmt.Sprintf("%.1fPiB", bf), nil } func resourceListString(l types.ResourceList) string { res := fmt.Sprintf("cpu: %s", l[types.ResourceCPU]) memory, ok := l[types.ResourceMemory] if ok { prettyMem, err := prettyByteSize(string(memory)) if err != nil { logrus.Infof("failed to parse the memory quantity: %s", memory) } else { memory = types.Quantity(prettyMem) } } res += fmt.Sprintf("\nmemory: %s", memory) if l[types.ResourceGPU] != "" { res += fmt.Sprintf("\ngpu: %s", l[types.ResourceGPU]) } return res } ================================================ FILE: mdz/pkg/cmd/server_start.go ================================================ package cmd import ( "fmt" "time" "github.com/cockroachdb/errors" "github.com/spf13/cobra" "github.com/tensorchord/openmodelz/agent/pkg/consts" "github.com/tensorchord/openmodelz/mdz/pkg/server" "github.com/tensorchord/openmodelz/mdz/pkg/telemetry" "github.com/tensorchord/openmodelz/mdz/pkg/version" ) var ( serverStartRuntime string serverStartDomain string = consts.Domain serverStartVersion string serverStartWithGPU bool enableModelZCloud bool modelzCloudUrl string modelzCloudAgentToken string modelzCloudRegion string ) // serverStartCmd represents the server start command var serverStartCmd = &cobra.Command{ Use: "start", Short: "Start the server", Long: `Start the server with the public IP of the machine. If not provided, the internal IP will be used automatically.`, Example: ` mdz server start mdz server start -v mdz server start 1.2.3.4`, PreRunE: preRunE, Args: cobra.RangeArgs(0, 1), RunE: commandServerStart, } func init() { serverCmd.AddCommand(serverStartCmd) // Here you will define your flags and configuration settings. // Cobra supports Persistent Flags which will work for this command // and all subcommands, e.g.: // Cobra supports local flags which will only run when this command // is called directly, e.g.: // serverStartCmd.Flags().StringVarP(&serverStartRuntime, "runtime", "r", "k3s", "Runtime to use (k3s, docker) in the started server") serverStartCmd.Flags().StringVarP(&serverStartVersion, "version", "", version.HelmChartVersion, "Version of the server to start") serverStartCmd.Flags().MarkHidden("version") serverStartCmd.Flags().BoolVarP(&serverStartWithGPU, "force-gpu", "g", false, "Start the server with GPU support (ignore the GPU detection)") serverStartCmd.Flags().StringVarP(&serverRegistryMirrorName, "mirror-name", "", "docker.io", "Mirror domain name of the registry") serverStartCmd.Flags().StringArrayVarP(&serverRegistryMirrorEndpoints, "mirror-endpoints", "", []string{}, "Mirror URL endpoints of the registry like `https://quay.io`") serverStartCmd.Flags().BoolVarP(&enableModelZCloud, "modelzcloud-enabled", "", false, "Enable ModelZ Cloud Management") serverStartCmd.Flags().StringVarP(&modelzCloudUrl, "modelzcloud-url", "", "https://cloud.modelz.ai", "ModelZ Cloud URL") serverStartCmd.Flags().StringVarP(&modelzCloudAgentToken, "modelzcloud-agent-token", "", "", "ModelZ Cloud Agent Token") serverStartCmd.Flags().StringVarP(&modelzCloudRegion, "modelzcloud-region", "", "on-premises", "ModelZ Cloud Region") } func preRunE(cmd *cobra.Command, args []string) error { err := commandInitLog(cmd, args) if err != nil { return err } // If enabled modelzcloud control plane, you need make configuration if enableModelZCloud { if modelzCloudUrl == "" || modelzCloudAgentToken == "" || modelzCloudRegion == "" { return fmt.Errorf("modelzcloud configuration is not complete") } } return nil } func commandServerStart(cmd *cobra.Command, args []string) error { var domain *string if len(args) > 0 { domainWithSuffix := fmt.Sprintf("%s.%s", args[0], serverStartDomain) domain = &domainWithSuffix } defer func(start time.Time) { telemetry.GetTelemetry().Record( "server start", telemetry.AddField("duration", time.Since(start).Seconds()), ) }(time.Now()) engine, err := server.NewStart(server.Options{ Verbose: serverVerbose, Runtime: server.Runtime(serverStartRuntime), OutputStream: cmd.ErrOrStderr(), RetryInternal: serverPollingInterval, Domain: domain, Version: serverStartVersion, ForceGPU: serverStartWithGPU, Mirror: server.Mirror{ Name: serverRegistryMirrorName, Endpoints: serverRegistryMirrorEndpoints, }, ModelZCloud: server.ModelZCloud{ Enabled: enableModelZCloud, URL: modelzCloudUrl, Token: modelzCloudAgentToken, Region: modelzCloudRegion, }, }) if err != nil { cmd.PrintErrf("Failed to start the server: %s\n", errors.Cause(err)) return err } result, err := engine.Run() if err != nil { cmd.PrintErrf("Failed to start the server: %s\n", errors.Cause(err)) return err } mdzURL = result.MDZURL if err := commandInit(cmd, args); err != nil { cmd.PrintErrf("Failed to start the server: %s\n", errors.Cause(err)) return err } cmd.Printf("🐋 Checking if the server is running...\n") // Retry until verify success. ticker := time.NewTicker(serverPollingInterval) for range ticker.C { if err := printServerVersion(cmd); err != nil { cmd.Printf("🐋 The server is not ready yet, retrying...\n") continue } break } cmd.Printf("🐳 The server is running at %s\n", mdzURL) cmd.Printf("🎉 You could set the environment variable to get started!\n\n") cmd.Printf("export MDZ_URL=%s\n", mdzURL) return nil } ================================================ FILE: mdz/pkg/cmd/server_stop.go ================================================ package cmd import ( "github.com/cockroachdb/errors" "github.com/spf13/cobra" "github.com/tensorchord/openmodelz/mdz/pkg/server" ) // serverStopCmd represents the server stop command var serverStopCmd = &cobra.Command{ Use: "stop", Short: "Stop the server", Long: `Stop the server`, Example: ` mdz server stop`, PreRunE: commandInitLog, RunE: commandServerStop, } func init() { serverCmd.AddCommand(serverStopCmd) // Here you will define your flags and configuration settings. // Cobra supports Persistent Flags which will work for this command // and all subcommands, e.g.: // Cobra supports local flags which will only run when this command // is called directly, e.g.: } func commandServerStop(cmd *cobra.Command, args []string) error { engine, err := server.NewStop(server.Options{ Verbose: serverVerbose, OutputStream: cmd.ErrOrStderr(), RetryInternal: serverPollingInterval, }) if err != nil { cmd.PrintErrf("Failed to stop the server: %s\n", errors.Cause(err)) return err } _, err = engine.Run() if err != nil { cmd.PrintErrf("Failed to stop the server: %s\n", errors.Cause(err)) return err } cmd.Printf("✅ Server stopped\n") return nil } ================================================ FILE: mdz/pkg/cmd/streams/in.go ================================================ package streams import ( "errors" "io" "os" "runtime" "github.com/moby/term" ) // In is an input stream to read user input. It implements [io.ReadCloser] // with additional utilities, such as putting the terminal in raw mode. type In struct { commonStream in io.ReadCloser } // Read implements the [io.Reader] interface. func (i *In) Read(p []byte) (int, error) { return i.in.Read(p) } // Close implements the [io.Closer] interface. func (i *In) Close() error { return i.in.Close() } // SetRawTerminal sets raw mode on the input terminal. It is a no-op if In // is not a TTY, or if the "NORAW" environment variable is set to a non-empty // value. func (i *In) SetRawTerminal() (err error) { if !i.isTerminal || os.Getenv("NORAW") != "" { return nil } i.state, err = term.SetRawTerminal(i.fd) return err } // CheckTty checks if we are trying to attach to a container TTY // from a non-TTY client input stream, and if so, returns an error. func (i *In) CheckTty(attachStdin, ttyMode bool) error { // In order to attach to a container tty, input stream for the client must // be a tty itself: redirecting or piping the client standard input is // incompatible with `docker run -t`, `docker exec -t` or `docker attach`. if ttyMode && attachStdin && !i.isTerminal { const eText = "the input device is not a TTY" if runtime.GOOS == "windows" { return errors.New(eText + ". If you are using mintty, try prefixing the command with 'winpty'") } return errors.New(eText) } return nil } // NewIn returns a new [In] from an [io.ReadCloser]. func NewIn(in io.ReadCloser) *In { i := &In{in: in} i.fd, i.isTerminal = term.GetFdInfo(in) return i } ================================================ FILE: mdz/pkg/cmd/streams/out.go ================================================ package streams import ( "io" "os" "github.com/moby/term" "github.com/sirupsen/logrus" ) // Out is an output stream to write normal program output. It implements // an [io.Writer], with additional utilities for detecting whether a terminal // is connected, getting the TTY size, and putting the terminal in raw mode. type Out struct { commonStream out io.Writer } func (o *Out) Write(p []byte) (int, error) { return o.out.Write(p) } // SetRawTerminal puts the output of the terminal connected to the stream // into raw mode. // // On UNIX, this does nothing. On Windows, it disables LF -> CRLF/ translation. // It is a no-op if Out is not a TTY, or if the "NORAW" environment variable is // set to a non-empty value. func (o *Out) SetRawTerminal() (err error) { if !o.isTerminal || os.Getenv("NORAW") != "" { return nil } o.state, err = term.SetRawTerminalOutput(o.fd) return err } // GetTtySize returns the height and width in characters of the TTY, or // zero for both if no TTY is connected. func (o *Out) GetTtySize() (height uint, width uint) { if !o.isTerminal { return 0, 0 } ws, err := term.GetWinsize(o.fd) if err != nil { logrus.WithError(err).Debug("Error getting TTY size") if ws == nil { return 0, 0 } } return uint(ws.Height), uint(ws.Width) } // NewOut returns a new [Out] from an [io.Writer]. func NewOut(out io.Writer) *Out { o := &Out{out: out} o.fd, o.isTerminal = term.GetFdInfo(out) return o } ================================================ FILE: mdz/pkg/cmd/streams/stream.go ================================================ package streams import ( "github.com/moby/term" ) type commonStream struct { fd uintptr isTerminal bool state *term.State } // FD returns the file descriptor number for this stream. func (s *commonStream) FD() uintptr { return s.fd } // IsTerminal returns true if this stream is connected to a terminal. func (s *commonStream) IsTerminal() bool { return s.isTerminal } // RestoreTerminal restores normal mode to the terminal. func (s *commonStream) RestoreTerminal() { if s.state != nil { _ = term.RestoreTerminal(s.fd, s.state) } } // SetIsTerminal overrides whether a terminal is connected. It is used to // override this property in unit-tests, and should not be depended on for // other purposes. func (s *commonStream) SetIsTerminal(isTerminal bool) { s.isTerminal = isTerminal } ================================================ FILE: mdz/pkg/cmd/version.go ================================================ package cmd import ( "github.com/cockroachdb/errors" "github.com/spf13/cobra" "github.com/tensorchord/openmodelz/mdz/pkg/version" ) // versionCmd represents the versionCmd var versionCmd = &cobra.Command{ Use: "version", Short: "Print the client and agent version information", Long: `Print the client and server version information`, Example: ` mdz version`, PreRunE: commandInit, RunE: commandVersion, } func init() { rootCmd.AddCommand(versionCmd) // Here you will define your flags and configuration settings. // Cobra supports Persistent Flags which will work for this command // and all subcommands, e.g.: // Cobra supports local flags which will only run when this command // is called directly, e.g.: } func commandVersion(cmd *cobra.Command, args []string) error { v := version.GetVersion() cmd.Println("Client:") cmd.Printf(" Version: \t%s\n", v.Version) cmd.Printf(" Build Date: \t%s\n", v.BuildDate) cmd.Printf(" Git Commit: \t%s\n", v.GitCommit) cmd.Printf(" Git State: \t%s\n", v.GitTreeState) cmd.Printf(" Go Version: \t%s\n", v.GoVersion) cmd.Printf(" Compiler: \t%s\n", v.Compiler) cmd.Printf(" Platform: \t%s\n", v.Platform) if err := printServerVersion(cmd); err != nil { cmd.PrintErrf("Failed to get server version: %v\n", errors.Cause(err)) return err } return nil } func printServerVersion(cmd *cobra.Command) error { info, err := agentClient.InfoGet(cmd.Context()) if err != nil { return err } cmd.Println("Server:") cmd.Printf(" Version: \t%s\n", info.Version.Version) cmd.Printf(" Build Date: \t%s\n", info.Version.BuildDate) cmd.Printf(" Git Commit: \t%s\n", info.Version.GitCommit) cmd.Printf(" Git State: \t%s\n", info.Version.GitTreeState) cmd.Printf(" Go Version: \t%s\n", info.Version.GoVersion) cmd.Printf(" Compiler: \t%s\n", info.Version.Compiler) cmd.Printf(" Platform: \t%s\n", info.Version.Platform) return nil } ================================================ FILE: mdz/pkg/server/agentd_run.go ================================================ package server import ( "fmt" "os/exec" "syscall" ) type agentDRunStep struct { options Options } // TODO(gaocegege): There is still a bug, thus it cannot be used actually. // The process will exit after the command returns. We need to put it in systemd. func (s *agentDRunStep) Run() error { fmt.Fprintf(s.options.OutputStream, "🚧 Running the agent for docker runtime...\n") cmd := exec.Command("/bin/sh", "-c", "mdz local-agent &") cmd.SysProcAttr = &syscall.SysProcAttr{ Pdeathsig: syscall.SIGKILL, } if s.options.Verbose { cmd.Stderr = s.options.OutputStream cmd.Stdout = s.options.OutputStream } else { cmd.Stdout = nil cmd.Stderr = nil } err := cmd.Run() if err != nil { return err } return nil } func (s *agentDRunStep) Verify() error { return nil } ================================================ FILE: mdz/pkg/server/engine.go ================================================ package server import ( "fmt" "io" "time" ) const ( AgentPort = 31112 ) type Options struct { Verbose bool OutputStream io.Writer Runtime Runtime Mirror Mirror RetryInternal time.Duration ServerIP string Domain *string Version string ForceGPU bool ModelZCloud ModelZCloud } type ModelZCloud struct { Enabled bool URL string Token string Region string } type Mirror struct { Name string Endpoints []string } func (m *Mirror) Configured() bool { return m.Name != "" && len(m.Endpoints) > 0 } type Runtime string var ( RuntimeK3s Runtime = "k3s" RuntimeDocker Runtime = "docker" ) type Engine struct { options Options Steps []Step } type Result struct { MDZURL string } func NewStart(o Options) (*Engine, error) { if o.Verbose { fmt.Fprintf(o.OutputStream, "Starting the server with config: %+v\n", o) } var engine *Engine switch o.Runtime { case RuntimeDocker: engine = &Engine{ options: o, Steps: []Step{ &agentDRunStep{ options: o, }, }, } default: engine = &Engine{ options: o, Steps: []Step{ // Install k3s and related tools. &k3sPrepare{ options: o, }, &k3sInstallStep{ options: o, }, &nginxInstallStep{ options: o, }, &gpuInstallStep{ options: o, }, &openModelZInstallStep{ options: o, }, }, } } return engine, nil } func NewStop(o Options) (*Engine, error) { return &Engine{ options: o, Steps: []Step{ // Kill all k3s and related tools. &k3sKillAllStep{ options: o, }, }, }, nil } func NewDestroy(o Options) (*Engine, error) { return &Engine{ options: o, Steps: []Step{ // Destroy all k3s and related tools. &k3sDestroyAllStep{ options: o, }, }, }, nil } func NewJoin(o Options) (*Engine, error) { return &Engine{ options: o, Steps: []Step{ &k3sJoinStep{ options: o, }, }, }, nil } type Step interface { Run() error Verify() error } func (e *Engine) Run() (*Result, error) { for _, step := range e.Steps { if err := step.Run(); err != nil { return nil, err } // Retry until verify success. ticker := time.NewTicker(e.options.RetryInternal) for range ticker.C { if err := step.Verify(); err == nil { ticker.Stop() break } } } if e.options.Domain != nil { return &Result{ MDZURL: fmt.Sprintf("http://%s", *e.options.Domain), }, nil } // Get the server IP. if resultDomain != "" { return &Result{ MDZURL: fmt.Sprintf("http://%s", resultDomain), }, nil } return &Result{ MDZURL: fmt.Sprintf("http://0.0.0.0:%d", AgentPort), }, nil } ================================================ FILE: mdz/pkg/server/gpu-resource.yaml ================================================ apiVersion: v1 kind: Namespace metadata: name: gpu-operator --- apiVersion: helm.cattle.io/v1 kind: HelmChart metadata: name: nvidia namespace: gpu-operator spec: chart: gpu-operator repo: https://helm.ngc.nvidia.com/nvidia targetNamespace: gpu-operator set: valuesContent: |- toolkit: env: - name: CONTAINERD_CONFIG value: /var/lib/rancher/k3s/agent/etc/containerd/config.toml - name: CONTAINERD_SOCKET value: /run/k3s/containerd/containerd.sock ================================================ FILE: mdz/pkg/server/gpu_install.go ================================================ package server import ( _ "embed" "fmt" "io" "os" "os/exec" "path/filepath" "regexp" "syscall" ) //go:embed gpu-resource.yaml var gpuYamlContent string // gpuInstallStep installs the GPU related resources. type gpuInstallStep struct { options Options } // check if the Nvidia Toolkit is installed on the host func (s *gpuInstallStep) hasNvidiaToolkit() bool { locations := []string{ "/usr/local/nvidia/toolkit", "/usr/bin", } binaryNames := []string{ "nvidia-container-runtime", "nvidia-container-runtime-experimental", } for _, location := range locations { for _, name := range binaryNames { path := filepath.Join(location, name) if _, err := os.Stat(path); err == nil { return true } } } return false } func (s *gpuInstallStep) hasNvidiaDevice() bool { output, err := exec.Command("/bin/sh", "-c", "lspci").Output() if err != nil { return false } regexNvidia := regexp.MustCompile("(?i)nvidia") return regexNvidia.Match(output) } func (s *gpuInstallStep) Run() error { if !s.options.ForceGPU { // detect GPU if !(s.hasNvidiaDevice() || s.hasNvidiaToolkit()) { fmt.Fprintf(s.options.OutputStream, "🚧 Nvidia Toolkit is missing, skip the GPU initialization.\n") return nil } } fmt.Fprintf(s.options.OutputStream, "🚧 Initializing the GPU resource...\n") cmd := exec.Command("/bin/sh", "-c", "sudo k3s kubectl apply -f -") cmd.SysProcAttr = &syscall.SysProcAttr{ Pdeathsig: syscall.SIGKILL, } stdin, err := cmd.StdinPipe() if err != nil { return err } if s.options.Verbose { cmd.Stderr = s.options.OutputStream cmd.Stdout = s.options.OutputStream } else { cmd.Stdout = nil cmd.Stderr = nil } if err := cmd.Start(); err != nil { return err } if _, err := io.WriteString(stdin, gpuYamlContent); err != nil { return err } // Close the input stream to finish the pipe. Then the command will use the // input from the pipe to start the next process. stdin.Close() if err := cmd.Wait(); err != nil { return err } return nil } func (s *gpuInstallStep) Verify() error { return nil } ================================================ FILE: mdz/pkg/server/k3s-install.sh ================================================ #!/bin/sh set -e set -o noglob # Usage: # curl ... | ENV_VAR=... sh - # or # ENV_VAR=... ./install.sh # # Example: # Installing a server without traefik: # curl ... | INSTALL_K3S_EXEC="--disable=traefik" sh - # Installing an agent to point at a server: # curl ... | K3S_TOKEN=xxx K3S_URL=https://server-url:6443 sh - # # Environment variables: # - K3S_* # Environment variables which begin with K3S_ will be preserved for the # systemd service to use. Setting K3S_URL without explicitly setting # a systemd exec command will default the command to "agent", and we # enforce that K3S_TOKEN is also set. # # - INSTALL_K3S_SKIP_DOWNLOAD # If set to true will not download k3s hash or binary. # # - INSTALL_K3S_FORCE_RESTART # If set to true will always restart the K3s service # # - INSTALL_K3S_SYMLINK # If set to 'skip' will not create symlinks, 'force' will overwrite, # default will symlink if command does not exist in path. # # - INSTALL_K3S_SKIP_ENABLE # If set to true will not enable or start k3s service. # # - INSTALL_K3S_SKIP_START # If set to true will not start k3s service. # # - INSTALL_K3S_VERSION # Version of k3s to download from github. Will attempt to download from the # stable channel if not specified. # # - INSTALL_K3S_COMMIT # Commit of k3s to download from temporary cloud storage. # * (for developer & QA use) # # - INSTALL_K3S_BIN_DIR # Directory to install k3s binary, links, and uninstall script to, or use # /usr/local/bin as the default # # - INSTALL_K3S_BIN_DIR_READ_ONLY # If set to true will not write files to INSTALL_K3S_BIN_DIR, forces # setting INSTALL_K3S_SKIP_DOWNLOAD=true # # - INSTALL_K3S_SYSTEMD_DIR # Directory to install systemd service and environment files to, or use # /etc/systemd/system as the default # # - INSTALL_K3S_EXEC or script arguments # Command with flags to use for launching k3s in the systemd service, if # the command is not specified will default to "agent" if K3S_URL is set # or "server" if not. The final systemd command resolves to a combination # of EXEC and script args ($@). # # The following commands result in the same behavior: # curl ... | INSTALL_K3S_EXEC="--disable=traefik" sh -s - # curl ... | INSTALL_K3S_EXEC="server --disable=traefik" sh -s - # curl ... | INSTALL_K3S_EXEC="server" sh -s - --disable=traefik # curl ... | sh -s - server --disable=traefik # curl ... | sh -s - --disable=traefik # # - INSTALL_K3S_NAME # Name of systemd service to create, will default from the k3s exec command # if not specified. If specified the name will be prefixed with 'k3s-'. # # - INSTALL_K3S_TYPE # Type of systemd service to create, will default from the k3s exec command # if not specified. # # - INSTALL_K3S_SELINUX_WARN # If set to true will continue if k3s-selinux policy is not found. # # - INSTALL_K3S_SKIP_SELINUX_RPM # If set to true will skip automatic installation of the k3s RPM. # # - INSTALL_K3S_CHANNEL_URL # Channel URL for fetching k3s download URL. # Defaults to 'https://update.k3s.io/v1-release/channels'. # # - INSTALL_K3S_CHANNEL # Channel to use for fetching k3s download URL. # Defaults to 'stable'. GITHUB_URL=https://github.com/k3s-io/k3s/releases STORAGE_URL=https://k3s-ci-builds.s3.amazonaws.com DOWNLOADER= # --- helper functions for logs --- info() { echo '[INFO] ' "$@" } warn() { echo '[WARN] ' "$@" >&2 } fatal() { echo '[ERROR] ' "$@" >&2 exit 1 } # --- fatal if no systemd or openrc --- verify_system() { if [ -x /sbin/openrc-run ]; then HAS_OPENRC=true return fi if [ -x /bin/systemctl ] || type systemctl > /dev/null 2>&1; then HAS_SYSTEMD=true return fi fatal 'Can not find systemd or openrc to use as a process supervisor for k3s' } # --- add quotes to command arguments --- quote() { for arg in "$@"; do printf '%s\n' "$arg" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/" done } # --- add indentation and trailing slash to quoted args --- quote_indent() { printf ' \\\n' for arg in "$@"; do printf '\t%s \\\n' "$(quote "$arg")" done } # --- escape most punctuation characters, except quotes, forward slash, and space --- escape() { printf '%s' "$@" | sed -e 's/\([][!#$%&()*;<=>?\_`{|}]\)/\\\1/g;' } # --- escape double quotes --- escape_dq() { printf '%s' "$@" | sed -e 's/"/\\"/g' } # --- ensures $K3S_URL is empty or begins with https://, exiting fatally otherwise --- verify_k3s_url() { case "${K3S_URL}" in "") ;; https://*) ;; *) fatal "Only https:// URLs are supported for K3S_URL (have ${K3S_URL})" ;; esac } # --- define needed environment variables --- setup_env() { # --- use command args if passed or create default --- case "$1" in # --- if we only have flags discover if command should be server or agent --- (-*|"") if [ -z "${K3S_URL}" ]; then CMD_K3S=server else if [ -z "${K3S_TOKEN}" ] && [ -z "${K3S_TOKEN_FILE}" ]; then fatal "Defaulted k3s exec command to 'agent' because K3S_URL is defined, but K3S_TOKEN or K3S_TOKEN_FILE is not defined." fi CMD_K3S=agent fi ;; # --- command is provided --- (*) CMD_K3S=$1 shift ;; esac verify_k3s_url CMD_K3S_EXEC="${CMD_K3S}$(quote_indent "$@")" # --- use systemd name if defined or create default --- if [ -n "${INSTALL_K3S_NAME}" ]; then SYSTEM_NAME=k3s-${INSTALL_K3S_NAME} else if [ "${CMD_K3S}" = server ]; then SYSTEM_NAME=k3s else SYSTEM_NAME=k3s-${CMD_K3S} fi fi # --- check for invalid characters in system name --- valid_chars=$(printf '%s' "${SYSTEM_NAME}" | sed -e 's/[][!#$%&()*;<=>?\_`{|}/[:space:]]/^/g;' ) if [ "${SYSTEM_NAME}" != "${valid_chars}" ]; then invalid_chars=$(printf '%s' "${valid_chars}" | sed -e 's/[^^]/ /g') fatal "Invalid characters for system name: ${SYSTEM_NAME} ${invalid_chars}" fi # --- use sudo if we are not already root --- SUDO=sudo if [ $(id -u) -eq 0 ]; then SUDO= fi # --- use systemd type if defined or create default --- if [ -n "${INSTALL_K3S_TYPE}" ]; then SYSTEMD_TYPE=${INSTALL_K3S_TYPE} else SYSTEMD_TYPE=notify fi # --- use binary install directory if defined or create default --- if [ -n "${INSTALL_K3S_BIN_DIR}" ]; then BIN_DIR=${INSTALL_K3S_BIN_DIR} else # --- use /usr/local/bin if root can write to it, otherwise use /opt/bin if it exists BIN_DIR=/usr/local/bin if ! $SUDO sh -c "touch ${BIN_DIR}/k3s-ro-test && rm -rf ${BIN_DIR}/k3s-ro-test"; then if [ -d /opt/bin ]; then BIN_DIR=/opt/bin fi fi fi # --- use systemd directory if defined or create default --- if [ -n "${INSTALL_K3S_SYSTEMD_DIR}" ]; then SYSTEMD_DIR="${INSTALL_K3S_SYSTEMD_DIR}" else SYSTEMD_DIR=/etc/systemd/system fi # --- set related files from system name --- SERVICE_K3S=${SYSTEM_NAME}.service UNINSTALL_K3S_SH=${UNINSTALL_K3S_SH:-${BIN_DIR}/${SYSTEM_NAME}-uninstall.sh} KILLALL_K3S_SH=${KILLALL_K3S_SH:-${BIN_DIR}/k3s-killall.sh} # --- use service or environment location depending on systemd/openrc --- if [ "${HAS_SYSTEMD}" = true ]; then FILE_K3S_SERVICE=${SYSTEMD_DIR}/${SERVICE_K3S} FILE_K3S_ENV=${SYSTEMD_DIR}/${SERVICE_K3S}.env elif [ "${HAS_OPENRC}" = true ]; then $SUDO mkdir -p /etc/rancher/k3s FILE_K3S_SERVICE=/etc/init.d/${SYSTEM_NAME} FILE_K3S_ENV=/etc/rancher/k3s/${SYSTEM_NAME}.env fi # --- get hash of config & exec for currently installed k3s --- PRE_INSTALL_HASHES=$(get_installed_hashes) # --- if bin directory is read only skip download --- if [ "${INSTALL_K3S_BIN_DIR_READ_ONLY}" = true ]; then INSTALL_K3S_SKIP_DOWNLOAD=true fi # --- setup channel values INSTALL_K3S_CHANNEL_URL=${INSTALL_K3S_CHANNEL_URL:-'https://update.k3s.io/v1-release/channels'} INSTALL_K3S_CHANNEL=${INSTALL_K3S_CHANNEL:-'stable'} } # --- check if skip download environment variable set --- can_skip_download_binary() { if [ "${INSTALL_K3S_SKIP_DOWNLOAD}" != true ] && [ "${INSTALL_K3S_SKIP_DOWNLOAD}" != binary ]; then return 1 fi } can_skip_download_selinux() { if [ "${INSTALL_K3S_SKIP_DOWNLOAD}" != true ] && [ "${INSTALL_K3S_SKIP_DOWNLOAD}" != selinux ]; then return 1 fi } # --- verify an executable k3s binary is installed --- verify_k3s_is_executable() { if [ ! -x ${BIN_DIR}/k3s ]; then fatal "Executable k3s binary not found at ${BIN_DIR}/k3s" fi } # --- set arch and suffix, fatal if architecture not supported --- setup_verify_arch() { if [ -z "$ARCH" ]; then ARCH=$(uname -m) fi case $ARCH in amd64) ARCH=amd64 SUFFIX= ;; x86_64) ARCH=amd64 SUFFIX= ;; arm64) ARCH=arm64 SUFFIX=-${ARCH} ;; s390x) ARCH=s390x SUFFIX=-${ARCH} ;; aarch64) ARCH=arm64 SUFFIX=-${ARCH} ;; arm*) ARCH=arm SUFFIX=-${ARCH}hf ;; *) fatal "Unsupported architecture $ARCH" esac } # --- verify existence of network downloader executable --- verify_downloader() { # Return failure if it doesn't exist or is no executable [ -x "$(command -v $1)" ] || return 1 # Set verified executable as our downloader program and return success DOWNLOADER=$1 return 0 } # --- create temporary directory and cleanup when done --- setup_tmp() { TMP_DIR=$(mktemp -d -t k3s-install.XXXXXXXXXX) TMP_HASH=${TMP_DIR}/k3s.hash TMP_BIN=${TMP_DIR}/k3s.bin cleanup() { code=$? set +e trap - EXIT rm -rf ${TMP_DIR} exit $code } trap cleanup INT EXIT } # --- use desired k3s version if defined or find version from channel --- get_release_version() { if [ -n "${INSTALL_K3S_COMMIT}" ]; then VERSION_K3S="commit ${INSTALL_K3S_COMMIT}" elif [ -n "${INSTALL_K3S_VERSION}" ]; then VERSION_K3S=${INSTALL_K3S_VERSION} else info "Finding release for channel ${INSTALL_K3S_CHANNEL}" version_url="${INSTALL_K3S_CHANNEL_URL}/${INSTALL_K3S_CHANNEL}" case $DOWNLOADER in curl) VERSION_K3S=$(curl -w '%{url_effective}' -L -s -S ${version_url} -o /dev/null | sed -e 's|.*/||') ;; wget) VERSION_K3S=$(wget -SqO /dev/null ${version_url} 2>&1 | grep -i Location | sed -e 's|.*/||') ;; *) fatal "Incorrect downloader executable '$DOWNLOADER'" ;; esac fi info "Using ${VERSION_K3S} as release" } # --- get k3s-selinux version --- get_k3s_selinux_version() { available_version="k3s-selinux-1.2-2.${rpm_target}.noarch.rpm" info "Finding available k3s-selinux versions" # run verify_downloader in case it binary installation was skipped verify_downloader curl || verify_downloader wget || fatal 'Can not find curl or wget for downloading files' case $DOWNLOADER in curl) DOWNLOADER_OPTS="-s" ;; wget) DOWNLOADER_OPTS="-q -O -" ;; *) fatal "Incorrect downloader executable '$DOWNLOADER'" ;; esac for i in {1..3}; do set +e if [ "${rpm_channel}" = "testing" ]; then version=$(timeout 5 ${DOWNLOADER} ${DOWNLOADER_OPTS} https://api.github.com/repos/k3s-io/k3s-selinux/releases | grep browser_download_url | awk '{ print $2 }' | grep -oE "[^\/]+${rpm_target}\.noarch\.rpm" | head -n 1) else version=$(timeout 5 ${DOWNLOADER} ${DOWNLOADER_OPTS} https://api.github.com/repos/k3s-io/k3s-selinux/releases/latest | grep browser_download_url | awk '{ print $2 }' | grep -oE "[^\/]+${rpm_target}\.noarch\.rpm") fi set -e if [ "${version}" != "" ]; then break fi sleep 1 done if [ "${version}" == "" ]; then warn "Failed to get available versions of k3s-selinux..defaulting to ${available_version}" return fi available_version=${version} } # --- download from github url --- download() { [ $# -eq 2 ] || fatal 'download needs exactly 2 arguments' case $DOWNLOADER in curl) curl -o $1 -skfL $2 ;; wget) wget -qO $1 $2 ;; *) fatal "Incorrect executable '$DOWNLOADER'" ;; esac # Abort if download command failed [ $? -eq 0 ] || fatal 'Download failed' } # --- download hash from github url --- download_hash() { if [ -n "${INSTALL_K3S_COMMIT}" ]; then HASH_URL=${STORAGE_URL}/k3s${SUFFIX}-${INSTALL_K3S_COMMIT}.sha256sum else HASH_URL=${GITHUB_URL}/download/${VERSION_K3S}/sha256sum-${ARCH}.txt fi info "Downloading hash ${HASH_URL}" download ${TMP_HASH} ${HASH_URL} HASH_EXPECTED=$(grep " k3s${SUFFIX}$" ${TMP_HASH}) HASH_EXPECTED=${HASH_EXPECTED%%[[:blank:]]*} } # --- check hash against installed version --- installed_hash_matches() { if [ -x ${BIN_DIR}/k3s ]; then HASH_INSTALLED=$(sha256sum ${BIN_DIR}/k3s) HASH_INSTALLED=${HASH_INSTALLED%%[[:blank:]]*} if [ "${HASH_EXPECTED}" = "${HASH_INSTALLED}" ]; then return fi fi return 1 } # --- download binary from github url --- download_binary() { if [ -n "${INSTALL_K3S_COMMIT}" ]; then BIN_URL=${STORAGE_URL}/k3s${SUFFIX}-${INSTALL_K3S_COMMIT} else BIN_URL=${GITHUB_URL}/download/${VERSION_K3S}/k3s${SUFFIX} fi info "Downloading binary ${BIN_URL}" download ${TMP_BIN} ${BIN_URL} } # --- verify downloaded binary hash --- verify_binary() { info "Verifying binary download" HASH_BIN=$(sha256sum ${TMP_BIN}) HASH_BIN=${HASH_BIN%%[[:blank:]]*} if [ "${HASH_EXPECTED}" != "${HASH_BIN}" ]; then fatal "Download sha256 does not match ${HASH_EXPECTED}, got ${HASH_BIN}" fi } # --- setup permissions and move binary to system directory --- setup_binary() { chmod 755 ${TMP_BIN} info "Installing k3s to ${BIN_DIR}/k3s" $SUDO chown root:root ${TMP_BIN} $SUDO mv -f ${TMP_BIN} ${BIN_DIR}/k3s } # --- setup selinux policy --- setup_selinux() { case ${INSTALL_K3S_CHANNEL} in *testing) rpm_channel=testing ;; *latest) rpm_channel=latest ;; *) rpm_channel=stable ;; esac rpm_site="rpm.rancher.io" if [ "${rpm_channel}" = "testing" ]; then rpm_site="rpm-testing.rancher.io" fi [ -r /etc/os-release ] && . /etc/os-release if [ `expr "${ID_LIKE}" : ".*suse.*"` != 0 ]; then rpm_target=sle rpm_site_infix=microos package_installer=zypper if [ "${ID_LIKE:-}" = suse ] && [ "${VARIANT_ID:-}" = sle-micro ]; then rpm_target=sle rpm_site_infix=slemicro package_installer=zypper fi elif [ "${ID_LIKE:-}" = coreos ] || [ "${VARIANT_ID:-}" = coreos ]; then rpm_target=coreos rpm_site_infix=coreos package_installer=rpm-ostree elif [ "${VERSION_ID%%.*}" = "7" ]; then rpm_target=el7 rpm_site_infix=centos/7 package_installer=yum elif [ "${VERSION_ID%%.*}" = "8" ] || [ "${VERSION_ID%%.*}" -gt "36" ]; then rpm_target=el8 rpm_site_infix=centos/8 package_installer=yum else rpm_target=el9 rpm_site_infix=centos/9 package_installer=yum fi if [ "${package_installer}" = "rpm-ostree" ] && [ -x /bin/yum ]; then package_installer=yum fi if [ "${package_installer}" = "yum" ] && [ -x /usr/bin/dnf ]; then package_installer=dnf fi policy_hint="please install: ${package_installer} install -y container-selinux ${package_installer} install -y https://${rpm_site}/k3s/${rpm_channel}/common/${rpm_site_infix}/noarch/${available_version} " if [ "$INSTALL_K3S_SKIP_SELINUX_RPM" = true ] || can_skip_download_selinux || [ ! -d /usr/share/selinux ]; then info "Skipping installation of SELinux RPM" else get_k3s_selinux_version install_selinux_rpm ${rpm_site} ${rpm_channel} ${rpm_target} ${rpm_site_infix} fi policy_error=fatal if [ "$INSTALL_K3S_SELINUX_WARN" = true ] || [ "${ID_LIKE:-}" = coreos ] || [ "${VARIANT_ID:-}" = coreos ]; then policy_error=warn fi if ! $SUDO chcon -u system_u -r object_r -t container_runtime_exec_t ${BIN_DIR}/k3s >/dev/null 2>&1; then if $SUDO grep '^\s*SELINUX=enforcing' /etc/selinux/config >/dev/null 2>&1; then $policy_error "Failed to apply container_runtime_exec_t to ${BIN_DIR}/k3s, ${policy_hint}" fi elif [ ! -f /usr/share/selinux/packages/k3s.pp ]; then if [ -x /usr/sbin/transactional-update ] || [ "${ID_LIKE:-}" = coreos ] || [ "${VARIANT_ID:-}" = coreos ]; then warn "Please reboot your machine to activate the changes and avoid data loss." else $policy_error "Failed to find the k3s-selinux policy, ${policy_hint}" fi fi } install_selinux_rpm() { if [ -r /etc/redhat-release ] || [ -r /etc/centos-release ] || [ -r /etc/oracle-release ] || [ -r /etc/fedora-release ] || [ "${ID_LIKE%%[ ]*}" = "suse" ]; then repodir=/etc/yum.repos.d if [ -d /etc/zypp/repos.d ]; then repodir=/etc/zypp/repos.d fi set +o noglob $SUDO rm -f ${repodir}/rancher-k3s-common*.repo set -o noglob if [ -r /etc/redhat-release ] && [ "${3}" = "el7" ]; then $SUDO yum install -y yum-utils $SUDO yum-config-manager --enable rhel-7-server-extras-rpms fi $SUDO tee ${repodir}/rancher-k3s-common.repo >/dev/null << EOF [rancher-k3s-common-${2}] name=Rancher K3s Common (${2}) baseurl=https://${1}/k3s/${2}/common/${4}/noarch enabled=1 gpgcheck=1 repo_gpgcheck=0 gpgkey=https://${1}/public.key EOF case ${3} in sle) rpm_installer="zypper --gpg-auto-import-keys" if [ "${TRANSACTIONAL_UPDATE=false}" != "true" ] && [ -x /usr/sbin/transactional-update ]; then transactional_update_run="transactional-update --no-selfupdate -d run" rpm_installer="transactional-update --no-selfupdate -d run ${rpm_installer}" : "${INSTALL_K3S_SKIP_START:=true}" fi # create the /var/lib/rpm-state in SLE systems to fix the prein selinux macro ${transactional_update_run} mkdir -p /var/lib/rpm-state ;; coreos) rpm_installer="rpm-ostree" # rpm_install_extra_args="--apply-live" : "${INSTALL_K3S_SKIP_START:=true}" ;; *) rpm_installer="yum" ;; esac if [ "${rpm_installer}" = "yum" ] && [ -x /usr/bin/dnf ]; then rpm_installer=dnf fi if rpm -q --quiet k3s-selinux; then # remove k3s-selinux module before upgrade to allow container-selinux to upgrade safely if check_available_upgrades container-selinux ${3} && check_available_upgrades k3s-selinux ${3}; then MODULE_PRIORITY=$($SUDO semodule --list=full | grep k3s | cut -f1 -d" ") if [ -n "${MODULE_PRIORITY}" ]; then $SUDO semodule -X $MODULE_PRIORITY -r k3s || true fi fi fi # shellcheck disable=SC2086 $SUDO ${rpm_installer} install -y "k3s-selinux" fi return } check_available_upgrades() { set +e case ${2} in sle) available_upgrades=$($SUDO zypper -q -t -s 11 se -s -u --type package $1 | tail -n 1 | grep -v "No matching" | awk '{print $3}') ;; coreos) # currently rpm-ostree does not support search functionality https://github.com/coreos/rpm-ostree/issues/1877 ;; *) available_upgrades=$($SUDO yum -q --refresh list $1 --upgrades | tail -n 1 | awk '{print $2}') ;; esac set -e if [ -n "${available_upgrades}" ]; then return 0 fi return 1 } # --- download and verify k3s --- download_and_verify() { if can_skip_download_binary; then info 'Skipping k3s download and verify' verify_k3s_is_executable return fi setup_verify_arch verify_downloader curl || verify_downloader wget || fatal 'Can not find curl or wget for downloading files' setup_tmp get_release_version download_hash if installed_hash_matches; then info 'Skipping binary downloaded, installed k3s matches hash' return fi download_binary verify_binary setup_binary } # --- add additional utility links --- create_symlinks() { [ "${INSTALL_K3S_BIN_DIR_READ_ONLY}" = true ] && return [ "${INSTALL_K3S_SYMLINK}" = skip ] && return for cmd in kubectl crictl ctr; do if [ ! -e ${BIN_DIR}/${cmd} ] || [ "${INSTALL_K3S_SYMLINK}" = force ]; then which_cmd=$(command -v ${cmd} 2>/dev/null || true) if [ -z "${which_cmd}" ] || [ "${INSTALL_K3S_SYMLINK}" = force ]; then info "Creating ${BIN_DIR}/${cmd} symlink to k3s" $SUDO ln -sf k3s ${BIN_DIR}/${cmd} else info "Skipping ${BIN_DIR}/${cmd} symlink to k3s, command exists in PATH at ${which_cmd}" fi else info "Skipping ${BIN_DIR}/${cmd} symlink to k3s, already exists" fi done } # --- create killall script --- create_killall() { [ "${INSTALL_K3S_BIN_DIR_READ_ONLY}" = true ] && return info "Creating killall script ${KILLALL_K3S_SH}" $SUDO tee ${KILLALL_K3S_SH} >/dev/null << \EOF #!/bin/sh [ $(id -u) -eq 0 ] || exec sudo $0 $@ for bin in /var/lib/rancher/k3s/data/**/bin/; do [ -d $bin ] && export PATH=$PATH:$bin:$bin/aux done set -x for service in /etc/systemd/system/k3s*.service; do [ -s $service ] && systemctl stop $(basename $service) done for service in /etc/init.d/k3s*; do [ -x $service ] && $service stop done pschildren() { ps -e -o ppid= -o pid= | \ sed -e 's/^\s*//g; s/\s\s*/\t/g;' | \ grep -w "^$1" | \ cut -f2 } pstree() { for pid in $@; do echo $pid for child in $(pschildren $pid); do pstree $child done done } killtree() { kill -9 $( { set +x; } 2>/dev/null; pstree $@; set -x; ) 2>/dev/null } remove_interfaces() { # Delete network interface(s) that match 'master cni0' ip link show 2>/dev/null | grep 'master cni0' | while read ignore iface ignore; do iface=${iface%%@*} [ -z "$iface" ] || ip link delete $iface done # Delete cni related interfaces ip link delete cni0 ip link delete flannel.1 ip link delete flannel-v6.1 ip link delete kube-ipvs0 ip link delete flannel-wg ip link delete flannel-wg-v6 # Restart tailscale if [ -n "$(command -v tailscale)" ]; then tailscale set --advertise-routes= fi } getshims() { ps -e -o pid= -o args= | sed -e 's/^ *//; s/\s\s*/\t/;' | grep -w 'k3s/data/[^/]*/bin/containerd-shim' | cut -f1 } killtree $({ set +x; } 2>/dev/null; getshims; set -x) do_unmount_and_remove() { set +x while read -r _ path _; do case "$path" in $1*) echo "$path" ;; esac done < /proc/self/mounts | sort -r | xargs -r -t -n 1 sh -c 'umount "$0" && rm -rf "$0"' set -x } do_unmount_and_remove '/run/k3s' do_unmount_and_remove '/var/lib/rancher/k3s' do_unmount_and_remove '/var/lib/kubelet/pods' do_unmount_and_remove '/var/lib/kubelet/plugins' do_unmount_and_remove '/run/netns/cni-' # Remove CNI namespaces ip netns show 2>/dev/null | grep cni- | xargs -r -t -n 1 ip netns delete remove_interfaces rm -rf /var/lib/cni/ iptables-save | grep -v KUBE- | grep -v CNI- | grep -iv flannel | iptables-restore ip6tables-save | grep -v KUBE- | grep -v CNI- | grep -iv flannel | ip6tables-restore EOF $SUDO chmod 755 ${KILLALL_K3S_SH} $SUDO chown root:root ${KILLALL_K3S_SH} } # --- create uninstall script --- create_uninstall() { [ "${INSTALL_K3S_BIN_DIR_READ_ONLY}" = true ] && return info "Creating uninstall script ${UNINSTALL_K3S_SH}" $SUDO tee ${UNINSTALL_K3S_SH} >/dev/null << EOF #!/bin/sh set -x [ \$(id -u) -eq 0 ] || exec sudo \$0 \$@ ${KILLALL_K3S_SH} if command -v systemctl; then systemctl disable ${SYSTEM_NAME} systemctl reset-failed ${SYSTEM_NAME} systemctl daemon-reload fi if command -v rc-update; then rc-update delete ${SYSTEM_NAME} default fi rm -f ${FILE_K3S_SERVICE} rm -f ${FILE_K3S_ENV} remove_uninstall() { rm -f ${UNINSTALL_K3S_SH} } trap remove_uninstall EXIT if (ls ${SYSTEMD_DIR}/k3s*.service || ls /etc/init.d/k3s*) >/dev/null 2>&1; then set +x; echo 'Additional k3s services installed, skipping uninstall of k3s'; set -x exit fi for cmd in kubectl crictl ctr; do if [ -L ${BIN_DIR}/\$cmd ]; then rm -f ${BIN_DIR}/\$cmd fi done rm -rf /etc/rancher/k3s rm -rf /run/k3s rm -rf /run/flannel rm -rf /var/lib/rancher/k3s rm -rf /var/lib/kubelet rm -f ${BIN_DIR}/k3s rm -f ${KILLALL_K3S_SH} if type yum >/dev/null 2>&1; then yum remove -y k3s-selinux rm -f /etc/yum.repos.d/rancher-k3s-common*.repo elif type rpm-ostree >/dev/null 2>&1; then rpm-ostree uninstall k3s-selinux rm -f /etc/yum.repos.d/rancher-k3s-common*.repo elif type zypper >/dev/null 2>&1; then uninstall_cmd="zypper remove -y k3s-selinux" if [ "\${TRANSACTIONAL_UPDATE=false}" != "true" ] && [ -x /usr/sbin/transactional-update ]; then uninstall_cmd="transactional-update --no-selfupdate -d run \$uninstall_cmd" fi \$uninstall_cmd rm -f /etc/zypp/repos.d/rancher-k3s-common*.repo fi EOF $SUDO chmod 755 ${UNINSTALL_K3S_SH} $SUDO chown root:root ${UNINSTALL_K3S_SH} } # --- disable current service if loaded -- systemd_disable() { $SUDO systemctl disable ${SYSTEM_NAME} >/dev/null 2>&1 || true $SUDO rm -f /etc/systemd/system/${SERVICE_K3S} || true $SUDO rm -f /etc/systemd/system/${SERVICE_K3S}.env || true } # --- capture current env and create file containing k3s_ variables --- create_env_file() { info "env: Creating environment file ${FILE_K3S_ENV}" $SUDO touch ${FILE_K3S_ENV} $SUDO chmod 0600 ${FILE_K3S_ENV} sh -c export | while read x v; do echo $v; done | grep -E '^(K3S|CONTAINERD)_' | $SUDO tee ${FILE_K3S_ENV} >/dev/null sh -c export | while read x v; do echo $v; done | grep -Ei '^(NO|HTTP|HTTPS)_PROXY' | $SUDO tee -a ${FILE_K3S_ENV} >/dev/null } # --- write systemd service file --- create_systemd_service_file() { info "systemd: Creating service file ${FILE_K3S_SERVICE}" $SUDO tee ${FILE_K3S_SERVICE} >/dev/null << EOF [Unit] Description=Lightweight Kubernetes Documentation=https://k3s.io Wants=network-online.target After=network-online.target [Install] WantedBy=multi-user.target [Service] Type=${SYSTEMD_TYPE} EnvironmentFile=-/etc/default/%N EnvironmentFile=-/etc/sysconfig/%N EnvironmentFile=-${FILE_K3S_ENV} KillMode=process Delegate=yes # Having non-zero Limit*s causes performance problems due to accounting overhead # in the kernel. We recommend using cgroups to do container-local accounting. LimitNOFILE=1048576 LimitNPROC=infinity LimitCORE=infinity TasksMax=infinity TimeoutStartSec=0 Restart=always RestartSec=5s ExecStartPre=/bin/sh -xc '! /usr/bin/systemctl is-enabled --quiet nm-cloud-setup.service' ExecStartPre=-/sbin/modprobe br_netfilter ExecStartPre=-/sbin/modprobe overlay ExecStart=${BIN_DIR}/k3s \\ ${CMD_K3S_EXEC} EOF } # --- write openrc service file --- create_openrc_service_file() { LOG_FILE=/var/log/${SYSTEM_NAME}.log info "openrc: Creating service file ${FILE_K3S_SERVICE}" $SUDO tee ${FILE_K3S_SERVICE} >/dev/null << EOF #!/sbin/openrc-run depend() { after network-online want cgroups } start_pre() { rm -f /tmp/k3s.* } supervisor=supervise-daemon name=${SYSTEM_NAME} command="${BIN_DIR}/k3s" command_args="$(escape_dq "${CMD_K3S_EXEC}") >>${LOG_FILE} 2>&1" output_log=${LOG_FILE} error_log=${LOG_FILE} pidfile="/var/run/${SYSTEM_NAME}.pid" respawn_delay=5 respawn_max=0 set -o allexport if [ -f /etc/environment ]; then . /etc/environment; fi if [ -f ${FILE_K3S_ENV} ]; then . ${FILE_K3S_ENV}; fi set +o allexport EOF $SUDO chmod 0755 ${FILE_K3S_SERVICE} $SUDO tee /etc/logrotate.d/${SYSTEM_NAME} >/dev/null << EOF ${LOG_FILE} { missingok notifempty copytruncate } EOF } # --- write systemd or openrc service file --- create_service_file() { [ "${HAS_SYSTEMD}" = true ] && create_systemd_service_file [ "${HAS_OPENRC}" = true ] && create_openrc_service_file return 0 } # --- get hashes of the current k3s bin and service files get_installed_hashes() { $SUDO sha256sum ${BIN_DIR}/k3s ${FILE_K3S_SERVICE} ${FILE_K3S_ENV} 2>&1 || true } # --- enable and start systemd service --- systemd_enable() { info "systemd: Enabling ${SYSTEM_NAME} unit" $SUDO systemctl enable ${FILE_K3S_SERVICE} >/dev/null $SUDO systemctl daemon-reload >/dev/null } systemd_start() { info "systemd: Starting ${SYSTEM_NAME}" $SUDO systemctl restart ${SYSTEM_NAME} } # --- enable and start openrc service --- openrc_enable() { info "openrc: Enabling ${SYSTEM_NAME} service for default runlevel" $SUDO rc-update add ${SYSTEM_NAME} default >/dev/null } openrc_start() { info "openrc: Starting ${SYSTEM_NAME}" $SUDO ${FILE_K3S_SERVICE} restart } # --- startup systemd or openrc service --- service_enable_and_start() { if [ -f "/proc/cgroups" ] && [ "$(grep memory /proc/cgroups | while read -r n n n enabled; do echo $enabled; done)" -eq 0 ]; then info 'Failed to find memory cgroup, you may need to add "cgroup_memory=1 cgroup_enable=memory" to your linux cmdline (/boot/cmdline.txt on a Raspberry Pi)' fi [ "${INSTALL_K3S_SKIP_ENABLE}" = true ] && return [ "${HAS_SYSTEMD}" = true ] && systemd_enable [ "${HAS_OPENRC}" = true ] && openrc_enable [ "${INSTALL_K3S_SKIP_START}" = true ] && return POST_INSTALL_HASHES=$(get_installed_hashes) if [ "${PRE_INSTALL_HASHES}" = "${POST_INSTALL_HASHES}" ] && [ "${INSTALL_K3S_FORCE_RESTART}" != true ]; then info 'No change detected so skipping service start' return fi if command -v iptables-save 1> /dev/null && command -v iptables-restore 1> /dev/null then $SUDO iptables-save | grep -v KUBE- | grep -iv flannel | $SUDO iptables-restore fi if command -v ip6tables-save 1> /dev/null && command -v ip6tables-restore 1> /dev/null then $SUDO ip6tables-save | grep -v KUBE- | grep -iv flannel | $SUDO ip6tables-restore fi [ "${HAS_SYSTEMD}" = true ] && systemd_start [ "${HAS_OPENRC}" = true ] && openrc_start return 0 } # --- re-evaluate args to include env command --- eval set -- $(escape "${INSTALL_K3S_EXEC}") $(quote "$@") # --- run the install process -- { verify_system setup_env "$@" download_and_verify setup_selinux create_symlinks create_killall create_uninstall systemd_disable create_env_file create_service_file service_enable_and_start } ================================================ FILE: mdz/pkg/server/k3s_destroy.go ================================================ package server import ( "fmt" "os/exec" "syscall" ) // k3sDestroyAllStep installs k3s and related tools. type k3sDestroyAllStep struct { options Options } func (s *k3sDestroyAllStep) Run() error { fmt.Fprintf(s.options.OutputStream, "🚧 Destroy the OpenModelz Cluster...\n") // TODO(gaocegege): Embed the script into the binary. cmd := exec.Command("/bin/sh", "-c", "/usr/local/bin/k3s-uninstall.sh") cmd.SysProcAttr = &syscall.SysProcAttr{ Pdeathsig: syscall.SIGKILL, } if s.options.Verbose { cmd.Stderr = s.options.OutputStream cmd.Stdout = s.options.OutputStream } else { cmd.Stdout = nil cmd.Stderr = nil } err := cmd.Run() if err != nil { return err } return nil } func (s *k3sDestroyAllStep) Verify() error { return nil } ================================================ FILE: mdz/pkg/server/k3s_install.go ================================================ package server import ( _ "embed" "fmt" "io" "os/exec" "syscall" ) //go:embed k3s-install.sh var bashContent string // k3sInstallStep installs k3s and related tools. type k3sInstallStep struct { options Options } func (s *k3sInstallStep) Run() error { checkCmd := exec.Command("/bin/sh", "-c", "sudo k3s kubectl get nodes") checkCmd.SysProcAttr = &syscall.SysProcAttr{ Pdeathsig: syscall.SIGKILL, } checkCmd.Stdout = nil checkCmd.Stderr = nil err := checkCmd.Run() if err == nil { fmt.Fprintf(s.options.OutputStream, "🚧 The server is already created, skip...\n") return nil } fmt.Fprintf(s.options.OutputStream, "🚧 Creating the server...\n") // TODO(gaocegege): Embed the script into the binary. // Always run start, do not check the hash to decide. cmd := exec.Command("/bin/sh", "-c", "INSTALL_K3S_VERSION=v1.27.3+k3s1 INSTALL_K3S_EXEC='--disable=traefik' INSTALL_K3S_FORCE_RESTART=true K3S_KUBECONFIG_MODE=644 K3S_TOKEN=openmodelz sh -") cmd.SysProcAttr = &syscall.SysProcAttr{ Pdeathsig: syscall.SIGKILL, } stdin, err := cmd.StdinPipe() if err != nil { return err } defer stdin.Close() // the doc says subProcess.Wait will close it, but I'm not sure, so I kept this line if s.options.Verbose { cmd.Stderr = s.options.OutputStream cmd.Stdout = s.options.OutputStream } else { cmd.Stdout = nil cmd.Stderr = nil } if err := cmd.Start(); err != nil { return err } if _, err := io.WriteString(stdin, bashContent); err != nil { return err } // Close the input stream to finish the pipe. Then the command will use the // input from the pipe to start the next process. stdin.Close() fmt.Fprintf(s.options.OutputStream, "🚧 Waiting for the server to be created...\n") if err := cmd.Wait(); err != nil { return err } return nil } func (s *k3sInstallStep) Verify() error { return nil } ================================================ FILE: mdz/pkg/server/k3s_join.go ================================================ package server import ( "fmt" "io" "os/exec" "syscall" ) // k3sJoinStep installs k3s and related tools. type k3sJoinStep struct { options Options } func (s *k3sJoinStep) Run() error { fmt.Fprintf(s.options.OutputStream, "🚧 Joining the cluster...\n") // TODO(gaocegege): Embed the script into the binary. cmdStr := fmt.Sprintf("INSTALL_K3S_FORCE_RESTART=true K3S_KUBECONFIG_MODE=644 K3S_TOKEN=openmodelz K3S_URL=https://%s:6443 sh -", s.options.ServerIP) cmd := exec.Command("/bin/sh", "-c", cmdStr) cmd.SysProcAttr = &syscall.SysProcAttr{ Pdeathsig: syscall.SIGKILL, } if s.options.Verbose { cmd.Stderr = s.options.OutputStream cmd.Stdout = s.options.OutputStream } else { cmd.Stdout = nil cmd.Stderr = nil } stdin, err := cmd.StdinPipe() if err != nil { return err } defer stdin.Close() // the doc says subProcess.Wait will close it, but I'm not sure, so I kept this line if s.options.Verbose { cmd.Stderr = s.options.OutputStream cmd.Stdout = s.options.OutputStream } else { cmd.Stdout = nil cmd.Stderr = nil } if err := cmd.Start(); err != nil { return err } if _, err := io.WriteString(stdin, bashContent); err != nil { return err } // Close the input stream to finish the pipe. Then the command will use the // input from the pipe to start the next process. stdin.Close() fmt.Fprintf(s.options.OutputStream, "🚧 Waiting for the server to be ready...\n") if err := cmd.Wait(); err != nil { return err } return nil } func (s *k3sJoinStep) Verify() error { return nil } ================================================ FILE: mdz/pkg/server/k3s_killall.go ================================================ package server import ( "fmt" "os/exec" "syscall" ) // k3sKillAllStep installs k3s and related tools. type k3sKillAllStep struct { options Options } func (s *k3sKillAllStep) Run() error { fmt.Fprintf(s.options.OutputStream, "🚧 Stopping the OpenModelz Cluster...\n") // TODO(gaocegege): Embed the script into the binary. cmd := exec.Command("/bin/sh", "-c", "/usr/local/bin/k3s-killall.sh") cmd.SysProcAttr = &syscall.SysProcAttr{ Pdeathsig: syscall.SIGKILL, } if s.options.Verbose { cmd.Stderr = s.options.OutputStream cmd.Stdout = s.options.OutputStream } else { cmd.Stdout = nil cmd.Stderr = nil } err := cmd.Run() if err != nil { return err } return nil } func (s *k3sKillAllStep) Verify() error { return nil } ================================================ FILE: mdz/pkg/server/k3s_prepare.go ================================================ package server import ( _ "embed" "fmt" "os/exec" "path/filepath" "strings" "syscall" "text/template" ) //go:embed registries.yaml var registriesContent string const mirrorPath = "/etc/rancher/k3s" const mirrorFile = "registries.yaml" // k3sPrepare install everything required by k3s. type k3sPrepare struct { options Options } func (s *k3sPrepare) Run() error { if !s.options.Mirror.Configured() { return nil } fmt.Fprintf(s.options.OutputStream, "🚧 Configure the mirror...\n") tmpl, err := template.New("registries").Parse(registriesContent) if err != nil { panic(err) } buf := strings.Builder{} err = tmpl.Execute(&buf, s.options.Mirror) if err != nil { panic(err) } cmd := exec.Command("/bin/sh", "-c", fmt.Sprintf( "sudo mkdir -p %s && sudo tee %s > /dev/null << EOF\n%s\nEOF", mirrorPath, filepath.Join(mirrorPath, mirrorFile), buf.String(), )) cmd.SysProcAttr = &syscall.SysProcAttr{ Pdeathsig: syscall.SIGKILL, } if s.options.Verbose { cmd.Stderr = s.options.OutputStream cmd.Stdout = s.options.OutputStream } else { cmd.Stdout = nil cmd.Stderr = nil } err = cmd.Run() if err != nil { return err } return nil } func (s *k3sPrepare) Verify() error { return nil } ================================================ FILE: mdz/pkg/server/nginx-dep.yaml ================================================ kind: Namespace apiVersion: v1 metadata: name: ingress-nginx --- apiVersion: helm.cattle.io/v1 kind: HelmChart metadata: name: ingress-nginx namespace: kube-system spec: chart: ingress-nginx repo: https://kubernetes.github.io/ingress-nginx targetNamespace: ingress-nginx version: v4.7.0 set: valuesContent: |- fullnameOverride: ingress-nginx controller: kind: DaemonSet hostNetwork: true hostPort: enabled: true service: enabled: true ports: http: 9000 https: 9001 publishService: enabled: false metrics: enabled: false serviceMonitor: enabled: false config: use-forwarded-headers: "true" ================================================ FILE: mdz/pkg/server/nginx_install.go ================================================ package server import ( _ "embed" "fmt" "io" "os/exec" "syscall" ) //go:embed nginx-dep.yaml var nginxYamlContent string // nginxInstallStep installs the nginx deployment. type nginxInstallStep struct { options Options } func (s *nginxInstallStep) Run() error { fmt.Fprintf(s.options.OutputStream, "🚧 Initializing the load balancer...\n") cmd := exec.Command("/bin/sh", "-c", "sudo k3s kubectl apply -f -") cmd.SysProcAttr = &syscall.SysProcAttr{ Pdeathsig: syscall.SIGKILL, } stdin, err := cmd.StdinPipe() if err != nil { return err } defer stdin.Close() // the doc says subProcess.Wait will close it, but I'm not sure, so I kept this line if s.options.Verbose { cmd.Stderr = s.options.OutputStream cmd.Stdout = s.options.OutputStream } else { cmd.Stdout = nil cmd.Stderr = nil } if err := cmd.Start(); err != nil { return err } if _, err := io.WriteString(stdin, nginxYamlContent); err != nil { return err } // Close the input stream to finish the pipe. Then the command will use the // input from the pipe to start the next process. stdin.Close() if err := cmd.Wait(); err != nil { return err } return nil } func (s *nginxInstallStep) Verify() error { return nil } ================================================ FILE: mdz/pkg/server/openmodelz.yaml ================================================ apiVersion: v1 kind: Namespace metadata: name: openmodelz labels: name: openmodelz --- apiVersion: helm.cattle.io/v1 kind: HelmChart metadata: name: openmodelz namespace: kube-system spec: chart: openmodelz repo: https://tensorchord.github.io/openmodelz-charts targetNamespace: openmodelz version: {{.Version}} set: valuesContent: |- fullnameOverride: openmodelz agent: ingress: enabled: true ipToDomain: {{.IpToDomain}} domain: "{{.Domain}}" modelzCloud: enabled: {{.EnableModelZCloud}} url: {{.ModelZCloudUrl}} token: {{.ModelZCloudAgentToken}} region: {{.ModelZCloudRegion}} ================================================ FILE: mdz/pkg/server/openmodelz_install.go ================================================ package server import ( _ "embed" "fmt" "html/template" "io" "os/exec" "regexp" "strings" "syscall" "github.com/sirupsen/logrus" ) //go:embed openmodelz.yaml var yamlContent string var resultDomain string // openModelZInstallStep installs the OpenModelZ deployments. type openModelZInstallStep struct { options Options } func (s *openModelZInstallStep) Run() error { fmt.Fprintf(s.options.OutputStream, "🚧 Initializing the server...\n") cmd := exec.Command("/bin/sh", "-c", "sudo k3s kubectl apply -f -") cmd.SysProcAttr = &syscall.SysProcAttr{ Pdeathsig: syscall.SIGKILL, } stdin, err := cmd.StdinPipe() if err != nil { return err } defer stdin.Close() // the doc says subProcess.Wait will close it, but I'm not sure, so I kept this line if s.options.Verbose { cmd.Stderr = s.options.OutputStream cmd.Stdout = s.options.OutputStream } else { cmd.Stdout = nil cmd.Stderr = nil } if err := cmd.Start(); err != nil { return err } variables := struct { Domain string IpToDomain bool Version string EnableModelZCloud bool ModelZCloudUrl string ModelZCloudAgentToken string ModelZCloudRegion string }{ Version: s.options.Version, } if s.options.Domain != nil { variables.Domain = *s.options.Domain variables.IpToDomain = false } else { fmt.Fprintf(s.options.OutputStream, "🚧 No domain provided, using the server IP...\n") variables.Domain = "" variables.IpToDomain = true } if s.options.ModelZCloud.Enabled { variables.EnableModelZCloud = true variables.ModelZCloudUrl = s.options.ModelZCloud.URL variables.ModelZCloudAgentToken = s.options.ModelZCloud.Token variables.ModelZCloudRegion = s.options.ModelZCloud.Region } else { variables.EnableModelZCloud = false variables.ModelZCloudUrl = "" variables.ModelZCloudAgentToken = "" variables.ModelZCloudRegion = "" } tmpl, err := template.New("openmodelz").Parse(yamlContent) if err != nil { panic(err) } buf := strings.Builder{} err = tmpl.Execute(&buf, variables) if err != nil { panic(err) } logrus.WithField("variables", variables). Debugf("Deploying OpenModelZ with the following variables") if _, err := io.WriteString(stdin, buf.String()); err != nil { return err } // Close the input stream to finish the pipe. Then the command will use the // input from the pipe to start the next process. stdin.Close() fmt.Fprintf(s.options.OutputStream, "🚧 Waiting for the server to be ready...\n") if err := cmd.Wait(); err != nil { return err } return nil } func (s *openModelZInstallStep) Verify() error { fmt.Fprintf(s.options.OutputStream, "🚧 Verifying the load balancer...\n") cmd := exec.Command("/bin/sh", "-c", "sudo k3s kubectl get svc -n ingress-nginx ingress-nginx-controller -o jsonpath={@.status.loadBalancer.ingress}") cmd.SysProcAttr = &syscall.SysProcAttr{ Pdeathsig: syscall.SIGKILL, } output, err := cmd.CombinedOutput() if err != nil { logrus.Debugf("failed to get the ingress ip: %v", err) return err } logrus.Debugf("kubectl get cmd output: %s\n", output) if len(output) <= 4 { return fmt.Errorf("cannot get the ingress ip: output is empty") } // Get the IP from the output lie this: `[{"ip":"192.168.71.93"}]` re := regexp.MustCompile(`(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}`) found := re.MatchString(string(output)) if !found { return fmt.Errorf("cannot get the ingress ip") } resultDomain = re.FindString(string(output)) return nil } ================================================ FILE: mdz/pkg/server/registries.yaml ================================================ mirrors: {{ .Name }}: endpoint: {{ range $endpoint := .Endpoints }}- "{{ $endpoint }}"{{ end }} ================================================ FILE: mdz/pkg/telemetry/telemetry.go ================================================ package telemetry import ( "io" "os" "path/filepath" "runtime" "sync" "time" "github.com/cockroachdb/errors" "github.com/google/uuid" segmentio "github.com/segmentio/analytics-go/v3" "github.com/sirupsen/logrus" "github.com/tensorchord/openmodelz/mdz/pkg/version" ) type TelemetryField func(*segmentio.Properties) type Telemetry interface { Record(command string, args ...TelemetryField) } type defaultTelemetry struct { client segmentio.Client uid string enabled bool } const telemetryToken = "65WHA9bxCNX74K3HjgplMOmsio9LkYSI" var ( once sync.Once telemetry *defaultTelemetry telemetryConfigFile string ) func init() { home, err := os.UserHomeDir() if err != nil { panic(err) } telemetryConfigFile = filepath.Join(home, ".config", "openmodelz", "telemetry") } func GetTelemetry() Telemetry { return telemetry } func Initialize(enabled bool) error { once.Do(func() { client, err := segmentio.NewWithConfig(telemetryToken, segmentio.Config{ BatchSize: 1, }) if err != nil { panic(err) } telemetry = &defaultTelemetry{ client: client, enabled: enabled, } }) return telemetry.init() } func (t *defaultTelemetry) init() error { if !t.enabled { return nil } // detect if the config file already exists _, err := os.Stat(telemetryConfigFile) if err != nil { if !os.IsNotExist(err) { return errors.Wrap(err, "failed to stat telemetry config file") } t.uid = uuid.New().String() return t.dumpConfig() } if err = t.loadConfig(); err != nil { return errors.Wrap(err, "failed to load telemetry config") } t.Idnetify() return nil } func (t *defaultTelemetry) dumpConfig() error { if err := os.MkdirAll(filepath.Dir(telemetryConfigFile), os.ModeDir|0700); err != nil { return errors.Wrap(err, "failed to create telemetry config directory") } file, err := os.Create(telemetryConfigFile) if err != nil { return errors.Wrap(err, "failed to create telemetry config file") } defer file.Close() _, err = file.WriteString(t.uid) if err != nil { return errors.Wrap(err, "failed to write telemetry config file") } return nil } func (t *defaultTelemetry) loadConfig() error { file, err := os.Open(telemetryConfigFile) if err != nil { return errors.Wrap(err, "failed to open telemetry config file") } defer file.Close() uid, err := io.ReadAll(file) if err != nil { return errors.Wrap(err, "failed to read telemetry config file") } t.uid = string(uid) return nil } func (t *defaultTelemetry) Idnetify() { if !t.enabled { return } v := version.GetOpenModelzVersion() if err := t.client.Enqueue(segmentio.Identify{ AnonymousId: t.uid, Context: &segmentio.Context{ OS: segmentio.OSInfo{ Name: runtime.GOOS, Version: runtime.GOARCH, }, App: segmentio.AppInfo{ Name: "openmodelz", Version: v, }, }, Timestamp: time.Now(), Traits: segmentio.NewTraits(), }); err != nil { logrus.WithError(err).Debug("failed to identify user") return } } func AddField(name string, value interface{}) TelemetryField { return func(p *segmentio.Properties) { p.Set(name, value) } } func (t *defaultTelemetry) Record(command string, fields ...TelemetryField) { if !t.enabled { return } logrus.WithField("UID", t.uid).WithField("command", command).Debug("send telemetry") track := segmentio.Track{ AnonymousId: t.uid, Event: command, Properties: segmentio.NewProperties(), } for _, field := range fields { field(&track.Properties) } if err := t.client.Enqueue(track); err != nil { logrus.WithError(err).Debug("failed to send telemetry") } // make sure the msg can be sent out t.client.Close() } ================================================ FILE: mdz/pkg/term/interrupt.go ================================================ /* Copyright 2016 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package term import ( "os" "os/signal" "sync" "syscall" ) // terminationSignals are signals that cause the program to exit in the // supported platforms (linux, darwin, windows). var terminationSignals = []os.Signal{syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT} // Handler guarantees execution of notifications after a critical section (the function passed // to a Run method), even in the presence of process termination. It guarantees exactly once // invocation of the provided notify functions. type Handler struct { notify []func() final func(os.Signal) once sync.Once } // Chain creates a new handler that invokes all notify functions when the critical section exits // and then invokes the optional handler's notifications. This allows critical sections to be // nested without losing exactly once invocations. Notify functions can invoke any cleanup needed // but should not exit (which is the responsibility of the parent handler). func Chain(handler *Handler, notify ...func()) *Handler { if handler == nil { return New(nil, notify...) } return New(handler.Signal, append(notify, handler.Close)...) } // New creates a new handler that guarantees all notify functions are run after the critical // section exits (or is interrupted by the OS), then invokes the final handler. If no final // handler is specified, the default final is `os.Exit(1)`. A handler can only be used for // one critical section. func New(final func(os.Signal), notify ...func()) *Handler { return &Handler{ final: final, notify: notify, } } // Close executes all the notification handlers if they have not yet been executed. func (h *Handler) Close() { h.once.Do(func() { for _, fn := range h.notify { fn() } }) } // Signal is called when an os.Signal is received, and guarantees that all notifications // are executed, then the final handler is executed. This function should only be called once // per Handler instance. func (h *Handler) Signal(s os.Signal) { h.once.Do(func() { for _, fn := range h.notify { fn() } if h.final == nil { os.Exit(1) } h.final(s) }) } // Run ensures that any notifications are invoked after the provided fn exits (even if the // process is interrupted by an OS termination signal). Notifications are only invoked once // per Handler instance, so calling Run more than once will not behave as the user expects. func (h *Handler) Run(fn func() error) error { ch := make(chan os.Signal, 1) signal.Notify(ch, terminationSignals...) defer func() { signal.Stop(ch) close(ch) }() go func() { sig, ok := <-ch if !ok { return } h.Signal(sig) }() defer h.Close() return fn() } ================================================ FILE: mdz/pkg/term/term.go ================================================ /* Copyright 2016 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package term import ( "io" "os" "runtime" "github.com/moby/term" ) // SafeFunc is a function to be invoked by TTY. type SafeFunc func() error // TTY helps invoke a function and preserve the state of the terminal, even if the process is // terminated during execution. It also provides support for terminal resizing for remote command // execution/attachment. type TTY struct { // In is a reader representing stdin. It is a required field. In io.Reader // Out is a writer representing stdout. It must be set to support terminal resizing. It is an // optional field. Out io.Writer // Raw is true if the terminal should be set raw. Raw bool // TryDev indicates the TTY should try to open /dev/tty if the provided input // is not a file descriptor. TryDev bool // Parent is an optional interrupt handler provided to this function - if provided // it will be invoked after the terminal state is restored. If it is not provided, // a signal received during the TTY will result in os.Exit(0) being invoked. Parent *Handler // sizeQueue is set after a call to MonitorSize() and is used to monitor SIGWINCH signals when the // user's terminal resizes. // sizeQueue *sizeQueue } func NewTTY() *TTY { tty := &TTY{} stdin, stdout, _ := term.StdStreams() tty.In = stdin tty.Out = stdout return tty } // IsTerminalIn returns true if t.In is a terminal. Does not check /dev/tty // even if TryDev is set. func (t TTY) IsTerminalIn() bool { return IsTerminal(t.In) } // IsTerminalOut returns true if t.Out is a terminal. Does not check /dev/tty // even if TryDev is set. func (t TTY) IsTerminalOut() bool { return IsTerminal(t.Out) } // IsTerminal returns whether the passed object is a terminal or not func IsTerminal(i interface{}) bool { _, terminal := term.GetFdInfo(i) return terminal } // AllowsColorOutput returns true if the specified writer is a terminal and // the process environment indicates color output is supported and desired. func AllowsColorOutput(w io.Writer) bool { if !IsTerminal(w) { return false } // https://en.wikipedia.org/wiki/Computer_terminal#Dumb_terminals if os.Getenv("TERM") == "dumb" { return false } // https://no-color.org/ if _, nocolor := os.LookupEnv("NO_COLOR"); nocolor { return false } // On Windows WT_SESSION is set by the modern terminal component. // Older terminals have poor support for UTF-8, VT escape codes, etc. if runtime.GOOS == "windows" && os.Getenv("WT_SESSION") == "" { return false } return true } // Safe invokes the provided function and will attempt to ensure that when the // function returns (or a termination signal is sent) that the terminal state // is reset to the condition it was in prior to the function being invoked. If // t.Raw is true the terminal will be put into raw mode prior to calling the function. // If the input file descriptor is not a TTY and TryDev is true, the /dev/tty file // will be opened (if available). func (t TTY) Safe(fn SafeFunc) error { inFd, isTerminal := term.GetFdInfo(t.In) if !isTerminal && t.TryDev { if f, err := os.Open("/dev/tty"); err == nil { defer f.Close() inFd = f.Fd() isTerminal = term.IsTerminal(inFd) } } if !isTerminal { return fn() } var state *term.State var err error if t.Raw { state, err = term.MakeRaw(inFd) } else { state, err = term.SaveState(inFd) } if err != nil { return err } return Chain(t.Parent, func() { term.RestoreTerminal(inFd, state) }).Run(fn) } ================================================ FILE: mdz/pkg/version/version.go ================================================ /* Copyright The TensorChord Inc. Copyright The BuildKit Authors. Copyright The containerd Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package version import ( "fmt" "regexp" "runtime" "strings" "sync" ) var ( // Package is filled at linking time Package = "github.com/tensorchord/openmodelz/agent" HelmChartVersion = "0.0.15" // Revision is filled with the VCS (e.g. git) revision being used to build // the program at linking time. Revision = "" version = "0.0.0+unknown" buildDate = "1970-01-01T00:00:00Z" // output from `date -u +'%Y-%m-%dT%H:%M:%SZ'` gitCommit = "" // output from `git rev-parse HEAD` gitTag = "" // output from `git describe --exact-match --tags HEAD` (if clean tree state) gitTreeState = "" // determined from `git status --porcelain`. either 'clean' or 'dirty' developmentFlag = "false" ) // Version contains OpenModelz version information type Version struct { Version string BuildDate string GitCommit string GitTag string GitTreeState string GoVersion string Compiler string Platform string } func (v Version) String() string { return v.Version } // SetGitTagForE2ETest sets the gitTag for test purpose. func SetGitTagForE2ETest(tag string) { gitTag = tag } // GetOpenModelzVersion gets OpenModelz version information func GetOpenModelzVersion() string { var versionStr string if gitCommit != "" && gitTag != "" && gitTreeState == "clean" && developmentFlag == "false" { // if we have a clean tree state and the current commit is tagged, // this is an official release. versionStr = gitTag } else { // otherwise formulate a version string based on as much metadata // information we have available. if strings.HasPrefix(version, "v") { versionStr = version } else { versionStr = "v" + version } if len(gitCommit) >= 7 { versionStr += "+" + gitCommit[0:7] if gitTreeState != "clean" { versionStr += ".dirty" } } else { versionStr += "+unknown" } } return versionStr } // GetVersion returns the version information func GetVersion() Version { return Version{ Version: GetOpenModelzVersion(), BuildDate: buildDate, GitCommit: gitCommit, GitTag: gitTag, GitTreeState: gitTreeState, GoVersion: runtime.Version(), Compiler: runtime.Compiler, Platform: fmt.Sprintf("%s/%s", runtime.GOOS, runtime.GOARCH), } } var ( reRelease *regexp.Regexp reDev *regexp.Regexp reOnce sync.Once ) func UserAgent() string { version := GetVersion().String() reOnce.Do(func() { reRelease = regexp.MustCompile(`^(v[0-9]+\.[0-9]+)\.[0-9]+$`) reDev = regexp.MustCompile(`^(v[0-9]+\.[0-9]+)\.[0-9]+`) }) if matches := reRelease.FindAllStringSubmatch(version, 1); len(matches) > 0 { version = matches[0][1] } else if matches := reDev.FindAllStringSubmatch(version, 1); len(matches) > 0 { version = matches[0][1] + "-dev" } return "modelz/" + version } ================================================ FILE: modelzetes/.dockerignore ================================================ ./faas-netes /yaml /yaml_armhf /yaml_arm64 /chart /contrib /artifacts /hack /docs /.git ================================================ FILE: modelzetes/.gitattributes ================================================ yaml/* linguist-generated=true ================================================ FILE: modelzetes/.gitignore ================================================ # Binaries for programs and plugins *.exe *.dll *.so *.dylib # Test binary, build with `go test -c` *.test # Output of the go coverage tool, specifically when used with LiteIDE *.out # Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736 .glide/ .idea bin/ **/password.txt **/gateway-password.txt .vscode of_kind_portforward.pid /kind* /kubectl /yaml_armhf /yaml_arm64 /broker-* /chart/pro-builder/out /chart/pro-builder/payload.txt /pgconnector.yaml jwt_key jwt_key.pub /*.pid .tools/ ================================================ FILE: modelzetes/Dockerfile ================================================ FROM ubuntu:22.04 LABEL maintainer="modelz-support@tensorchord.ai" COPY modelzetes /usr/bin/modelzetes ENTRYPOINT ["/usr/bin/modelzetes"] ================================================ FILE: modelzetes/LICENSE ================================================ MIT License Copyright (c) 2023 TensorChord Inc. Copyright (c) 2020 OpenFaaS Author(s) Copyright (c) 2017 Alex Ellis Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: modelzetes/Makefile ================================================ # Copyright 2022 TensorChord Inc. # # The old school Makefile, following are required targets. The Makefile is written # to allow building multiple binaries. You are free to add more targets or change # existing implementations, as long as the semantics are preserved. # # make - default to 'build' target # make lint - code analysis # make test - run unit test (or plus integration test) # make build - alias to build-local target # make build-local - build local binary targets # make build-linux - build linux binary targets # make container - build containers # $ docker login registry -u username -p xxxxx # make push - push containers # make clean - clean up targets # # Not included but recommended targets: # make e2e-test # # The makefile is also responsible to populate project version information. # # # Tweak the variables based on your project. # # This repo's root import path (under GOPATH). ROOT := github.com/tensorchord/openmodelz/modelzetes # Target binaries. You can build multiple binaries for a single project. TARGETS := modelzetes # Container image prefix and suffix added to targets. # The final built images are: # $[REGISTRY]/$[IMAGE_PREFIX]$[TARGET]$[IMAGE_SUFFIX]:$[VERSION] # $[REGISTRY] is an item from $[REGISTRIES], $[TARGET] is an item from $[TARGETS]. IMAGE_PREFIX ?= $(strip ) IMAGE_SUFFIX ?= $(strip ) # Container registries. REGISTRY ?= ghcr.io/tensorchord # Container registry for base images. BASE_REGISTRY ?= docker.io BASE_REGISTRY_USER ?= modelzai # Disable CGO by default. CGO_ENABLED ?= 0 # # These variables should not need tweaking. # # It's necessary to set this because some environments don't link sh -> bash. export SHELL := bash # It's necessary to set the errexit flags for the bash shell. export SHELLOPTS := errexit PACKAGE_NAME := github.com/tensorchord/openmodelz/modelzetes GOLANG_CROSS_VERSION ?= v1.17.6 # Project main package location (can be multiple ones). CMD_DIR := ./cmd # Project output directory. OUTPUT_DIR := ./bin DEBUG_DIR := ./debug-bin # Build directory. BUILD_DIR := ./build # Current version of the project. VERSION ?= $(shell git describe --match 'v[0-9]*' --always --tags --abbrev=0) BUILD_DATE=$(shell date -u +'%Y-%m-%dT%H:%M:%SZ') GIT_COMMIT=$(shell git rev-parse HEAD) GIT_TAG=$(shell if [ -z "`git status --porcelain`" ]; then git describe --exact-match --tags HEAD 2>/dev/null; fi) GIT_TREE_STATE=$(shell if [ -z "`git status --porcelain`" ]; then echo "clean" ; else echo "dirty"; fi) GITSHA ?= $(shell git rev-parse --short HEAD) # Track code version with Docker Label. DOCKER_LABELS ?= git-describe="$(shell date -u +v%Y%m%d)-$(shell git describe --tags --always --dirty)" # Golang standard bin directory. GOPATH ?= $(shell go env GOPATH) GOROOT ?= $(shell go env GOROOT) BIN_DIR := $(GOPATH)/bin GOLANGCI_LINT := $(BIN_DIR)/golangci-lint # check if we need embed the dashboard DASHBOARD_BUILD ?= debug # Default golang flags used in build and test # -mod=vendor: force go to use the vendor files instead of using the `$GOPATH/pkg/mod` # -p: the number of programs that can be run in parallel # -count: run each test and benchmark 1 times. Set this flag to disable test cache export GOFLAGS ?= -count=1 # # Define all targets. At least the following commands are required: # # All targets. .PHONY: help lint test build container push addlicense debug debug-local build-local generate clean test-local addlicense-install release build-image .DEFAULT_GOAL:=build build: build-local ## Build the release version of envd help: ## Display this help @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z0-9_-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) debug: debug-local ## Build the debug version of envd # more info about `GOGC` env: https://github.com/golangci/golangci-lint#memory-usage-of-golangci-lint lint: $(GOLANGCI_LINT) ## Lint GO code @$(GOLANGCI_LINT) run $(GOLANGCI_LINT): curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $$(go env GOPATH)/bin mockgen-install: go install github.com/golang/mock/mockgen@v1.6.0 addlicense-install: go install github.com/google/addlicense@latest sqlc-install: go install github.com/kyleconroy/sqlc/cmd/sqlc@latest # https://github.com/swaggo/swag/pull/1322, we should use master instead of latest for now. swag-install: go install github.com/swaggo/swag/cmd/swag@v1.8.7 build-local: @for target in $(TARGETS); do \ CGO_ENABLED=$(CGO_ENABLED) GOOS=$(GOOS) GOARCH=$(GOARCH) go build -tags $(DASHBOARD_BUILD) -trimpath -v -o $(OUTPUT_DIR)/$${target} \ -ldflags "-s -w -X $(ROOT)/pkg/version.version=$(VERSION) -X $(ROOT)/pkg/version.buildDate=$(BUILD_DATE) -X $(ROOT)/pkg/version.gitCommit=$(GIT_COMMIT) -X $(ROOT)/pkg/version.gitTreeState=$(GIT_TREE_STATE)" \ $(CMD_DIR)/$${target}; \ done # It is used by vscode to attach into the process. debug-local: @for target in $(TARGETS); do \ CGO_ENABLED=$(CGO_ENABLED) go build -tags $(DASHBOARD_BUILD) -trimpath \ -v -o $(DEBUG_DIR)/$${target} \ -gcflags='all=-N -l' \ $(CMD_DIR)/$${target}; \ done addlicense: addlicense-install ## Add license to GO code files addlicense -l mpl -c "TensorChord Inc." $$(find . -type f -name '*.go' | grep -v pkg/docs/docs.go) test-local: @go test -tags=$(DASHBOARD_BUILD) -v -race -coverprofile=coverage.out ./... test: ## Run the tests @go test -tags=$(DASHBOARD_BUILD) -race -coverpkg=./pkg/... -coverprofile=coverage.out ./... @go tool cover -func coverage.out | tail -n 1 | awk '{ print "Total coverage: " $$3 }' clean: ## Clean the outputs and artifacts @-rm -vrf ${OUTPUT_DIR} @-rm -vrf ${DEBUG_DIR} @-rm -vrf build dist .eggs *.egg-info fmt: swag-install ## Run go fmt against code. go fmt ./... swag fmt vet: ## Run go vet against code. go vet ./... swag: swag-install swag init -g ./cmd/modelzetes/main.go --parseDependency --output ./pkg/docs build-image: build-local docker build -t ${BASE_REGISTRY}/${BASE_REGISTRY_USER}/modelzetes:dev -f Dockerfile ./bin docker push ${BASE_REGISTRY}/${BASE_REGISTRY_USER}/modelzetes:dev release: @if [ ! -f ".release-env" ]; then \ echo "\033[91m.release-env is required for release\033[0m";\ exit 1;\ fi docker run \ --rm \ --privileged \ -e CGO_ENABLED=1 \ --env-file .release-env \ -v /var/run/docker.sock:/var/run/docker.sock \ -v `pwd`:/go/src/$(PACKAGE_NAME) \ -v `pwd`/sysroot:/sysroot \ -w /go/src/$(PACKAGE_NAME) \ goreleaser/goreleaser-cross:${GOLANG_CROSS_VERSION} \ release --rm-dist ================================================ FILE: modelzetes/artifacts/crds/tensorchord.ai_inferences.yaml ================================================ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.5.0 creationTimestamp: null name: inferences.tensorchord.ai spec: group: tensorchord.ai names: kind: Inference listKind: InferenceList plural: inferences singular: inference scope: Namespaced versions: - additionalPrinterColumns: - jsonPath: .spec.image name: Image type: string name: v2alpha1 schema: openAPIV3Schema: description: Inference describes an Inference type: object required: - spec properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object spec: description: InferenceSpec defines the desired state of Inference type: object required: - image - name properties: annotations: description: Annotations are metadata for inferences which may be used by the faas-provider or the gateway type: object additionalProperties: type: string command: description: Command to run when starting the type: string constraints: description: Constraints are specific to the operator. type: array items: type: string envVars: description: EnvVars can be provided to set environment variables for the inference runtime. type: object additionalProperties: type: string framework: description: Framework is the inference framework. type: string http_probe_path: description: HTTPProbePath is the path of the http probe. type: string image: type: string labels: description: Labels are metadata for inferences which may be used by the faas-provider or the gateway type: object additionalProperties: type: string name: type: string port: description: Port is the port exposed by the inference. type: integer format: int32 resources: description: Limits for inference type: object properties: claims: description: "Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. \n This is an alpha field and requires enabling the DynamicResourceAllocation feature gate. \n This field is immutable. It can only be set for containers." type: array items: description: ResourceClaim references one entry in PodSpec.ResourceClaims. type: object required: - name properties: name: description: Name must match the name of one entry in pod.spec.resourceClaims of the Pod where this field is used. It makes that resource available inside a container. type: string x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map limits: description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object additionalProperties: pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ anyOf: - type: integer - type: string x-kubernetes-int-or-string: true requests: description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object additionalProperties: pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ anyOf: - type: integer - type: string x-kubernetes-int-or-string: true scaling: description: Scaling is the scaling configuration for the inference. type: object properties: max_replicas: description: MaxReplicas is the upper limit for the number of replicas to which the autoscaler can scale up. It cannot be less that minReplicas. It defaults to 1. type: integer format: int32 min_replicas: description: MinReplicas is the lower limit for the number of replicas to which the autoscaler can scale down. It defaults to 0. type: integer format: int32 startup_duration: description: StartupDuration is the duration of startup time. type: integer format: int32 target_load: description: TargetLoad is the target load. In capacity mode, it is the expected number of the inflight requests per replica. type: integer format: int32 type: description: Type is the scaling type. It can be either "capacity" or "rps". Default is "capacity". type: string zero_duration: description: ZeroDuration is the duration of zero load before scaling down to zero. Default is 5 minutes. type: integer format: int32 secrets: description: Secrets list of secrets to be made available to inference type: array items: type: string served: true storage: true subresources: {} status: acceptedNames: kind: "" plural: "" conditions: [] storedVersions: [] ================================================ FILE: modelzetes/artifacts/samples/v2alpha1.yaml ================================================ apiVersion: tensorchord.ai/v2alpha1 kind: Inference metadata: name: demo namespace: default spec: name: demo framework: mosec image: modelzai/llm-bloomz-560m:23.06.13 scaling: min_replicas: 0 max_replicas: 1 target_load: 100 type: capacity zero_duration: 60 startup_duration: 600 resources: requests: cpu: "3" memory: 12Gi ================================================ FILE: modelzetes/buf.yaml ================================================ apiVersion: v1 items: - apiVersion: tensorchord.ai/v1 kind: Inference metadata: creationTimestamp: "2023-06-02T07:37:53Z" generation: 1 labels: inference: 895f81b5-7e60-49b0-a2c1-145fc884cac7 name: 895f81b5-7e60-49b0-a2c1-145fc884cac7 namespace: modelz-a39caa1c-0b03-4054-b75f-1f1cf8424b01 resourceVersion: "76627989" uid: 8f0b842a-be19-4dae-9fa0-986e830b0670 spec: annotations: ai.tensorchord.docker.image: us-central1-docker.pkg.dev/nth-guide-378813/modelzai/mosec-imagebind:23.05.2 ai.tensorchord.domain: https://imagebind-ogeboq7ciyo3sq0t.modelz.tech ai.tensorchord.source: docker constraints: - ai.tensorchord.server-resource=nvidia-tesla-t4-4c-16g image: us-central1-docker.pkg.dev/nth-guide-378813/modelzai/mosec-imagebind:23.05.2 labels: ai.tensorchord.framework: mosec ai.tensorchord.name: imagebind ai.tensorchord.region: us-central1 ai.tensorchord.scale.max: "1" ai.tensorchord.scale.min: "0" ai.tensorchord.scale.target: "10" ai.tensorchord.scale.type: capacity ai.tensorchord.scale.zero-duration: 1m0s ai.tensorchord.server-resource: nvidia-tesla-t4-4c-16g ai.tensorchord.startup-duration: 10m0s ai.tensorchord.vendor: gcp name: 895f81b5-7e60-49b0-a2c1-145fc884cac7 resources: limits: nvidia.com/gpu: "1" requests: cpu: "3" memory: 12Gi nvidia.com/gpu: "1" - apiVersion: tensorchord.ai/v1 kind: Inference metadata: creationTimestamp: "2023-05-06T09:28:13Z" generation: 1 labels: inference: c5e72dcd-bee6-4b82-b08b-47cc9f4a2c1c name: c5e72dcd-bee6-4b82-b08b-47cc9f4a2c1c namespace: modelz-a39caa1c-0b03-4054-b75f-1f1cf8424b01 resourceVersion: "50967692" uid: 1f201ac0-ed45-4cf3-b084-47752fabfb5c spec: annotations: ai.tensorchord.docker.image: docker.io/starkind/moss:v1.11 ai.tensorchord.domain: https://moss-jmclvinro4plugtp.modelz.tech ai.tensorchord.source: docker constraints: - ai.tensorchord.server-resource=nvidia-ada-l4-8c-32g image: docker.io/starkind/moss:v1.11 labels: ai.tensorchord.framework: gradio ai.tensorchord.name: moss ai.tensorchord.port: "7860" ai.tensorchord.region: us-central1 ai.tensorchord.scale.max: "1" ai.tensorchord.scale.min: "0" ai.tensorchord.scale.target: "10" ai.tensorchord.scale.type: capacity ai.tensorchord.scale.zero-duration: 5m0s ai.tensorchord.server-resource: nvidia-ada-l4-8c-32g ai.tensorchord.startup-duration: 10m0s ai.tensorchord.vendor: gcp name: c5e72dcd-bee6-4b82-b08b-47cc9f4a2c1c resources: limits: nvidia.com/gpu: "1" requests: cpu: 6500m memory: 24Gi nvidia.com/gpu: "1" - apiVersion: tensorchord.ai/v1 kind: Inference metadata: creationTimestamp: "2023-05-15T03:41:02Z" generation: 3 labels: inference: 0b1e753e-9703-41c8-9311-b6bb943bcbda name: 0b1e753e-9703-41c8-9311-b6bb943bcbda namespace: modelz-a9d660cf-2537-4e48-bcaf-adf8470a83c0 resourceVersion: "60441705" uid: ee8818d1-1e46-4d0b-bab3-3b1893ba4703 spec: annotations: ai.tensorchord.docker.image: us-central1-docker.pkg.dev/nth-guide-378813/modelzai/gradio-stable-diffusion:23.04.4 ai.tensorchord.domain: https://sd-uif34dg3x17kb21j.modelz.tech ai.tensorchord.inference.spec: '{"name":"0b1e753e-9703-41c8-9311-b6bb943bcbda","image":"us-central1-docker.pkg.dev/nth-guide-378813/modelzai/gradio-stable-diffusion:23.04.4","envVars":{"GRADIO_SERVER_NAME":"0.0.0.0","GRADIO_SERVER_PORT":"7860","HF_ENDPOINT":"http://hfserver.default:8080"},"constraints":["ai.tensorchord.server-resource=nvidia-ada-l4-8c-32g"],"labels":{"ai.tensorchord.framework":"gradio","ai.tensorchord.name":"sd","ai.tensorchord.port":"7860","ai.tensorchord.region":"us-central1","ai.tensorchord.scale.max":"1","ai.tensorchord.scale.min":"0","ai.tensorchord.scale.target":"10","ai.tensorchord.scale.type":"capacity","ai.tensorchord.scale.zero-duration":"5m0s","ai.tensorchord.server-resource":"nvidia-ada-l4-8c-32g","ai.tensorchord.startup-duration":"10m0s","ai.tensorchord.vendor":"gcp","app":"0b1e753e-9703-41c8-9311-b6bb943bcbda","controller":"0b1e753e-9703-41c8-9311-b6bb943bcbda","inference":"0b1e753e-9703-41c8-9311-b6bb943bcbda"},"annotations":{"ai.tensorchord.docker.image":"us-central1-docker.pkg.dev/nth-guide-378813/modelzai/gradio-stable-diffusion:23.04.4","ai.tensorchord.domain":"https://sd-uif34dg3x17kb21j.modelz.tech","ai.tensorchord.inference.spec":"{\"name\":\"0b1e753e-9703-41c8-9311-b6bb943bcbda\",\"image\":\"us-central1-docker.pkg.dev/nth-guide-378813/modelzai/gradio-stable-diffusion:23.04.4\",\"constraints\":[\"ai.tensorchord.server-resource=nvidia-ada-l4-8c-32g\"],\"labels\":{\"ai.tensorchord.framework\":\"gradio\",\"ai.tensorchord.name\":\"sd\",\"ai.tensorchord.port\":\"7860\",\"ai.tensorchord.region\":\"us-central1\",\"ai.tensorchord.scale.max\":\"1\",\"ai.tensorchord.scale.min\":\"0\",\"ai.tensorchord.scale.target\":\"10\",\"ai.tensorchord.scale.type\":\"capacity\",\"ai.tensorchord.scale.zero-duration\":\"5m0s\",\"ai.tensorchord.server-resource\":\"nvidia-ada-l4-8c-32g\",\"ai.tensorchord.startup-duration\":\"10m0s\",\"ai.tensorchord.vendor\":\"gcp\"},\"annotations\":{\"ai.tensorchord.docker.image\":\"us-central1-docker.pkg.dev/nth-guide-378813/modelzai/gradio-stable-diffusion:23.04.4\",\"ai.tensorchord.domain\":\"https://sd-uif34dg3x17kb21j.modelz.tech\",\"ai.tensorchord.source\":\"docker\"},\"resources\":{\"limits\":{\"nvidia.com/gpu\":\"1\"},\"requests\":{\"cpu\":\"6500m\",\"memory\":\"24Gi\",\"nvidia.com/gpu\":\"1\"}}}","ai.tensorchord.source":"docker","prometheus.io.scrape":"false"},"resources":{"limits":{"nvidia.com/gpu":"1"},"requests":{"cpu":"6500m","memory":"24Gi","nvidia.com/gpu":"1"}}}' ai.tensorchord.source: docker prometheus.io.scrape: "false" constraints: - ai.tensorchord.server-resource=nvidia-ada-l4-8c-32g envVars: GRADIO_SERVER_NAME: 0.0.0.0 GRADIO_SERVER_PORT: "7860" HF_ENDPOINT: http://hfserver.default:8080 image: us-central1-docker.pkg.dev/nth-guide-378813/modelzai/gradio-stable-diffusion:23.04.4 labels: ai.tensorchord.framework: gradio ai.tensorchord.name: sd ai.tensorchord.port: "7860" ai.tensorchord.region: us-central1 ai.tensorchord.scale.max: "1" ai.tensorchord.scale.min: "0" ai.tensorchord.scale.target: "10" ai.tensorchord.scale.type: capacity ai.tensorchord.scale.zero-duration: 5m0s ai.tensorchord.server-resource: nvidia-ada-l4-8c-32g ai.tensorchord.startup-duration: 10m0s ai.tensorchord.vendor: gcp app: 0b1e753e-9703-41c8-9311-b6bb943bcbda controller: 0b1e753e-9703-41c8-9311-b6bb943bcbda inference: 0b1e753e-9703-41c8-9311-b6bb943bcbda name: 0b1e753e-9703-41c8-9311-b6bb943bcbda resources: limits: nvidia.com/gpu: "1" requests: cpu: 6500m memory: 24Gi nvidia.com/gpu: "1" - apiVersion: tensorchord.ai/v1 kind: Inference metadata: creationTimestamp: "2023-05-18T11:50:26Z" generation: 1 labels: inference: 2fb46857-d0ea-4237-8165-4defbe037bbd name: 2fb46857-d0ea-4237-8165-4defbe037bbd namespace: modelz-a9d660cf-2537-4e48-bcaf-adf8470a83c0 resourceVersion: "62499285" uid: 7edbbfe2-809f-4196-9ac1-fc47f731e817 spec: annotations: ai.tensorchord.docker.image: us-central1-docker.pkg.dev/nth-guide-378813/modelzai/mosec-whisper:23.04.1 ai.tensorchord.domain: https://ws-c2om23pvkpaeirf9.modelz.tech ai.tensorchord.source: docker constraints: - ai.tensorchord.server-resource=nvidia-ada-l4-8c-32g envVars: HF_HUB_OFFLINE: "true" image: us-central1-docker.pkg.dev/nth-guide-378813/modelzai/mosec-whisper:23.04.1 labels: ai.tensorchord.framework: mosec ai.tensorchord.name: ws ai.tensorchord.region: us-central1 ai.tensorchord.scale.max: "1" ai.tensorchord.scale.min: "0" ai.tensorchord.scale.target: "10" ai.tensorchord.scale.type: capacity ai.tensorchord.scale.zero-duration: 5m0s ai.tensorchord.server-resource: nvidia-ada-l4-8c-32g ai.tensorchord.startup-duration: 10m0s ai.tensorchord.vendor: gcp name: 2fb46857-d0ea-4237-8165-4defbe037bbd resources: limits: nvidia.com/gpu: "1" requests: cpu: 6500m memory: 24Gi nvidia.com/gpu: "1" - apiVersion: tensorchord.ai/v1 kind: Inference metadata: creationTimestamp: "2023-06-02T12:55:10Z" generation: 1 labels: inference: 41d0d82e-31e4-4704-9e0d-49eec10a455b name: 41d0d82e-31e4-4704-9e0d-49eec10a455b namespace: modelz-a9d660cf-2537-4e48-bcaf-adf8470a83c0 resourceVersion: "76837944" uid: b562acde-3849-4ecb-ac8e-30ba5252c7a5 spec: annotations: ai.tensorchord.docker.image: us-central1-docker.pkg.dev/nth-guide-378813/modelzai/mosec-stable-diffusion:23.04.1 ai.tensorchord.domain: https://mosec-sd-3n7j7i3oh28sp2jw.modelz.tech ai.tensorchord.source: docker ai.tensorchord.template-id: 0d603dd5-6d74-4e94-bc70-e5f223dd9d81 constraints: - ai.tensorchord.server-resource=nvidia-tesla-t4-4c-16g envVars: HF_HUB_OFFLINE: "true" image: us-central1-docker.pkg.dev/nth-guide-378813/modelzai/mosec-stable-diffusion:23.04.1 labels: ai.tensorchord.framework: mosec ai.tensorchord.name: mosec-sd ai.tensorchord.region: us-central1 ai.tensorchord.scale.max: "1" ai.tensorchord.scale.min: "0" ai.tensorchord.scale.target: "10" ai.tensorchord.scale.type: capacity ai.tensorchord.scale.zero-duration: 5m0s ai.tensorchord.server-resource: nvidia-tesla-t4-4c-16g ai.tensorchord.startup-duration: 10m0s ai.tensorchord.vendor: gcp name: 41d0d82e-31e4-4704-9e0d-49eec10a455b resources: limits: nvidia.com/gpu: "1" requests: cpu: "3" memory: 12Gi nvidia.com/gpu: "1" - apiVersion: tensorchord.ai/v1 kind: Inference metadata: creationTimestamp: "2023-06-02T12:53:20Z" generation: 1 labels: inference: 431e5913-f5f3-45dd-a1b3-139917a07cf9 name: 431e5913-f5f3-45dd-a1b3-139917a07cf9 namespace: modelz-a9d660cf-2537-4e48-bcaf-adf8470a83c0 resourceVersion: "76836748" uid: 682dec57-3eb6-4925-af3b-c56a81073009 spec: annotations: ai.tensorchord.command: python app.py ai.tensorchord.docker.image: us-central1-docker.pkg.dev/nth-guide-378813/modelzai/gradio-ramananth1-dolly-v2:23.05.1 ai.tensorchord.domain: https://ddddd2-wqkeotxfansceqp2.modelz.tech ai.tensorchord.source: docker ai.tensorchord.template-id: 074852a9-b324-4c02-95b1-da06ec98a964 constraints: - ai.tensorchord.server-resource=nvidia-ada-l4-8c-32g image: us-central1-docker.pkg.dev/nth-guide-378813/modelzai/gradio-ramananth1-dolly-v2:23.05.1 labels: ai.tensorchord.framework: gradio ai.tensorchord.name: ddddd2 ai.tensorchord.port: "7860" ai.tensorchord.region: us-central1 ai.tensorchord.scale.max: "1" ai.tensorchord.scale.min: "0" ai.tensorchord.scale.target: "10" ai.tensorchord.scale.type: capacity ai.tensorchord.scale.zero-duration: 5m0s ai.tensorchord.server-resource: nvidia-ada-l4-8c-32g ai.tensorchord.startup-duration: 10m0s ai.tensorchord.vendor: gcp name: 431e5913-f5f3-45dd-a1b3-139917a07cf9 resources: limits: nvidia.com/gpu: "1" requests: cpu: 6500m memory: 24Gi nvidia.com/gpu: "1" - apiVersion: tensorchord.ai/v1 kind: Inference metadata: creationTimestamp: "2023-06-02T07:38:54Z" generation: 1 labels: inference: 660a2a0a-d9b1-4b86-823b-4dc86f89cd02 name: 660a2a0a-d9b1-4b86-823b-4dc86f89cd02 namespace: modelz-a9d660cf-2537-4e48-bcaf-adf8470a83c0 resourceVersion: "76628665" uid: f89d820c-7c7e-43d3-a73b-cafe57627006 spec: annotations: ai.tensorchord.command: python app.py ai.tensorchord.docker.image: us-central1-docker.pkg.dev/nth-guide-378813/modelzai/gradio-ramananth1-dolly-v2:23.05.1 ai.tensorchord.domain: https://ddd-oky85wto4kxp0uyz.modelz.tech ai.tensorchord.source: docker constraints: - ai.tensorchord.server-resource=nvidia-ada-l4-8c-32g image: us-central1-docker.pkg.dev/nth-guide-378813/modelzai/gradio-ramananth1-dolly-v2:23.05.1 labels: ai.tensorchord.framework: gradio ai.tensorchord.name: ddd ai.tensorchord.port: "7860" ai.tensorchord.region: us-central1 ai.tensorchord.scale.max: "1" ai.tensorchord.scale.min: "0" ai.tensorchord.scale.target: "10" ai.tensorchord.scale.type: capacity ai.tensorchord.scale.zero-duration: 5m0s ai.tensorchord.server-resource: nvidia-ada-l4-8c-32g ai.tensorchord.startup-duration: 10m0s ai.tensorchord.vendor: gcp name: 660a2a0a-d9b1-4b86-823b-4dc86f89cd02 resources: limits: nvidia.com/gpu: "1" requests: cpu: 6500m memory: 24Gi nvidia.com/gpu: "1" - apiVersion: tensorchord.ai/v1 kind: Inference metadata: creationTimestamp: "2023-06-02T07:29:34Z" generation: 1 labels: inference: 744ef0a4-51ea-438c-b013-f5c3b1c1548f name: 744ef0a4-51ea-438c-b013-f5c3b1c1548f namespace: modelz-a9d660cf-2537-4e48-bcaf-adf8470a83c0 resourceVersion: "76622421" uid: a7ae0743-e166-4591-aec0-33a23101506d spec: annotations: ai.tensorchord.command: python app.py ai.tensorchord.docker.image: us-central1-docker.pkg.dev/nth-guide-378813/modelzai/gradio-ramananth1-dolly-v2:23.05.1 ai.tensorchord.domain: https://dollyv2-9fhuv221e3py7j8t.modelz.tech ai.tensorchord.source: docker constraints: - ai.tensorchord.server-resource=nvidia-ada-l4-8c-32g image: us-central1-docker.pkg.dev/nth-guide-378813/modelzai/gradio-ramananth1-dolly-v2:23.05.1 labels: ai.tensorchord.framework: gradio ai.tensorchord.name: dollyv2 ai.tensorchord.port: "7860" ai.tensorchord.region: us-central1 ai.tensorchord.scale.max: "1" ai.tensorchord.scale.min: "0" ai.tensorchord.scale.target: "10" ai.tensorchord.scale.type: capacity ai.tensorchord.scale.zero-duration: 5m0s ai.tensorchord.server-resource: nvidia-ada-l4-8c-32g ai.tensorchord.startup-duration: 10m0s ai.tensorchord.vendor: gcp name: 744ef0a4-51ea-438c-b013-f5c3b1c1548f resources: limits: nvidia.com/gpu: "1" requests: cpu: 6500m memory: 24Gi nvidia.com/gpu: "1" - apiVersion: tensorchord.ai/v1 kind: Inference metadata: creationTimestamp: "2023-06-02T07:30:30Z" generation: 1 labels: inference: aeaf50f9-1b7e-4a37-87a1-6947a65fc07c name: aeaf50f9-1b7e-4a37-87a1-6947a65fc07c namespace: modelz-a9d660cf-2537-4e48-bcaf-adf8470a83c0 resourceVersion: "76623043" uid: 8ce0ace2-a710-4ba0-96ad-be2f100ad8fb spec: annotations: ai.tensorchord.command: python app.py ai.tensorchord.docker.image: us-central1-docker.pkg.dev/nth-guide-378813/modelzai/gradio-ramananth1-dolly-v2:23.05.1 ai.tensorchord.domain: https://dollyv2-l3idk5o2yhzpgqmc.modelz.tech ai.tensorchord.source: docker constraints: - ai.tensorchord.server-resource=nvidia-ada-l4-8c-32g image: us-central1-docker.pkg.dev/nth-guide-378813/modelzai/gradio-ramananth1-dolly-v2:23.05.1 labels: ai.tensorchord.framework: gradio ai.tensorchord.name: dollyv2 ai.tensorchord.port: "7860" ai.tensorchord.region: us-central1 ai.tensorchord.scale.max: "1" ai.tensorchord.scale.min: "0" ai.tensorchord.scale.target: "10" ai.tensorchord.scale.type: capacity ai.tensorchord.scale.zero-duration: 5m0s ai.tensorchord.server-resource: nvidia-ada-l4-8c-32g ai.tensorchord.startup-duration: 10m0s ai.tensorchord.vendor: gcp name: aeaf50f9-1b7e-4a37-87a1-6947a65fc07c resources: limits: nvidia.com/gpu: "1" requests: cpu: 6500m memory: 24Gi nvidia.com/gpu: "1" - apiVersion: tensorchord.ai/v1 kind: Inference metadata: creationTimestamp: "2023-06-27T04:20:35Z" generation: 1 labels: inference: 8ed9973e-8bf0-45d3-8f9a-24a228e1a1f9 name: 8ed9973e-8bf0-45d3-8f9a-24a228e1a1f9 namespace: modelz-ba5ce029-180a-445d-89a8-bb111b29c0fe resourceVersion: "98647435" uid: 92b09491-086f-4be0-baf1-c4a6d82cb2c5 spec: annotations: ai.tensorchord.command: python app.py ai.tensorchord.domain: https://test-y5qie3qkgqba99fw.modelz.tech ai.tensorchord.huggingface.space: https://huggingface.co/spaces/abidlabs/en2fr ai.tensorchord.source: huggingface constraints: - ai.tensorchord.server-resource=nvidia-tesla-t4-4c-16g image: registry.hf.space/abidlabs-en2fr:latest labels: ai.tensorchord.framework: gradio ai.tensorchord.name: test ai.tensorchord.port: "7860" ai.tensorchord.region: us-central1 ai.tensorchord.scale.max: "1" ai.tensorchord.scale.min: "1" ai.tensorchord.scale.target: "10" ai.tensorchord.scale.type: capacity ai.tensorchord.scale.zero-duration: 5m0s ai.tensorchord.server-resource: nvidia-tesla-t4-4c-16g ai.tensorchord.startup-duration: 10m0s ai.tensorchord.vendor: gcp name: 8ed9973e-8bf0-45d3-8f9a-24a228e1a1f9 resources: limits: nvidia.com/gpu: "1" requests: cpu: "3" memory: 12Gi nvidia.com/gpu: "1" - apiVersion: tensorchord.ai/v1 kind: Inference metadata: creationTimestamp: "2023-06-15T06:59:52Z" generation: 1 labels: inference: b33863ba-9c0f-4edb-8f95-00a522a0c335 name: b33863ba-9c0f-4edb-8f95-00a522a0c335 namespace: modelz-ba5ce029-180a-445d-89a8-bb111b29c0fe resourceVersion: "88434776" uid: 48cf0cdc-defa-4986-ad55-13a4769252e3 spec: annotations: ai.tensorchord.docker.image: us-central1-docker.pkg.dev/nth-guide-378813/modelzai/gradio-stable-diffusion:23.04.4 ai.tensorchord.domain: https://test-gradio-yf9ws7k1yimrgrcd.modelz.tech ai.tensorchord.source: docker ai.tensorchord.template-id: 9c0137e7-5732-49f9-b6bd-e920f1c6e4d4 constraints: - ai.tensorchord.server-resource=nvidia-tesla-t4-4c-16g image: us-central1-docker.pkg.dev/nth-guide-378813/modelzai/gradio-stable-diffusion:23.04.4 labels: ai.tensorchord.framework: gradio ai.tensorchord.name: test-gradio ai.tensorchord.port: "7860" ai.tensorchord.region: us-central1 ai.tensorchord.scale.max: "1" ai.tensorchord.scale.min: "0" ai.tensorchord.scale.target: "10" ai.tensorchord.scale.type: capacity ai.tensorchord.scale.zero-duration: 5m0s ai.tensorchord.server-resource: nvidia-tesla-t4-4c-16g ai.tensorchord.startup-duration: 10m0s ai.tensorchord.vendor: gcp name: b33863ba-9c0f-4edb-8f95-00a522a0c335 resources: limits: nvidia.com/gpu: "1" requests: cpu: "3" memory: 12Gi nvidia.com/gpu: "1" - apiVersion: tensorchord.ai/v1 kind: Inference metadata: creationTimestamp: "2023-06-05T03:27:22Z" generation: 1 labels: inference: 0741b1e5-f556-439d-adc7-9cba932d2d73 name: 0741b1e5-f556-439d-adc7-9cba932d2d73 namespace: modelz-cd4a928c-7c66-4934-beb2-98dd82d672ce resourceVersion: "79293297" uid: e2183d12-afa4-443c-ab59-01af0f581aba spec: annotations: ai.tensorchord.docker.image: us-central1-docker.pkg.dev/nth-guide-378813/modelzai/mosec-whisper:23.04.1 ai.tensorchord.domain: https://whisper-9yx10apmsgdtb0di.modelz.tech ai.tensorchord.source: docker ai.tensorchord.template-id: c2974e48-7fc3-4690-8910-a2e98abc82d1 constraints: - ai.tensorchord.server-resource=nvidia-tesla-t4-4c-16g envVars: HF_HUB_OFFLINE: "true" image: us-central1-docker.pkg.dev/nth-guide-378813/modelzai/mosec-whisper:23.04.1 labels: ai.tensorchord.framework: mosec ai.tensorchord.name: whisper ai.tensorchord.region: us-central1 ai.tensorchord.scale.max: "1" ai.tensorchord.scale.min: "0" ai.tensorchord.scale.target: "10" ai.tensorchord.scale.type: capacity ai.tensorchord.scale.zero-duration: 5m0s ai.tensorchord.server-resource: nvidia-tesla-t4-4c-16g ai.tensorchord.startup-duration: 10m0s ai.tensorchord.vendor: gcp name: 0741b1e5-f556-439d-adc7-9cba932d2d73 resources: limits: nvidia.com/gpu: "1" requests: cpu: "3" memory: 12Gi nvidia.com/gpu: "1" - apiVersion: tensorchord.ai/v1 kind: Inference metadata: creationTimestamp: "2023-06-05T03:25:48Z" generation: 1 labels: inference: 3c2328a1-4c61-41b8-b670-6df5813d8c51 name: 3c2328a1-4c61-41b8-b670-6df5813d8c51 namespace: modelz-cd4a928c-7c66-4934-beb2-98dd82d672ce resourceVersion: "79292168" uid: f0da8e43-2235-4910-98d2-76621fdd2e4a spec: annotations: ai.tensorchord.command: python app.py ai.tensorchord.docker.image: us-central1-docker.pkg.dev/nth-guide-378813/modelzai/gradio-ramananth1-dolly-v2:23.05.1 ai.tensorchord.domain: https://dolly-p52q4f5qhpnayw00.modelz.tech ai.tensorchord.source: docker ai.tensorchord.template-id: 10a47537-d1cc-4aa5-a9bd-173060cf812e constraints: - ai.tensorchord.server-resource=nvidia-ada-l4-8c-32g image: us-central1-docker.pkg.dev/nth-guide-378813/modelzai/gradio-ramananth1-dolly-v2:23.05.1 labels: ai.tensorchord.framework: gradio ai.tensorchord.name: dolly ai.tensorchord.port: "7860" ai.tensorchord.region: us-central1 ai.tensorchord.scale.max: "1" ai.tensorchord.scale.min: "0" ai.tensorchord.scale.target: "10" ai.tensorchord.scale.type: capacity ai.tensorchord.scale.zero-duration: 5m0s ai.tensorchord.server-resource: nvidia-ada-l4-8c-32g ai.tensorchord.startup-duration: 10m0s ai.tensorchord.vendor: gcp name: 3c2328a1-4c61-41b8-b670-6df5813d8c51 resources: limits: nvidia.com/gpu: "1" requests: cpu: 6500m memory: 24Gi nvidia.com/gpu: "1" - apiVersion: tensorchord.ai/v1 kind: Inference metadata: creationTimestamp: "2023-06-29T03:27:54Z" generation: 1 labels: inference: 480f37dc-57b5-4677-a2e7-9e34621dc4e3 name: 480f37dc-57b5-4677-a2e7-9e34621dc4e3 namespace: modelz-cd4a928c-7c66-4934-beb2-98dd82d672ce resourceVersion: "100326009" uid: fc090985-3971-41f3-8777-6785ea328fa5 spec: annotations: ai.tensorchord.docker.image: us-central1-docker.pkg.dev/nth-guide-378813/modelzai/mosec-tts-vits:23.06.7 ai.tensorchord.domain: https://vits-bench-f5sn326og2gb20lv.modelz.tech ai.tensorchord.source: docker constraints: - ai.tensorchord.server-resource=nvidia-tesla-t4-4c-16g image: us-central1-docker.pkg.dev/nth-guide-378813/modelzai/mosec-tts-vits:23.06.7 labels: ai.tensorchord.framework: mosec ai.tensorchord.name: vits-bench ai.tensorchord.region: us-central1 ai.tensorchord.scale.max: "3" ai.tensorchord.scale.min: "0" ai.tensorchord.scale.target: "5" ai.tensorchord.scale.type: capacity ai.tensorchord.scale.zero-duration: 1m0s ai.tensorchord.server-resource: nvidia-tesla-t4-4c-16g ai.tensorchord.startup-duration: 10m0s ai.tensorchord.vendor: gcp name: 480f37dc-57b5-4677-a2e7-9e34621dc4e3 resources: limits: nvidia.com/gpu: "1" requests: cpu: "3" memory: 12Gi nvidia.com/gpu: "1" - apiVersion: tensorchord.ai/v1 kind: Inference metadata: creationTimestamp: "2023-06-05T03:26:49Z" generation: 1 labels: inference: 5f10d4e3-e9b0-46e1-9265-6d2ecb98930e name: 5f10d4e3-e9b0-46e1-9265-6d2ecb98930e namespace: modelz-cd4a928c-7c66-4934-beb2-98dd82d672ce resourceVersion: "79292878" uid: d77bb280-b397-4e41-a2f5-9bf180f61596 spec: annotations: ai.tensorchord.command: python app.py ai.tensorchord.docker.image: us-central1-docker.pkg.dev/nth-guide-378813/modelzai/gradio-stabilityai-stablelm-tuned-alpha-chat:23.05.1 ai.tensorchord.domain: https://stablelm-7p1b95fcxhyeiriy.modelz.tech ai.tensorchord.source: docker ai.tensorchord.template-id: 64462bf4-2373-4273-ae76-433707410bcd constraints: - ai.tensorchord.server-resource=nvidia-ada-l4-8c-32g image: us-central1-docker.pkg.dev/nth-guide-378813/modelzai/gradio-stabilityai-stablelm-tuned-alpha-chat:23.05.1 labels: ai.tensorchord.framework: gradio ai.tensorchord.name: stablelm ai.tensorchord.port: "7860" ai.tensorchord.region: us-central1 ai.tensorchord.scale.max: "1" ai.tensorchord.scale.min: "0" ai.tensorchord.scale.target: "10" ai.tensorchord.scale.type: capacity ai.tensorchord.scale.zero-duration: 5m0s ai.tensorchord.server-resource: nvidia-ada-l4-8c-32g ai.tensorchord.startup-duration: 10m0s ai.tensorchord.vendor: gcp name: 5f10d4e3-e9b0-46e1-9265-6d2ecb98930e resources: limits: nvidia.com/gpu: "1" requests: cpu: 6500m memory: 24Gi nvidia.com/gpu: "1" - apiVersion: tensorchord.ai/v1 kind: Inference metadata: creationTimestamp: "2023-06-05T03:27:12Z" generation: 1 labels: inference: 81343170-1688-491c-a139-87037f3b8dcd name: 81343170-1688-491c-a139-87037f3b8dcd namespace: modelz-cd4a928c-7c66-4934-beb2-98dd82d672ce resourceVersion: "79293174" uid: 8d63f8de-7bf6-40a8-8997-fa69fce037d0 spec: annotations: ai.tensorchord.docker.image: us-central1-docker.pkg.dev/nth-guide-378813/modelzai/gradio-visual-chatgpt:23.04.1 ai.tensorchord.domain: https://visual-chatgpt-0e0iez8r17xzxksf.modelz.tech ai.tensorchord.source: docker ai.tensorchord.template-id: 78100725-18ca-41e9-b7a8-7ac573f6280b constraints: - ai.tensorchord.server-resource=nvidia-tesla-t4-4c-16g envVars: OPENAI_API_KEY: a image: us-central1-docker.pkg.dev/nth-guide-378813/modelzai/gradio-visual-chatgpt:23.04.1 labels: ai.tensorchord.framework: gradio ai.tensorchord.name: visual-chatgpt ai.tensorchord.port: "7860" ai.tensorchord.region: us-central1 ai.tensorchord.scale.max: "1" ai.tensorchord.scale.min: "0" ai.tensorchord.scale.target: "10" ai.tensorchord.scale.type: capacity ai.tensorchord.scale.zero-duration: 5m0s ai.tensorchord.server-resource: nvidia-tesla-t4-4c-16g ai.tensorchord.startup-duration: 10m0s ai.tensorchord.vendor: gcp name: 81343170-1688-491c-a139-87037f3b8dcd resources: limits: nvidia.com/gpu: "1" requests: cpu: "3" memory: 12Gi nvidia.com/gpu: "1" - apiVersion: tensorchord.ai/v1 kind: Inference metadata: creationTimestamp: "2023-06-09T10:50:48Z" generation: 1 labels: inference: 8e365318-53ed-449e-ac15-f7ab8750d79f name: 8e365318-53ed-449e-ac15-f7ab8750d79f namespace: modelz-cd4a928c-7c66-4934-beb2-98dd82d672ce resourceVersion: "83183128" uid: 5fca43a9-a9e4-4bd0-8b9b-5ffa5b730ab8 spec: annotations: ai.tensorchord.docker.image: us-central1-docker.pkg.dev/nth-guide-378813/modelzai/llm-bloomz-560m:23.06.12 ai.tensorchord.domain: https://a-k7l6t2w8osvf4trq.modelz.tech ai.tensorchord.source: docker ai.tensorchord.template-id: 92721d83-2dba-460f-9bde-b7eee4ea4950 constraints: - ai.tensorchord.server-resource=cpu-4c-16g image: us-central1-docker.pkg.dev/nth-guide-378813/modelzai/llm-bloomz-560m:23.06.12 labels: ai.tensorchord.framework: other ai.tensorchord.name: a ai.tensorchord.region: us-central1 ai.tensorchord.scale.max: "1" ai.tensorchord.scale.min: "0" ai.tensorchord.scale.target: "10" ai.tensorchord.scale.type: capacity ai.tensorchord.scale.zero-duration: 5m0s ai.tensorchord.server-resource: cpu-4c-16g ai.tensorchord.startup-duration: 10m0s ai.tensorchord.vendor: gcp name: 8e365318-53ed-449e-ac15-f7ab8750d79f resources: requests: cpu: "3" memory: 8Gi - apiVersion: tensorchord.ai/v1 kind: Inference metadata: creationTimestamp: "2023-06-19T09:15:48Z" generation: 1 labels: inference: c6637a2e-0812-4600-b9d5-d1fbc01a68f8 name: c6637a2e-0812-4600-b9d5-d1fbc01a68f8 namespace: modelz-cd4a928c-7c66-4934-beb2-98dd82d672ce resourceVersion: "92057903" uid: b59cdece-e083-40bc-8a39-25d30ad9a753 spec: annotations: ai.tensorchord.command: python app.py ai.tensorchord.domain: https://hf-9oln0cnymw5eyzzf.modelz.tech ai.tensorchord.huggingface.space: https://huggingface.co/spaces/HuggingFaceH4/falcon-chat ai.tensorchord.source: huggingface constraints: - ai.tensorchord.server-resource=cpu-4c-16g image: registry.hf.space/huggingfaceh4-falcon-chat:latest labels: ai.tensorchord.framework: gradio ai.tensorchord.name: hf ai.tensorchord.port: "7860" ai.tensorchord.region: us-central1 ai.tensorchord.scale.max: "1" ai.tensorchord.scale.min: "0" ai.tensorchord.scale.target: "10" ai.tensorchord.scale.type: capacity ai.tensorchord.scale.zero-duration: 5m0s ai.tensorchord.server-resource: cpu-4c-16g ai.tensorchord.startup-duration: 10m0s ai.tensorchord.vendor: gcp name: c6637a2e-0812-4600-b9d5-d1fbc01a68f8 resources: requests: cpu: "3" memory: 8Gi - apiVersion: tensorchord.ai/v1 kind: Inference metadata: creationTimestamp: "2023-06-05T03:26:36Z" generation: 1 labels: inference: c7f18c10-4c99-471c-b9d3-dd906e02cc7f name: c7f18c10-4c99-471c-b9d3-dd906e02cc7f namespace: modelz-cd4a928c-7c66-4934-beb2-98dd82d672ce resourceVersion: "79292725" uid: 36d9a032-2cda-4abb-ad8e-aad182e6850f spec: annotations: ai.tensorchord.docker.image: us-central1-docker.pkg.dev/nth-guide-378813/modelzai/gradio-stable-diffusion:23.04.4 ai.tensorchord.domain: https://sd-web-co4ubmbo9k6c751q.modelz.tech ai.tensorchord.source: docker ai.tensorchord.template-id: 755b5ba6-ce50-4ef9-bbe0-e00c3cecb380 constraints: - ai.tensorchord.server-resource=nvidia-tesla-t4-4c-16g image: us-central1-docker.pkg.dev/nth-guide-378813/modelzai/gradio-stable-diffusion:23.04.4 labels: ai.tensorchord.framework: gradio ai.tensorchord.name: sd-web ai.tensorchord.port: "7860" ai.tensorchord.region: us-central1 ai.tensorchord.scale.max: "1" ai.tensorchord.scale.min: "0" ai.tensorchord.scale.target: "10" ai.tensorchord.scale.type: capacity ai.tensorchord.scale.zero-duration: 5m0s ai.tensorchord.server-resource: nvidia-tesla-t4-4c-16g ai.tensorchord.startup-duration: 10m0s ai.tensorchord.vendor: gcp name: c7f18c10-4c99-471c-b9d3-dd906e02cc7f resources: limits: nvidia.com/gpu: "1" requests: cpu: "3" memory: 12Gi nvidia.com/gpu: "1" - apiVersion: tensorchord.ai/v1 kind: Inference metadata: creationTimestamp: "2023-06-05T03:26:16Z" generation: 1 labels: inference: d0c06992-94b5-4a99-90b4-a9900335acfe name: d0c06992-94b5-4a99-90b4-a9900335acfe namespace: modelz-cd4a928c-7c66-4934-beb2-98dd82d672ce resourceVersion: "79292483" uid: 42b1485b-9ed6-49ed-81b7-fb9742931300 spec: annotations: ai.tensorchord.command: python app.py ai.tensorchord.docker.image: us-central1-docker.pkg.dev/nth-guide-378813/modelzai/gradio-microsoft-hugginggpt:23.05.1 ai.tensorchord.domain: https://huggingpt-5vi5th3l7llhgq4h.modelz.tech ai.tensorchord.source: docker ai.tensorchord.template-id: 8dbec890-ff7b-467e-ada2-e37f055f5c4f constraints: - ai.tensorchord.server-resource=nvidia-ada-l4-8c-32g image: us-central1-docker.pkg.dev/nth-guide-378813/modelzai/gradio-microsoft-hugginggpt:23.05.1 labels: ai.tensorchord.framework: gradio ai.tensorchord.name: huggingpt ai.tensorchord.port: "7860" ai.tensorchord.region: us-central1 ai.tensorchord.scale.max: "1" ai.tensorchord.scale.min: "0" ai.tensorchord.scale.target: "10" ai.tensorchord.scale.type: capacity ai.tensorchord.scale.zero-duration: 5m0s ai.tensorchord.server-resource: nvidia-ada-l4-8c-32g ai.tensorchord.startup-duration: 10m0s ai.tensorchord.vendor: gcp name: d0c06992-94b5-4a99-90b4-a9900335acfe resources: limits: nvidia.com/gpu: "1" requests: cpu: 6500m memory: 24Gi nvidia.com/gpu: "1" - apiVersion: tensorchord.ai/v1 kind: Inference metadata: creationTimestamp: "2023-06-05T03:26:26Z" generation: 1 labels: inference: f3d6bedf-447d-4f9b-b27f-1f03f43a1f97 name: f3d6bedf-447d-4f9b-b27f-1f03f43a1f97 namespace: modelz-cd4a928c-7c66-4934-beb2-98dd82d672ce resourceVersion: "79292608" uid: 68ee40dd-6366-4229-941a-16a5c019956d spec: annotations: ai.tensorchord.docker.image: us-central1-docker.pkg.dev/nth-guide-378813/modelzai/mosec-stable-diffusion:23.04.1 ai.tensorchord.domain: https://sd-rl41fip7qafvrjzl.modelz.tech ai.tensorchord.source: docker ai.tensorchord.template-id: 6fe4cfb5-4e8b-4d34-ba03-de534343361e constraints: - ai.tensorchord.server-resource=nvidia-tesla-t4-4c-16g envVars: HF_HUB_OFFLINE: "true" image: us-central1-docker.pkg.dev/nth-guide-378813/modelzai/mosec-stable-diffusion:23.04.1 labels: ai.tensorchord.framework: mosec ai.tensorchord.name: sd ai.tensorchord.region: us-central1 ai.tensorchord.scale.max: "1" ai.tensorchord.scale.min: "0" ai.tensorchord.scale.target: "10" ai.tensorchord.scale.type: capacity ai.tensorchord.scale.zero-duration: 5m0s ai.tensorchord.server-resource: nvidia-tesla-t4-4c-16g ai.tensorchord.startup-duration: 10m0s ai.tensorchord.vendor: gcp name: f3d6bedf-447d-4f9b-b27f-1f03f43a1f97 resources: limits: nvidia.com/gpu: "1" requests: cpu: "3" memory: 12Gi nvidia.com/gpu: "1" - apiVersion: tensorchord.ai/v1 kind: Inference metadata: creationTimestamp: "2023-05-26T08:34:40Z" generation: 2 labels: inference: 94e62a6d-adde-4ff2-9053-7b15b6f18727 name: 94e62a6d-adde-4ff2-9053-7b15b6f18727 namespace: modelz-d3524a71-c17c-4c92-8faf-8603f02f4713 resourceVersion: "99387184" uid: 5b11d701-b474-4cad-8fff-bf107c5b3992 spec: annotations: ai.tensorchord.docker.image: us-central1-docker.pkg.dev/nth-guide-378813/modelzai/mosec-whisper:23.04.1 ai.tensorchord.domain: https://demo-e9ijwrk2il0kvzl7.modelz.tech ai.tensorchord.inference.spec: '{"name":"94e62a6d-adde-4ff2-9053-7b15b6f18727","image":"us-central1-docker.pkg.dev/nth-guide-378813/modelzai/mosec-whisper:23.04.1","envVars":{"HF_HUB_OFFLINE":"true"},"constraints":["ai.tensorchord.server-resource=nvidia-tesla-t4-4c-16g"],"labels":{"ai.tensorchord.framework":"mosec","ai.tensorchord.name":"demo","ai.tensorchord.region":"us-central1","ai.tensorchord.scale.max":"1","ai.tensorchord.scale.min":"0","ai.tensorchord.scale.target":"10","ai.tensorchord.scale.type":"capacity","ai.tensorchord.scale.zero-duration":"5m0s","ai.tensorchord.server-resource":"nvidia-tesla-t4-4c-16g","ai.tensorchord.startup-duration":"10m0s","ai.tensorchord.vendor":"gcp"},"annotations":{"ai.tensorchord.docker.image":"us-central1-docker.pkg.dev/nth-guide-378813/modelzai/mosec-whisper:23.04.1","ai.tensorchord.domain":"https://demo-e9ijwrk2il0kvzl7.modelz.tech","ai.tensorchord.source":"docker"},"resources":{"limits":{"nvidia.com/gpu":"1"},"requests":{"cpu":"3","memory":"12Gi","nvidia.com/gpu":"1"}}}' ai.tensorchord.source: docker prometheus.io.scrape: "false" constraints: - ai.tensorchord.server-resource=nvidia-tesla-t4-4c-16g envVars: HF_ENDPOINT: http://hfserver.default:8080 HF_HUB_OFFLINE: "true" MOSEC_PORT: "8080" image: us-central1-docker.pkg.dev/nth-guide-378813/modelzai/mosec-whisper:23.04.1 labels: ai.tensorchord.framework: mosec ai.tensorchord.name: demo ai.tensorchord.region: us-central1 ai.tensorchord.scale.max: "0" ai.tensorchord.scale.min: "0" ai.tensorchord.scale.target: "10" ai.tensorchord.scale.type: capacity ai.tensorchord.scale.zero-duration: 5m0s ai.tensorchord.server-resource: nvidia-tesla-t4-4c-16g ai.tensorchord.startup-duration: 10m0s ai.tensorchord.vendor: gcp app: 94e62a6d-adde-4ff2-9053-7b15b6f18727 controller: 94e62a6d-adde-4ff2-9053-7b15b6f18727 inference: 94e62a6d-adde-4ff2-9053-7b15b6f18727 name: 94e62a6d-adde-4ff2-9053-7b15b6f18727 resources: limits: nvidia.com/gpu: "1" requests: cpu: "3" memory: 12Gi nvidia.com/gpu: "1" kind: List metadata: resourceVersion: "" selfLink: "" ================================================ FILE: modelzetes/cmd/modelzetes/main.go ================================================ package main import ( "fmt" "os" cli "github.com/urfave/cli/v2" "k8s.io/klog" "github.com/tensorchord/openmodelz/modelzetes/pkg/app" "github.com/tensorchord/openmodelz/modelzetes/pkg/version" ) func run(args []string) error { cli.VersionPrinter = func(c *cli.Context) { fmt.Println(c.App.Name, version.Package, c.App.Version, version.Revision) } klog.InitFlags(nil) a := app.New() return a.Run(args) } func handleErr(err error) { if err == nil { return } klog.Error(err) os.Exit(1) } func main() { err := run(os.Args) handleErr(err) } ================================================ FILE: modelzetes/hack/boilerplate.go.txt ================================================ /* Copyright 2019-2023 TensorChord Inc. Licensed under the MIT license. See LICENSE file in the project root for full license information. */ ================================================ FILE: modelzetes/hack/print-codegen-version.sh ================================================ #!/bin/bash # This scripts exists primarily so that it can be used in the Makefile. # It is needed because the `($shell ...)` command was having issues with the pipe. # Extracting it to a script was the simplest solution. grep 'k8s.io/code-generator' go.mod | awk '{print $2}' ================================================ FILE: modelzetes/hack/update-codegen.sh ================================================ #!/usr/bin/env bash # copied from: https://github.com/weaveworks/flagger/tree/master/hack set -o errexit set -o nounset set -o pipefail SCRIPT_ROOT=$(git rev-parse --show-toplevel)/modelzetes echo ">> SCRIPT_ROOT ${SCRIPT_ROOT}" GET_PKG_LOCATION() { pkg_name="${1:-}" pkg_location="$(go list -m -f '{{.Dir}}' "${pkg_name}" 2>/dev/null)" if [ "${pkg_location}" = "" ]; then echo "${pkg_name} is missing. Running 'go mod download'." go mod download pkg_location=$(go list -m -f '{{.Dir}}' "${pkg_name}") fi echo "${pkg_location}" } # Grab code-generator version from go.sum CODEGEN_PKG="$(GET_PKG_LOCATION "k8s.io/code-generator")" echo ">> Using ${CODEGEN_PKG}" # Grab openapi-gen version from go.mod OPENAPI_PKG="$(GET_PKG_LOCATION 'k8s.io/kube-openapi')" echo ">> Using ${OPENAPI_PKG}" echo ">> Using ${CODEGEN_PKG}" # code-generator does work with go.mod but makes assumptions about # the project living in `$GOPATH/src`. To work around this and support # any location; create a temporary directory, use this as an output # base, and copy everything back once generated. TEMP_DIR=$(mktemp -d) cleanup() { echo ">> Removing ${TEMP_DIR}" rm -rf ${TEMP_DIR} } trap "cleanup" EXIT SIGINT echo ">> Temporary output directory ${TEMP_DIR}" # Ensure we can execute. chmod +x ${CODEGEN_PKG}/generate-groups.sh ${CODEGEN_PKG}/generate-groups.sh all \ github.com/tensorchord/openmodelz/modelzetes/pkg/client github.com/tensorchord/openmodelz/modelzetes/pkg/apis \ modelzetes:v2alpha1 \ --output-base "${TEMP_DIR}" \ --go-header-file ${SCRIPT_ROOT}/hack/boilerplate.go.txt # Copy everything back. cp -r "${TEMP_DIR}/github.com/tensorchord/openmodelz/modelzetes/." "${SCRIPT_ROOT}/" ================================================ FILE: modelzetes/hack/update-crds.sh ================================================ #!/bin/bash export controllergen="$GOPATH/bin/controller-gen" export PKG=sigs.k8s.io/controller-tools/cmd/controller-gen@v0.7.0 if [ ! -e "$controllergen" ]; then echo "Getting $PKG" go install $PKG fi "$controllergen" \ crd \ schemapatch:manifests=./artifacts/crds \ paths=./pkg/apis/modelzetes/v2alpha1 \ output:dir=./artifacts/crds ================================================ FILE: modelzetes/hack/verify-codegen.sh ================================================ #!/usr/bin/env bash set -o errexit set -o nounset set -o pipefail SCRIPT_ROOT=$(git rev-parse --show-toplevel)/modelzetes DIFFROOT="${SCRIPT_ROOT}/pkg" TMP_DIFFROOT="${SCRIPT_ROOT}/_tmp/pkg" _tmp="${SCRIPT_ROOT}/_tmp" cleanup() { rm -rf "${_tmp}" } trap "cleanup" EXIT SIGINT cleanup mkdir -p "${TMP_DIFFROOT}" cp -a "${DIFFROOT}"/* "${TMP_DIFFROOT}" "${SCRIPT_ROOT}/hack/update-codegen.sh" echo "diffing ${DIFFROOT} against freshly generated codegen" ret=0 diff -Naupr "${DIFFROOT}" "${TMP_DIFFROOT}" || ret=$? cp -a "${TMP_DIFFROOT}"/* "${DIFFROOT}" if [[ $ret -eq 0 ]] then echo "${DIFFROOT} up to date." else echo "${DIFFROOT} is out of date. Please run hack/update-codegen.sh" exit 1 fi ================================================ FILE: modelzetes/pkg/apis/modelzetes/register.go ================================================ package modelzetes const ( GroupName = "tensorchord.ai" ) ================================================ FILE: modelzetes/pkg/apis/modelzetes/v2alpha1/doc.go ================================================ // +k8s:deepcopy-gen=package,register // Package v2alpha1 is the modelzetes API. // +groupName=tensorchord.ai package v2alpha1 ================================================ FILE: modelzetes/pkg/apis/modelzetes/v2alpha1/register.go ================================================ package v2alpha1 import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" controller "github.com/tensorchord/openmodelz/modelzetes/pkg/apis/modelzetes" ) // SchemeGroupVersion is group version used to register these objects var SchemeGroupVersion = schema.GroupVersion{Group: controller.GroupName, Version: "v2alpha1"} // Resource takes an unqualified resource and returns a Group qualified GroupResource func Resource(resource string) schema.GroupResource { return SchemeGroupVersion.WithResource(resource).GroupResource() } var ( // localSchemeBuilder and AddToScheme will stay in k8s.io/kubernetes. SchemeBuilder runtime.SchemeBuilder localSchemeBuilder = &SchemeBuilder AddToScheme = localSchemeBuilder.AddToScheme Kind = "Inference" ) func init() { // We only register manually written functions here. The registration of the // generated functions takes place in the generated files. The separation // makes the code compile even when the generated files are missing. localSchemeBuilder.Register(addKnownTypes) } // Adds the list of known types to api.Scheme. func addKnownTypes(scheme *runtime.Scheme) error { scheme.AddKnownTypes(SchemeGroupVersion, &Inference{}, &InferenceList{}, ) metav1.AddToGroupVersion(scheme, SchemeGroupVersion) return nil } ================================================ FILE: modelzetes/pkg/apis/modelzetes/v2alpha1/types.go ================================================ package v2alpha1 import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // +genclient // +genclient:noStatus // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object // +kubebuilder:printcolumn:name="Image",type=string,JSONPath=`.spec.image` // Inference describes an Inference type Inference struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` Spec InferenceSpec `json:"spec"` } // InferenceSpec defines the desired state of Inference type InferenceSpec struct { Name string `json:"name"` Image string `json:"image"` // Scaling is the scaling configuration for the inference. Scaling *ScalingConfig `json:"scaling,omitempty"` // Framework is the inference framework. Framework Framework `json:"framework,omitempty"` // Port is the port exposed by the inference. Port *int32 `json:"port,omitempty"` // HTTPProbePath is the path of the http probe. HTTPProbePath *string `json:"http_probe_path,omitempty"` // Command to run when starting the Command *string `json:"command,omitempty"` // EnvVars can be provided to set environment variables for the inference runtime. EnvVars map[string]string `json:"envVars,omitempty"` // Constraints are specific to the operator. Constraints []string `json:"constraints,omitempty"` // Secrets list of secrets to be made available to inference Secrets []string `json:"secrets,omitempty"` // Labels are metadata for inferences which may be used by the // faas-provider or the gateway Labels map[string]string `json:"labels,omitempty"` // Annotations are metadata for inferences which may be used by the // faas-provider or the gateway Annotations map[string]string `json:"annotations,omitempty"` // Limits for inference Resources *v1.ResourceRequirements `json:"resources,omitempty"` } // Framework is the inference framework. It is only used to set the default port // and command. For example, if the framework is "gradio", the default port is // 7860 and the default command is "python app.py". You could override these // defaults by setting the port and command fields and framework to `other`. type Framework string const ( FrameworkGradio Framework = "gradio" FrameworkStreamlit Framework = "streamlit" FrameworkMosec Framework = "mosec" FrameworkOther Framework = "other" ) type ScalingConfig struct { // MinReplicas is the lower limit for the number of replicas to which the // autoscaler can scale down. It defaults to 0. MinReplicas *int32 `json:"min_replicas,omitempty"` // MaxReplicas is the upper limit for the number of replicas to which the // autoscaler can scale up. It cannot be less that minReplicas. It defaults // to 1. MaxReplicas *int32 `json:"max_replicas,omitempty"` // TargetLoad is the target load. In capacity mode, it is the expected number of the inflight requests per replica. TargetLoad *int32 `json:"target_load,omitempty"` // Type is the scaling type. It can be either "capacity" or "rps". Default is "capacity". Type *ScalingType `json:"type,omitempty"` // ZeroDuration is the duration of zero load before scaling down to zero. Default is 5 minutes. ZeroDuration *int32 `json:"zero_duration,omitempty"` // StartupDuration is the duration of startup time. StartupDuration *int32 `json:"startup_duration,omitempty"` } type ScalingType string const ( ScalingTypeCapacity ScalingType = "capacity" ScalingTypeRPS ScalingType = "rps" ) // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object // InferenceList is a list of inference resources type InferenceList struct { metav1.TypeMeta `json:",inline"` metav1.ListMeta `json:"metadata"` Items []Inference `json:"items"` } ================================================ FILE: modelzetes/pkg/apis/modelzetes/v2alpha1/zz_generated.deepcopy.go ================================================ //go:build !ignore_autogenerated // +build !ignore_autogenerated /* Copyright 2019-2023 TensorChord Inc. Licensed under the MIT license. See LICENSE file in the project root for full license information. */ // Code generated by deepcopy-gen. DO NOT EDIT. package v2alpha1 import ( v1 "k8s.io/api/core/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Inference) DeepCopyInto(out *Inference) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) in.Spec.DeepCopyInto(&out.Spec) return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Inference. func (in *Inference) DeepCopy() *Inference { if in == nil { return nil } out := new(Inference) in.DeepCopyInto(out) return out } // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. func (in *Inference) DeepCopyObject() runtime.Object { if c := in.DeepCopy(); c != nil { return c } return nil } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *InferenceList) DeepCopyInto(out *InferenceList) { *out = *in out.TypeMeta = in.TypeMeta in.ListMeta.DeepCopyInto(&out.ListMeta) if in.Items != nil { in, out := &in.Items, &out.Items *out = make([]Inference, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } } return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InferenceList. func (in *InferenceList) DeepCopy() *InferenceList { if in == nil { return nil } out := new(InferenceList) in.DeepCopyInto(out) return out } // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. func (in *InferenceList) DeepCopyObject() runtime.Object { if c := in.DeepCopy(); c != nil { return c } return nil } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *InferenceSpec) DeepCopyInto(out *InferenceSpec) { *out = *in if in.Scaling != nil { in, out := &in.Scaling, &out.Scaling *out = new(ScalingConfig) (*in).DeepCopyInto(*out) } if in.Port != nil { in, out := &in.Port, &out.Port *out = new(int32) **out = **in } if in.HTTPProbePath != nil { in, out := &in.HTTPProbePath, &out.HTTPProbePath *out = new(string) **out = **in } if in.Command != nil { in, out := &in.Command, &out.Command *out = new(string) **out = **in } if in.EnvVars != nil { in, out := &in.EnvVars, &out.EnvVars *out = make(map[string]string, len(*in)) for key, val := range *in { (*out)[key] = val } } if in.Constraints != nil { in, out := &in.Constraints, &out.Constraints *out = make([]string, len(*in)) copy(*out, *in) } if in.Secrets != nil { in, out := &in.Secrets, &out.Secrets *out = make([]string, len(*in)) copy(*out, *in) } if in.Labels != nil { in, out := &in.Labels, &out.Labels *out = make(map[string]string, len(*in)) for key, val := range *in { (*out)[key] = val } } if in.Annotations != nil { in, out := &in.Annotations, &out.Annotations *out = make(map[string]string, len(*in)) for key, val := range *in { (*out)[key] = val } } if in.Resources != nil { in, out := &in.Resources, &out.Resources *out = new(v1.ResourceRequirements) (*in).DeepCopyInto(*out) } return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InferenceSpec. func (in *InferenceSpec) DeepCopy() *InferenceSpec { if in == nil { return nil } out := new(InferenceSpec) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ScalingConfig) DeepCopyInto(out *ScalingConfig) { *out = *in if in.MinReplicas != nil { in, out := &in.MinReplicas, &out.MinReplicas *out = new(int32) **out = **in } if in.MaxReplicas != nil { in, out := &in.MaxReplicas, &out.MaxReplicas *out = new(int32) **out = **in } if in.TargetLoad != nil { in, out := &in.TargetLoad, &out.TargetLoad *out = new(int32) **out = **in } if in.Type != nil { in, out := &in.Type, &out.Type *out = new(ScalingType) **out = **in } if in.ZeroDuration != nil { in, out := &in.ZeroDuration, &out.ZeroDuration *out = new(int32) **out = **in } if in.StartupDuration != nil { in, out := &in.StartupDuration, &out.StartupDuration *out = new(int32) **out = **in } return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ScalingConfig. func (in *ScalingConfig) DeepCopy() *ScalingConfig { if in == nil { return nil } out := new(ScalingConfig) in.DeepCopyInto(out) return out } ================================================ FILE: modelzetes/pkg/app/config.go ================================================ package app import ( cli "github.com/urfave/cli/v2" "github.com/tensorchord/openmodelz/modelzetes/pkg/config" ) func configFromCLI(c *cli.Context) config.Config { cfg := config.Config{} // kubernetes cfg.KubeConfig.Kubeconfig = c.String(flagKubeConfig) cfg.KubeConfig.MasterURL = c.String(flagMasterURL) cfg.KubeConfig.QPS = c.Int(flagQPS) cfg.KubeConfig.Burst = c.Int(flagBurst) cfg.KubeConfig.ResyncPeriod = c.Duration(flagResyncPeriod) // controller cfg.Controller.ThreadCount = c.Int(flagControllerThreads) // metrics cfg.Metrics.ServerPort = c.Int(flagMetricsServerPort) // huggingface cfg.HuggingfaceProxy.Endpoint = c.String(flagHuggingfaceEndpoint) // probes cfg.Probes.Readiness.InitialDelaySeconds = c.Int(flagProbeReadinessInitialDelaySeconds) cfg.Probes.Readiness.PeriodSeconds = c.Int(flagProbeReadinessPeriodSeconds) cfg.Probes.Readiness.TimeoutSeconds = c.Int(flagProbeReadinessTimeoutSeconds) cfg.Probes.Liveness.InitialDelaySeconds = c.Int(flagProbeLivenessInitialDelaySeconds) cfg.Probes.Liveness.PeriodSeconds = c.Int(flagProbeLivenessPeriodSeconds) cfg.Probes.Liveness.TimeoutSeconds = c.Int(flagProbeLivenessTimeoutSeconds) cfg.Probes.Startup.InitialDelaySeconds = c.Int(flagProbeStartupInitialDelaySeconds) cfg.Probes.Startup.PeriodSeconds = c.Int(flagProbeStartupPeriodSeconds) cfg.Probes.Startup.TimeoutSeconds = c.Int(flagProbeStartupTimeoutSeconds) // inference cfg.Inference.ImagePullPolicy = c.String(flagInferenceImagePullPolicy) cfg.Inference.SetUpRuntimeClassNvidia = c.Bool(flagInferenceSetUpRuntimeClassNvidia) return cfg } ================================================ FILE: modelzetes/pkg/app/root.go ================================================ // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. package app import ( "flag" "time" "github.com/cockroachdb/errors" cli "github.com/urfave/cli/v2" "k8s.io/klog" "github.com/tensorchord/openmodelz/modelzetes/pkg/controller" "github.com/tensorchord/openmodelz/modelzetes/pkg/signals" "github.com/tensorchord/openmodelz/modelzetes/pkg/version" ) const ( flagDebug = "debug" // metrics flagMetricsServerPort = "metrics-server-port" // kubernetes flagMasterURL = "master-url" flagKubeConfig = "kube-config" flagQPS = "kube-qps" flagBurst = "kube-burst" flagResyncPeriod = "kube-resync-period" // controller flagControllerThreads = "controller-thread-count" // huggingface flagHuggingfaceEndpoint = "huggingface-endpoint" // probes flagProbeReadinessInitialDelaySeconds = "probe-readiness-initial-delay-seconds" flagProbeReadinessPeriodSeconds = "probe-readiness-period-seconds" flagProbeReadinessTimeoutSeconds = "probe-readiness-timeout-seconds" flagProbeLivenessInitialDelaySeconds = "probe-liveness-initial-delay-seconds" flagProbeLivenessPeriodSeconds = "probe-liveness-period-seconds" flagProbeLivenessTimeoutSeconds = "probe-liveness-timeout-seconds" flagProbeStartupInitialDelaySeconds = "probe-startup-initial-delay-seconds" flagProbeStartupPeriodSeconds = "probe-startup-period-seconds" flagProbeStartupTimeoutSeconds = "probe-startup-timeout-seconds" // inference flagInferenceImagePullPolicy = "inference-image-pull-policy" flagInferenceSetUpRuntimeClassNvidia = "inference-set-up-runtime-class-nvidia" ) type App struct { *cli.App } func New() App { internalApp := cli.NewApp() internalApp.EnableBashCompletion = true internalApp.Name = "modelzetes" internalApp.Usage = "kubernetes operator for modelz" internalApp.HideHelpCommand = true internalApp.HideVersion = false internalApp.Version = version.GetVersion().String() internalApp.Flags = []cli.Flag{ &cli.BoolFlag{ Name: flagDebug, Usage: "enable debug output in logs", EnvVars: []string{"DEBUG"}, }, &cli.IntFlag{ Name: flagMetricsServerPort, Value: 8081, Usage: "port to listen on", EnvVars: []string{"MODELZETES_SERVER_PORT"}, Aliases: []string{"p"}, }, &cli.StringFlag{ Name: flagMasterURL, Usage: "URL to master for kubernetes cluster", EnvVars: []string{"MODELZETES_MASTER_URL"}, Aliases: []string{"mu"}, }, &cli.StringFlag{ Name: flagKubeConfig, Usage: "Path to kubeconfig file. If not provided, will use in-cluster config", EnvVars: []string{"MODELZETES_KUBE_CONFIG"}, Aliases: []string{"kc"}, }, &cli.IntFlag{ Name: flagQPS, Usage: "QPS for kubernetes client", Value: 100, EnvVars: []string{"MODELZETES_KUBE_QPS"}, Aliases: []string{"kq"}, }, &cli.IntFlag{ Name: flagBurst, Value: 250, Usage: "Burst for kubernetes client", EnvVars: []string{"MODELZETES_KUBE_BURST"}, Aliases: []string{"kb"}, }, &cli.DurationFlag{ Name: flagResyncPeriod, Value: time.Minute * 5, Usage: "Resync period for kubernetes client", EnvVars: []string{"MODELZETES_KUBE_RESYNC_PERIOD"}, Aliases: []string{"kr"}, }, &cli.IntFlag{ Name: flagControllerThreads, Value: 1, Usage: "Number of threads to use for controller", EnvVars: []string{"MODELZETES_CONTROLLER_THREAD_COUNT"}, Aliases: []string{"ct"}, }, &cli.StringFlag{ Name: flagHuggingfaceEndpoint, Usage: "Endpoint for huggingface modelz API. If not provided, will use " + "https://huggingface.co by default", EnvVars: []string{"MODELZETES_HUGGINGFACE_ENDPOINT"}, Aliases: []string{"he"}, }, &cli.IntFlag{ Name: flagProbeReadinessInitialDelaySeconds, Value: 2, Usage: "Initial delay for readiness probe", EnvVars: []string{"MODELZETES_PROBE_READINESS_INITIAL_DELAY_SECONDS"}, Aliases: []string{"prids"}, }, &cli.IntFlag{ Name: flagProbeReadinessPeriodSeconds, Value: 1, Usage: "Period for readiness probe", EnvVars: []string{"MODELZETES_PROBE_READINESS_PERIOD_SECONDS"}, Aliases: []string{"prps"}, }, &cli.IntFlag{ Name: flagProbeReadinessTimeoutSeconds, Value: 1, Usage: "Timeout for readiness probe", EnvVars: []string{"MODELZETES_PROBE_READINESS_TIMEOUT_SECONDS"}, Aliases: []string{"prts"}, }, &cli.IntFlag{ Name: flagProbeLivenessInitialDelaySeconds, Value: 2, Usage: "Initial delay for liveness probe", EnvVars: []string{"MODELZETES_PROBE_LIVENESS_INITIAL_DELAY_SECONDS"}, Aliases: []string{"plids"}, }, &cli.IntFlag{ Name: flagProbeLivenessPeriodSeconds, Value: 1, Usage: "Period for liveness probe", EnvVars: []string{"MODELZETES_PROBE_LIVENESS_PERIOD_SECONDS"}, Aliases: []string{"plps"}, }, &cli.IntFlag{ Name: flagProbeLivenessTimeoutSeconds, Value: 1, Usage: "Timeout for liveness probe", EnvVars: []string{"MODELZETES_PROBE_LIVENESS_TIMEOUT_SECONDS"}, Aliases: []string{"plts"}, }, &cli.IntFlag{ Name: flagProbeStartupInitialDelaySeconds, Value: 0, Usage: "Initial delay for startup probe", EnvVars: []string{"MODELZETES_PROBE_STARTUP_INITIAL_DELAY_SECONDS"}, Aliases: []string{"psids"}, }, &cli.IntFlag{ Name: flagProbeStartupPeriodSeconds, Value: 2, Usage: "Period for startup probe", EnvVars: []string{"MODELZETES_PROBE_STARTUP_PERIOD_SECONDS"}, Aliases: []string{"psps"}, }, &cli.IntFlag{ Name: flagProbeStartupTimeoutSeconds, Value: 1, Usage: "Timeout for startup probe", EnvVars: []string{"MODELZETES_PROBE_STARTUP_TIMEOUT_SECONDS"}, Aliases: []string{"psts"}, }, &cli.StringFlag{ Name: flagInferenceImagePullPolicy, Usage: "Image pull policy for inference service.", Value: "IfNotPresent", EnvVars: []string{"MODELZETES_INFERENCE_IMAGE_PULL_POLICY"}, Aliases: []string{"iipp"}, }, &cli.BoolFlag{ Name: flagInferenceSetUpRuntimeClassNvidia, Usage: "If true, will set up the Nvidia RuntimeClassName to the inference deployment.", EnvVars: []string{"MODELZETES_INFERENCE_SET_UP_RUNTIME_CLASS_NVIDIA"}, }, } internalApp.Action = runServer // Deal with debug flag. var debugEnabled bool internalApp.Before = func(context *cli.Context) error { debugEnabled = context.Bool(flagDebug) fs := flag.NewFlagSet("", flag.PanicOnError) klog.InitFlags(fs) if debugEnabled { fs.Set("v", "10") } else { fs.Set("v", "0") } return nil } return App{ App: internalApp, } } func runServer(clicontext *cli.Context) error { c := configFromCLI(clicontext) cfgString, _ := c.GetString() klog.V(0).Info("config: ", cfgString) if err := c.Validate(); err != nil { if clicontext.Bool(flagDebug) { return errors.Wrap(err, "invalid config: "+cfgString) } else { return errors.Wrap(err, "invalid config") } } // set up signals so we handle the first shutdown signal gracefully stopCh := signals.SetupSignalHandler() s, err := controller.New(c, stopCh) if err != nil { return errors.Wrap(err, "failed to create server") } return s.Run(c.Controller.ThreadCount, stopCh) } ================================================ FILE: modelzetes/pkg/client/clientset/versioned/clientset.go ================================================ /* Copyright 2019-2023 TensorChord Inc. Licensed under the MIT license. See LICENSE file in the project root for full license information. */ // Code generated by client-gen. DO NOT EDIT. package versioned import ( "fmt" "net/http" tensorchordv2alpha1 "github.com/tensorchord/openmodelz/modelzetes/pkg/client/clientset/versioned/typed/modelzetes/v2alpha1" discovery "k8s.io/client-go/discovery" rest "k8s.io/client-go/rest" flowcontrol "k8s.io/client-go/util/flowcontrol" ) type Interface interface { Discovery() discovery.DiscoveryInterface TensorchordV2alpha1() tensorchordv2alpha1.TensorchordV2alpha1Interface } // Clientset contains the clients for groups. type Clientset struct { *discovery.DiscoveryClient tensorchordV2alpha1 *tensorchordv2alpha1.TensorchordV2alpha1Client } // TensorchordV2alpha1 retrieves the TensorchordV2alpha1Client func (c *Clientset) TensorchordV2alpha1() tensorchordv2alpha1.TensorchordV2alpha1Interface { return c.tensorchordV2alpha1 } // Discovery retrieves the DiscoveryClient func (c *Clientset) Discovery() discovery.DiscoveryInterface { if c == nil { return nil } return c.DiscoveryClient } // NewForConfig creates a new Clientset for the given config. // If config's RateLimiter is not set and QPS and Burst are acceptable, // NewForConfig will generate a rate-limiter in configShallowCopy. // NewForConfig is equivalent to NewForConfigAndClient(c, httpClient), // where httpClient was generated with rest.HTTPClientFor(c). func NewForConfig(c *rest.Config) (*Clientset, error) { configShallowCopy := *c if configShallowCopy.UserAgent == "" { configShallowCopy.UserAgent = rest.DefaultKubernetesUserAgent() } // share the transport between all clients httpClient, err := rest.HTTPClientFor(&configShallowCopy) if err != nil { return nil, err } return NewForConfigAndClient(&configShallowCopy, httpClient) } // NewForConfigAndClient creates a new Clientset for the given config and http client. // Note the http client provided takes precedence over the configured transport values. // If config's RateLimiter is not set and QPS and Burst are acceptable, // NewForConfigAndClient will generate a rate-limiter in configShallowCopy. func NewForConfigAndClient(c *rest.Config, httpClient *http.Client) (*Clientset, error) { configShallowCopy := *c if configShallowCopy.RateLimiter == nil && configShallowCopy.QPS > 0 { if configShallowCopy.Burst <= 0 { return nil, fmt.Errorf("burst is required to be greater than 0 when RateLimiter is not set and QPS is set to greater than 0") } configShallowCopy.RateLimiter = flowcontrol.NewTokenBucketRateLimiter(configShallowCopy.QPS, configShallowCopy.Burst) } var cs Clientset var err error cs.tensorchordV2alpha1, err = tensorchordv2alpha1.NewForConfigAndClient(&configShallowCopy, httpClient) if err != nil { return nil, err } cs.DiscoveryClient, err = discovery.NewDiscoveryClientForConfigAndClient(&configShallowCopy, httpClient) if err != nil { return nil, err } return &cs, nil } // NewForConfigOrDie creates a new Clientset for the given config and // panics if there is an error in the config. func NewForConfigOrDie(c *rest.Config) *Clientset { cs, err := NewForConfig(c) if err != nil { panic(err) } return cs } // New creates a new Clientset for the given RESTClient. func New(c rest.Interface) *Clientset { var cs Clientset cs.tensorchordV2alpha1 = tensorchordv2alpha1.New(c) cs.DiscoveryClient = discovery.NewDiscoveryClient(c) return &cs } ================================================ FILE: modelzetes/pkg/client/clientset/versioned/doc.go ================================================ /* Copyright 2019-2023 TensorChord Inc. Licensed under the MIT license. See LICENSE file in the project root for full license information. */ // Code generated by client-gen. DO NOT EDIT. // This package has the automatically generated clientset. package versioned ================================================ FILE: modelzetes/pkg/client/clientset/versioned/fake/clientset_generated.go ================================================ /* Copyright 2019-2023 TensorChord Inc. Licensed under the MIT license. See LICENSE file in the project root for full license information. */ // Code generated by client-gen. DO NOT EDIT. package fake import ( clientset "github.com/tensorchord/openmodelz/modelzetes/pkg/client/clientset/versioned" tensorchordv2alpha1 "github.com/tensorchord/openmodelz/modelzetes/pkg/client/clientset/versioned/typed/modelzetes/v2alpha1" faketensorchordv2alpha1 "github.com/tensorchord/openmodelz/modelzetes/pkg/client/clientset/versioned/typed/modelzetes/v2alpha1/fake" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/discovery" fakediscovery "k8s.io/client-go/discovery/fake" "k8s.io/client-go/testing" ) // NewSimpleClientset returns a clientset that will respond with the provided objects. // It's backed by a very simple object tracker that processes creates, updates and deletions as-is, // without applying any validations and/or defaults. It shouldn't be considered a replacement // for a real clientset and is mostly useful in simple unit tests. func NewSimpleClientset(objects ...runtime.Object) *Clientset { o := testing.NewObjectTracker(scheme, codecs.UniversalDecoder()) for _, obj := range objects { if err := o.Add(obj); err != nil { panic(err) } } cs := &Clientset{tracker: o} cs.discovery = &fakediscovery.FakeDiscovery{Fake: &cs.Fake} cs.AddReactor("*", "*", testing.ObjectReaction(o)) cs.AddWatchReactor("*", func(action testing.Action) (handled bool, ret watch.Interface, err error) { gvr := action.GetResource() ns := action.GetNamespace() watch, err := o.Watch(gvr, ns) if err != nil { return false, nil, err } return true, watch, nil }) return cs } // Clientset implements clientset.Interface. Meant to be embedded into a // struct to get a default implementation. This makes faking out just the method // you want to test easier. type Clientset struct { testing.Fake discovery *fakediscovery.FakeDiscovery tracker testing.ObjectTracker } func (c *Clientset) Discovery() discovery.DiscoveryInterface { return c.discovery } func (c *Clientset) Tracker() testing.ObjectTracker { return c.tracker } var ( _ clientset.Interface = &Clientset{} _ testing.FakeClient = &Clientset{} ) // TensorchordV2alpha1 retrieves the TensorchordV2alpha1Client func (c *Clientset) TensorchordV2alpha1() tensorchordv2alpha1.TensorchordV2alpha1Interface { return &faketensorchordv2alpha1.FakeTensorchordV2alpha1{Fake: &c.Fake} } ================================================ FILE: modelzetes/pkg/client/clientset/versioned/fake/doc.go ================================================ /* Copyright 2019-2023 TensorChord Inc. Licensed under the MIT license. See LICENSE file in the project root for full license information. */ // Code generated by client-gen. DO NOT EDIT. // This package has the automatically generated fake clientset. package fake ================================================ FILE: modelzetes/pkg/client/clientset/versioned/fake/register.go ================================================ /* Copyright 2019-2023 TensorChord Inc. Licensed under the MIT license. See LICENSE file in the project root for full license information. */ // Code generated by client-gen. DO NOT EDIT. package fake import ( tensorchordv2alpha1 "github.com/tensorchord/openmodelz/modelzetes/pkg/apis/modelzetes/v2alpha1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" schema "k8s.io/apimachinery/pkg/runtime/schema" serializer "k8s.io/apimachinery/pkg/runtime/serializer" utilruntime "k8s.io/apimachinery/pkg/util/runtime" ) var scheme = runtime.NewScheme() var codecs = serializer.NewCodecFactory(scheme) var localSchemeBuilder = runtime.SchemeBuilder{ tensorchordv2alpha1.AddToScheme, } // AddToScheme adds all types of this clientset into the given scheme. This allows composition // of clientsets, like in: // // import ( // "k8s.io/client-go/kubernetes" // clientsetscheme "k8s.io/client-go/kubernetes/scheme" // aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme" // ) // // kclientset, _ := kubernetes.NewForConfig(c) // _ = aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme) // // After this, RawExtensions in Kubernetes types will serialize kube-aggregator types // correctly. var AddToScheme = localSchemeBuilder.AddToScheme func init() { v1.AddToGroupVersion(scheme, schema.GroupVersion{Version: "v1"}) utilruntime.Must(AddToScheme(scheme)) } ================================================ FILE: modelzetes/pkg/client/clientset/versioned/scheme/doc.go ================================================ /* Copyright 2019-2023 TensorChord Inc. Licensed under the MIT license. See LICENSE file in the project root for full license information. */ // Code generated by client-gen. DO NOT EDIT. // This package contains the scheme of the automatically generated clientset. package scheme ================================================ FILE: modelzetes/pkg/client/clientset/versioned/scheme/register.go ================================================ /* Copyright 2019-2023 TensorChord Inc. Licensed under the MIT license. See LICENSE file in the project root for full license information. */ // Code generated by client-gen. DO NOT EDIT. package scheme import ( tensorchordv2alpha1 "github.com/tensorchord/openmodelz/modelzetes/pkg/apis/modelzetes/v2alpha1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" schema "k8s.io/apimachinery/pkg/runtime/schema" serializer "k8s.io/apimachinery/pkg/runtime/serializer" utilruntime "k8s.io/apimachinery/pkg/util/runtime" ) var Scheme = runtime.NewScheme() var Codecs = serializer.NewCodecFactory(Scheme) var ParameterCodec = runtime.NewParameterCodec(Scheme) var localSchemeBuilder = runtime.SchemeBuilder{ tensorchordv2alpha1.AddToScheme, } // AddToScheme adds all types of this clientset into the given scheme. This allows composition // of clientsets, like in: // // import ( // "k8s.io/client-go/kubernetes" // clientsetscheme "k8s.io/client-go/kubernetes/scheme" // aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme" // ) // // kclientset, _ := kubernetes.NewForConfig(c) // _ = aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme) // // After this, RawExtensions in Kubernetes types will serialize kube-aggregator types // correctly. var AddToScheme = localSchemeBuilder.AddToScheme func init() { v1.AddToGroupVersion(Scheme, schema.GroupVersion{Version: "v1"}) utilruntime.Must(AddToScheme(Scheme)) } ================================================ FILE: modelzetes/pkg/client/clientset/versioned/typed/modelzetes/v2alpha1/doc.go ================================================ /* Copyright 2019-2023 TensorChord Inc. Licensed under the MIT license. See LICENSE file in the project root for full license information. */ // Code generated by client-gen. DO NOT EDIT. // This package has the automatically generated typed clients. package v2alpha1 ================================================ FILE: modelzetes/pkg/client/clientset/versioned/typed/modelzetes/v2alpha1/fake/doc.go ================================================ /* Copyright 2019-2023 TensorChord Inc. Licensed under the MIT license. See LICENSE file in the project root for full license information. */ // Code generated by client-gen. DO NOT EDIT. // Package fake has the automatically generated clients. package fake ================================================ FILE: modelzetes/pkg/client/clientset/versioned/typed/modelzetes/v2alpha1/fake/fake_inference.go ================================================ /* Copyright 2019-2023 TensorChord Inc. Licensed under the MIT license. See LICENSE file in the project root for full license information. */ // Code generated by client-gen. DO NOT EDIT. package fake import ( "context" v2alpha1 "github.com/tensorchord/openmodelz/modelzetes/pkg/apis/modelzetes/v2alpha1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" labels "k8s.io/apimachinery/pkg/labels" schema "k8s.io/apimachinery/pkg/runtime/schema" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" testing "k8s.io/client-go/testing" ) // FakeInferences implements InferenceInterface type FakeInferences struct { Fake *FakeTensorchordV2alpha1 ns string } var inferencesResource = schema.GroupVersionResource{Group: "tensorchord.ai", Version: "v2alpha1", Resource: "inferences"} var inferencesKind = schema.GroupVersionKind{Group: "tensorchord.ai", Version: "v2alpha1", Kind: "Inference"} // Get takes name of the inference, and returns the corresponding inference object, and an error if there is any. func (c *FakeInferences) Get(ctx context.Context, name string, options v1.GetOptions) (result *v2alpha1.Inference, err error) { obj, err := c.Fake. Invokes(testing.NewGetAction(inferencesResource, c.ns, name), &v2alpha1.Inference{}) if obj == nil { return nil, err } return obj.(*v2alpha1.Inference), err } // List takes label and field selectors, and returns the list of Inferences that match those selectors. func (c *FakeInferences) List(ctx context.Context, opts v1.ListOptions) (result *v2alpha1.InferenceList, err error) { obj, err := c.Fake. Invokes(testing.NewListAction(inferencesResource, inferencesKind, c.ns, opts), &v2alpha1.InferenceList{}) if obj == nil { return nil, err } label, _, _ := testing.ExtractFromListOptions(opts) if label == nil { label = labels.Everything() } list := &v2alpha1.InferenceList{ListMeta: obj.(*v2alpha1.InferenceList).ListMeta} for _, item := range obj.(*v2alpha1.InferenceList).Items { if label.Matches(labels.Set(item.Labels)) { list.Items = append(list.Items, item) } } return list, err } // Watch returns a watch.Interface that watches the requested inferences. func (c *FakeInferences) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { return c.Fake. InvokesWatch(testing.NewWatchAction(inferencesResource, c.ns, opts)) } // Create takes the representation of a inference and creates it. Returns the server's representation of the inference, and an error, if there is any. func (c *FakeInferences) Create(ctx context.Context, inference *v2alpha1.Inference, opts v1.CreateOptions) (result *v2alpha1.Inference, err error) { obj, err := c.Fake. Invokes(testing.NewCreateAction(inferencesResource, c.ns, inference), &v2alpha1.Inference{}) if obj == nil { return nil, err } return obj.(*v2alpha1.Inference), err } // Update takes the representation of a inference and updates it. Returns the server's representation of the inference, and an error, if there is any. func (c *FakeInferences) Update(ctx context.Context, inference *v2alpha1.Inference, opts v1.UpdateOptions) (result *v2alpha1.Inference, err error) { obj, err := c.Fake. Invokes(testing.NewUpdateAction(inferencesResource, c.ns, inference), &v2alpha1.Inference{}) if obj == nil { return nil, err } return obj.(*v2alpha1.Inference), err } // Delete takes name of the inference and deletes it. Returns an error if one occurs. func (c *FakeInferences) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { _, err := c.Fake. Invokes(testing.NewDeleteActionWithOptions(inferencesResource, c.ns, name, opts), &v2alpha1.Inference{}) return err } // DeleteCollection deletes a collection of objects. func (c *FakeInferences) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { action := testing.NewDeleteCollectionAction(inferencesResource, c.ns, listOpts) _, err := c.Fake.Invokes(action, &v2alpha1.InferenceList{}) return err } // Patch applies the patch and returns the patched inference. func (c *FakeInferences) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v2alpha1.Inference, err error) { obj, err := c.Fake. Invokes(testing.NewPatchSubresourceAction(inferencesResource, c.ns, name, pt, data, subresources...), &v2alpha1.Inference{}) if obj == nil { return nil, err } return obj.(*v2alpha1.Inference), err } ================================================ FILE: modelzetes/pkg/client/clientset/versioned/typed/modelzetes/v2alpha1/fake/fake_modelzetes_client.go ================================================ /* Copyright 2019-2023 TensorChord Inc. Licensed under the MIT license. See LICENSE file in the project root for full license information. */ // Code generated by client-gen. DO NOT EDIT. package fake import ( v2alpha1 "github.com/tensorchord/openmodelz/modelzetes/pkg/client/clientset/versioned/typed/modelzetes/v2alpha1" rest "k8s.io/client-go/rest" testing "k8s.io/client-go/testing" ) type FakeTensorchordV2alpha1 struct { *testing.Fake } func (c *FakeTensorchordV2alpha1) Inferences(namespace string) v2alpha1.InferenceInterface { return &FakeInferences{c, namespace} } // RESTClient returns a RESTClient that is used to communicate // with API server by this client implementation. func (c *FakeTensorchordV2alpha1) RESTClient() rest.Interface { var ret *rest.RESTClient return ret } ================================================ FILE: modelzetes/pkg/client/clientset/versioned/typed/modelzetes/v2alpha1/generated_expansion.go ================================================ /* Copyright 2019-2023 TensorChord Inc. Licensed under the MIT license. See LICENSE file in the project root for full license information. */ // Code generated by client-gen. DO NOT EDIT. package v2alpha1 type InferenceExpansion interface{} ================================================ FILE: modelzetes/pkg/client/clientset/versioned/typed/modelzetes/v2alpha1/inference.go ================================================ /* Copyright 2019-2023 TensorChord Inc. Licensed under the MIT license. See LICENSE file in the project root for full license information. */ // Code generated by client-gen. DO NOT EDIT. package v2alpha1 import ( "context" "time" v2alpha1 "github.com/tensorchord/openmodelz/modelzetes/pkg/apis/modelzetes/v2alpha1" scheme "github.com/tensorchord/openmodelz/modelzetes/pkg/client/clientset/versioned/scheme" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" rest "k8s.io/client-go/rest" ) // InferencesGetter has a method to return a InferenceInterface. // A group's client should implement this interface. type InferencesGetter interface { Inferences(namespace string) InferenceInterface } // InferenceInterface has methods to work with Inference resources. type InferenceInterface interface { Create(ctx context.Context, inference *v2alpha1.Inference, opts v1.CreateOptions) (*v2alpha1.Inference, error) Update(ctx context.Context, inference *v2alpha1.Inference, opts v1.UpdateOptions) (*v2alpha1.Inference, error) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error Get(ctx context.Context, name string, opts v1.GetOptions) (*v2alpha1.Inference, error) List(ctx context.Context, opts v1.ListOptions) (*v2alpha1.InferenceList, error) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v2alpha1.Inference, err error) InferenceExpansion } // inferences implements InferenceInterface type inferences struct { client rest.Interface ns string } // newInferences returns a Inferences func newInferences(c *TensorchordV2alpha1Client, namespace string) *inferences { return &inferences{ client: c.RESTClient(), ns: namespace, } } // Get takes name of the inference, and returns the corresponding inference object, and an error if there is any. func (c *inferences) Get(ctx context.Context, name string, options v1.GetOptions) (result *v2alpha1.Inference, err error) { result = &v2alpha1.Inference{} err = c.client.Get(). Namespace(c.ns). Resource("inferences"). Name(name). VersionedParams(&options, scheme.ParameterCodec). Do(ctx). Into(result) return } // List takes label and field selectors, and returns the list of Inferences that match those selectors. func (c *inferences) List(ctx context.Context, opts v1.ListOptions) (result *v2alpha1.InferenceList, err error) { var timeout time.Duration if opts.TimeoutSeconds != nil { timeout = time.Duration(*opts.TimeoutSeconds) * time.Second } result = &v2alpha1.InferenceList{} err = c.client.Get(). Namespace(c.ns). Resource("inferences"). VersionedParams(&opts, scheme.ParameterCodec). Timeout(timeout). Do(ctx). Into(result) return } // Watch returns a watch.Interface that watches the requested inferences. func (c *inferences) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { var timeout time.Duration if opts.TimeoutSeconds != nil { timeout = time.Duration(*opts.TimeoutSeconds) * time.Second } opts.Watch = true return c.client.Get(). Namespace(c.ns). Resource("inferences"). VersionedParams(&opts, scheme.ParameterCodec). Timeout(timeout). Watch(ctx) } // Create takes the representation of a inference and creates it. Returns the server's representation of the inference, and an error, if there is any. func (c *inferences) Create(ctx context.Context, inference *v2alpha1.Inference, opts v1.CreateOptions) (result *v2alpha1.Inference, err error) { result = &v2alpha1.Inference{} err = c.client.Post(). Namespace(c.ns). Resource("inferences"). VersionedParams(&opts, scheme.ParameterCodec). Body(inference). Do(ctx). Into(result) return } // Update takes the representation of a inference and updates it. Returns the server's representation of the inference, and an error, if there is any. func (c *inferences) Update(ctx context.Context, inference *v2alpha1.Inference, opts v1.UpdateOptions) (result *v2alpha1.Inference, err error) { result = &v2alpha1.Inference{} err = c.client.Put(). Namespace(c.ns). Resource("inferences"). Name(inference.Name). VersionedParams(&opts, scheme.ParameterCodec). Body(inference). Do(ctx). Into(result) return } // Delete takes name of the inference and deletes it. Returns an error if one occurs. func (c *inferences) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { return c.client.Delete(). Namespace(c.ns). Resource("inferences"). Name(name). Body(&opts). Do(ctx). Error() } // DeleteCollection deletes a collection of objects. func (c *inferences) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { var timeout time.Duration if listOpts.TimeoutSeconds != nil { timeout = time.Duration(*listOpts.TimeoutSeconds) * time.Second } return c.client.Delete(). Namespace(c.ns). Resource("inferences"). VersionedParams(&listOpts, scheme.ParameterCodec). Timeout(timeout). Body(&opts). Do(ctx). Error() } // Patch applies the patch and returns the patched inference. func (c *inferences) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v2alpha1.Inference, err error) { result = &v2alpha1.Inference{} err = c.client.Patch(pt). Namespace(c.ns). Resource("inferences"). Name(name). SubResource(subresources...). VersionedParams(&opts, scheme.ParameterCodec). Body(data). Do(ctx). Into(result) return } ================================================ FILE: modelzetes/pkg/client/clientset/versioned/typed/modelzetes/v2alpha1/modelzetes_client.go ================================================ /* Copyright 2019-2023 TensorChord Inc. Licensed under the MIT license. See LICENSE file in the project root for full license information. */ // Code generated by client-gen. DO NOT EDIT. package v2alpha1 import ( "net/http" v2alpha1 "github.com/tensorchord/openmodelz/modelzetes/pkg/apis/modelzetes/v2alpha1" "github.com/tensorchord/openmodelz/modelzetes/pkg/client/clientset/versioned/scheme" rest "k8s.io/client-go/rest" ) type TensorchordV2alpha1Interface interface { RESTClient() rest.Interface InferencesGetter } // TensorchordV2alpha1Client is used to interact with features provided by the tensorchord.ai group. type TensorchordV2alpha1Client struct { restClient rest.Interface } func (c *TensorchordV2alpha1Client) Inferences(namespace string) InferenceInterface { return newInferences(c, namespace) } // NewForConfig creates a new TensorchordV2alpha1Client for the given config. // NewForConfig is equivalent to NewForConfigAndClient(c, httpClient), // where httpClient was generated with rest.HTTPClientFor(c). func NewForConfig(c *rest.Config) (*TensorchordV2alpha1Client, error) { config := *c if err := setConfigDefaults(&config); err != nil { return nil, err } httpClient, err := rest.HTTPClientFor(&config) if err != nil { return nil, err } return NewForConfigAndClient(&config, httpClient) } // NewForConfigAndClient creates a new TensorchordV2alpha1Client for the given config and http client. // Note the http client provided takes precedence over the configured transport values. func NewForConfigAndClient(c *rest.Config, h *http.Client) (*TensorchordV2alpha1Client, error) { config := *c if err := setConfigDefaults(&config); err != nil { return nil, err } client, err := rest.RESTClientForConfigAndClient(&config, h) if err != nil { return nil, err } return &TensorchordV2alpha1Client{client}, nil } // NewForConfigOrDie creates a new TensorchordV2alpha1Client for the given config and // panics if there is an error in the config. func NewForConfigOrDie(c *rest.Config) *TensorchordV2alpha1Client { client, err := NewForConfig(c) if err != nil { panic(err) } return client } // New creates a new TensorchordV2alpha1Client for the given RESTClient. func New(c rest.Interface) *TensorchordV2alpha1Client { return &TensorchordV2alpha1Client{c} } func setConfigDefaults(config *rest.Config) error { gv := v2alpha1.SchemeGroupVersion config.GroupVersion = &gv config.APIPath = "/apis" config.NegotiatedSerializer = scheme.Codecs.WithoutConversion() if config.UserAgent == "" { config.UserAgent = rest.DefaultKubernetesUserAgent() } return nil } // RESTClient returns a RESTClient that is used to communicate // with API server by this client implementation. func (c *TensorchordV2alpha1Client) RESTClient() rest.Interface { if c == nil { return nil } return c.restClient } ================================================ FILE: modelzetes/pkg/client/informers/externalversions/factory.go ================================================ /* Copyright 2019-2023 TensorChord Inc. Licensed under the MIT license. See LICENSE file in the project root for full license information. */ // Code generated by informer-gen. DO NOT EDIT. package externalversions import ( reflect "reflect" sync "sync" time "time" versioned "github.com/tensorchord/openmodelz/modelzetes/pkg/client/clientset/versioned" internalinterfaces "github.com/tensorchord/openmodelz/modelzetes/pkg/client/informers/externalversions/internalinterfaces" modelzetes "github.com/tensorchord/openmodelz/modelzetes/pkg/client/informers/externalversions/modelzetes" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" schema "k8s.io/apimachinery/pkg/runtime/schema" cache "k8s.io/client-go/tools/cache" ) // SharedInformerOption defines the functional option type for SharedInformerFactory. type SharedInformerOption func(*sharedInformerFactory) *sharedInformerFactory type sharedInformerFactory struct { client versioned.Interface namespace string tweakListOptions internalinterfaces.TweakListOptionsFunc lock sync.Mutex defaultResync time.Duration customResync map[reflect.Type]time.Duration informers map[reflect.Type]cache.SharedIndexInformer // startedInformers is used for tracking which informers have been started. // This allows Start() to be called multiple times safely. startedInformers map[reflect.Type]bool // wg tracks how many goroutines were started. wg sync.WaitGroup // shuttingDown is true when Shutdown has been called. It may still be running // because it needs to wait for goroutines. shuttingDown bool } // WithCustomResyncConfig sets a custom resync period for the specified informer types. func WithCustomResyncConfig(resyncConfig map[v1.Object]time.Duration) SharedInformerOption { return func(factory *sharedInformerFactory) *sharedInformerFactory { for k, v := range resyncConfig { factory.customResync[reflect.TypeOf(k)] = v } return factory } } // WithTweakListOptions sets a custom filter on all listers of the configured SharedInformerFactory. func WithTweakListOptions(tweakListOptions internalinterfaces.TweakListOptionsFunc) SharedInformerOption { return func(factory *sharedInformerFactory) *sharedInformerFactory { factory.tweakListOptions = tweakListOptions return factory } } // WithNamespace limits the SharedInformerFactory to the specified namespace. func WithNamespace(namespace string) SharedInformerOption { return func(factory *sharedInformerFactory) *sharedInformerFactory { factory.namespace = namespace return factory } } // NewSharedInformerFactory constructs a new instance of sharedInformerFactory for all namespaces. func NewSharedInformerFactory(client versioned.Interface, defaultResync time.Duration) SharedInformerFactory { return NewSharedInformerFactoryWithOptions(client, defaultResync) } // NewFilteredSharedInformerFactory constructs a new instance of sharedInformerFactory. // Listers obtained via this SharedInformerFactory will be subject to the same filters // as specified here. // Deprecated: Please use NewSharedInformerFactoryWithOptions instead func NewFilteredSharedInformerFactory(client versioned.Interface, defaultResync time.Duration, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) SharedInformerFactory { return NewSharedInformerFactoryWithOptions(client, defaultResync, WithNamespace(namespace), WithTweakListOptions(tweakListOptions)) } // NewSharedInformerFactoryWithOptions constructs a new instance of a SharedInformerFactory with additional options. func NewSharedInformerFactoryWithOptions(client versioned.Interface, defaultResync time.Duration, options ...SharedInformerOption) SharedInformerFactory { factory := &sharedInformerFactory{ client: client, namespace: v1.NamespaceAll, defaultResync: defaultResync, informers: make(map[reflect.Type]cache.SharedIndexInformer), startedInformers: make(map[reflect.Type]bool), customResync: make(map[reflect.Type]time.Duration), } // Apply all options for _, opt := range options { factory = opt(factory) } return factory } func (f *sharedInformerFactory) Start(stopCh <-chan struct{}) { f.lock.Lock() defer f.lock.Unlock() if f.shuttingDown { return } for informerType, informer := range f.informers { if !f.startedInformers[informerType] { f.wg.Add(1) // We need a new variable in each loop iteration, // otherwise the goroutine would use the loop variable // and that keeps changing. informer := informer go func() { defer f.wg.Done() informer.Run(stopCh) }() f.startedInformers[informerType] = true } } } func (f *sharedInformerFactory) Shutdown() { f.lock.Lock() f.shuttingDown = true f.lock.Unlock() // Will return immediately if there is nothing to wait for. f.wg.Wait() } func (f *sharedInformerFactory) WaitForCacheSync(stopCh <-chan struct{}) map[reflect.Type]bool { informers := func() map[reflect.Type]cache.SharedIndexInformer { f.lock.Lock() defer f.lock.Unlock() informers := map[reflect.Type]cache.SharedIndexInformer{} for informerType, informer := range f.informers { if f.startedInformers[informerType] { informers[informerType] = informer } } return informers }() res := map[reflect.Type]bool{} for informType, informer := range informers { res[informType] = cache.WaitForCacheSync(stopCh, informer.HasSynced) } return res } // InternalInformerFor returns the SharedIndexInformer for obj using an internal // client. func (f *sharedInformerFactory) InformerFor(obj runtime.Object, newFunc internalinterfaces.NewInformerFunc) cache.SharedIndexInformer { f.lock.Lock() defer f.lock.Unlock() informerType := reflect.TypeOf(obj) informer, exists := f.informers[informerType] if exists { return informer } resyncPeriod, exists := f.customResync[informerType] if !exists { resyncPeriod = f.defaultResync } informer = newFunc(f.client, resyncPeriod) f.informers[informerType] = informer return informer } // SharedInformerFactory provides shared informers for resources in all known // API group versions. // // It is typically used like this: // // ctx, cancel := context.Background() // defer cancel() // factory := NewSharedInformerFactory(client, resyncPeriod) // defer factory.WaitForStop() // Returns immediately if nothing was started. // genericInformer := factory.ForResource(resource) // typedInformer := factory.SomeAPIGroup().V1().SomeType() // factory.Start(ctx.Done()) // Start processing these informers. // synced := factory.WaitForCacheSync(ctx.Done()) // for v, ok := range synced { // if !ok { // fmt.Fprintf(os.Stderr, "caches failed to sync: %v", v) // return // } // } // // // Creating informers can also be created after Start, but then // // Start must be called again: // anotherGenericInformer := factory.ForResource(resource) // factory.Start(ctx.Done()) type SharedInformerFactory interface { internalinterfaces.SharedInformerFactory // Start initializes all requested informers. They are handled in goroutines // which run until the stop channel gets closed. Start(stopCh <-chan struct{}) // Shutdown marks a factory as shutting down. At that point no new // informers can be started anymore and Start will return without // doing anything. // // In addition, Shutdown blocks until all goroutines have terminated. For that // to happen, the close channel(s) that they were started with must be closed, // either before Shutdown gets called or while it is waiting. // // Shutdown may be called multiple times, even concurrently. All such calls will // block until all goroutines have terminated. Shutdown() // WaitForCacheSync blocks until all started informers' caches were synced // or the stop channel gets closed. WaitForCacheSync(stopCh <-chan struct{}) map[reflect.Type]bool // ForResource gives generic access to a shared informer of the matching type. ForResource(resource schema.GroupVersionResource) (GenericInformer, error) // InternalInformerFor returns the SharedIndexInformer for obj using an internal // client. InformerFor(obj runtime.Object, newFunc internalinterfaces.NewInformerFunc) cache.SharedIndexInformer Tensorchord() modelzetes.Interface } func (f *sharedInformerFactory) Tensorchord() modelzetes.Interface { return modelzetes.New(f, f.namespace, f.tweakListOptions) } ================================================ FILE: modelzetes/pkg/client/informers/externalversions/generic.go ================================================ /* Copyright 2019-2023 TensorChord Inc. Licensed under the MIT license. See LICENSE file in the project root for full license information. */ // Code generated by informer-gen. DO NOT EDIT. package externalversions import ( "fmt" v2alpha1 "github.com/tensorchord/openmodelz/modelzetes/pkg/apis/modelzetes/v2alpha1" schema "k8s.io/apimachinery/pkg/runtime/schema" cache "k8s.io/client-go/tools/cache" ) // GenericInformer is type of SharedIndexInformer which will locate and delegate to other // sharedInformers based on type type GenericInformer interface { Informer() cache.SharedIndexInformer Lister() cache.GenericLister } type genericInformer struct { informer cache.SharedIndexInformer resource schema.GroupResource } // Informer returns the SharedIndexInformer. func (f *genericInformer) Informer() cache.SharedIndexInformer { return f.informer } // Lister returns the GenericLister. func (f *genericInformer) Lister() cache.GenericLister { return cache.NewGenericLister(f.Informer().GetIndexer(), f.resource) } // ForResource gives generic access to a shared informer of the matching type // TODO extend this to unknown resources with a client pool func (f *sharedInformerFactory) ForResource(resource schema.GroupVersionResource) (GenericInformer, error) { switch resource { // Group=tensorchord.ai, Version=v2alpha1 case v2alpha1.SchemeGroupVersion.WithResource("inferences"): return &genericInformer{resource: resource.GroupResource(), informer: f.Tensorchord().V2alpha1().Inferences().Informer()}, nil } return nil, fmt.Errorf("no informer found for %v", resource) } ================================================ FILE: modelzetes/pkg/client/informers/externalversions/internalinterfaces/factory_interfaces.go ================================================ /* Copyright 2019-2023 TensorChord Inc. Licensed under the MIT license. See LICENSE file in the project root for full license information. */ // Code generated by informer-gen. DO NOT EDIT. package internalinterfaces import ( time "time" versioned "github.com/tensorchord/openmodelz/modelzetes/pkg/client/clientset/versioned" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" cache "k8s.io/client-go/tools/cache" ) // NewInformerFunc takes versioned.Interface and time.Duration to return a SharedIndexInformer. type NewInformerFunc func(versioned.Interface, time.Duration) cache.SharedIndexInformer // SharedInformerFactory a small interface to allow for adding an informer without an import cycle type SharedInformerFactory interface { Start(stopCh <-chan struct{}) InformerFor(obj runtime.Object, newFunc NewInformerFunc) cache.SharedIndexInformer } // TweakListOptionsFunc is a function that transforms a v1.ListOptions. type TweakListOptionsFunc func(*v1.ListOptions) ================================================ FILE: modelzetes/pkg/client/informers/externalversions/modelzetes/interface.go ================================================ /* Copyright 2019-2023 TensorChord Inc. Licensed under the MIT license. See LICENSE file in the project root for full license information. */ // Code generated by informer-gen. DO NOT EDIT. package modelzetes import ( internalinterfaces "github.com/tensorchord/openmodelz/modelzetes/pkg/client/informers/externalversions/internalinterfaces" v2alpha1 "github.com/tensorchord/openmodelz/modelzetes/pkg/client/informers/externalversions/modelzetes/v2alpha1" ) // Interface provides access to each of this group's versions. type Interface interface { // V2alpha1 provides access to shared informers for resources in V2alpha1. V2alpha1() v2alpha1.Interface } type group struct { factory internalinterfaces.SharedInformerFactory namespace string tweakListOptions internalinterfaces.TweakListOptionsFunc } // New returns a new Interface. func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) Interface { return &group{factory: f, namespace: namespace, tweakListOptions: tweakListOptions} } // V2alpha1 returns a new v2alpha1.Interface. func (g *group) V2alpha1() v2alpha1.Interface { return v2alpha1.New(g.factory, g.namespace, g.tweakListOptions) } ================================================ FILE: modelzetes/pkg/client/informers/externalversions/modelzetes/v2alpha1/inference.go ================================================ /* Copyright 2019-2023 TensorChord Inc. Licensed under the MIT license. See LICENSE file in the project root for full license information. */ // Code generated by informer-gen. DO NOT EDIT. package v2alpha1 import ( "context" time "time" modelzetesv2alpha1 "github.com/tensorchord/openmodelz/modelzetes/pkg/apis/modelzetes/v2alpha1" versioned "github.com/tensorchord/openmodelz/modelzetes/pkg/client/clientset/versioned" internalinterfaces "github.com/tensorchord/openmodelz/modelzetes/pkg/client/informers/externalversions/internalinterfaces" v2alpha1 "github.com/tensorchord/openmodelz/modelzetes/pkg/client/listers/modelzetes/v2alpha1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" watch "k8s.io/apimachinery/pkg/watch" cache "k8s.io/client-go/tools/cache" ) // InferenceInformer provides access to a shared informer and lister for // Inferences. type InferenceInformer interface { Informer() cache.SharedIndexInformer Lister() v2alpha1.InferenceLister } type inferenceInformer struct { factory internalinterfaces.SharedInformerFactory tweakListOptions internalinterfaces.TweakListOptionsFunc namespace string } // NewInferenceInformer constructs a new informer for Inference type. // Always prefer using an informer factory to get a shared informer instead of getting an independent // one. This reduces memory footprint and number of connections to the server. func NewInferenceInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { return NewFilteredInferenceInformer(client, namespace, resyncPeriod, indexers, nil) } // NewFilteredInferenceInformer constructs a new informer for Inference type. // Always prefer using an informer factory to get a shared informer instead of getting an independent // one. This reduces memory footprint and number of connections to the server. func NewFilteredInferenceInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { return cache.NewSharedIndexInformer( &cache.ListWatch{ ListFunc: func(options v1.ListOptions) (runtime.Object, error) { if tweakListOptions != nil { tweakListOptions(&options) } return client.TensorchordV2alpha1().Inferences(namespace).List(context.TODO(), options) }, WatchFunc: func(options v1.ListOptions) (watch.Interface, error) { if tweakListOptions != nil { tweakListOptions(&options) } return client.TensorchordV2alpha1().Inferences(namespace).Watch(context.TODO(), options) }, }, &modelzetesv2alpha1.Inference{}, resyncPeriod, indexers, ) } func (f *inferenceInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { return NewFilteredInferenceInformer(client, f.namespace, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions) } func (f *inferenceInformer) Informer() cache.SharedIndexInformer { return f.factory.InformerFor(&modelzetesv2alpha1.Inference{}, f.defaultInformer) } func (f *inferenceInformer) Lister() v2alpha1.InferenceLister { return v2alpha1.NewInferenceLister(f.Informer().GetIndexer()) } ================================================ FILE: modelzetes/pkg/client/informers/externalversions/modelzetes/v2alpha1/interface.go ================================================ /* Copyright 2019-2023 TensorChord Inc. Licensed under the MIT license. See LICENSE file in the project root for full license information. */ // Code generated by informer-gen. DO NOT EDIT. package v2alpha1 import ( internalinterfaces "github.com/tensorchord/openmodelz/modelzetes/pkg/client/informers/externalversions/internalinterfaces" ) // Interface provides access to all the informers in this group version. type Interface interface { // Inferences returns a InferenceInformer. Inferences() InferenceInformer } type version struct { factory internalinterfaces.SharedInformerFactory namespace string tweakListOptions internalinterfaces.TweakListOptionsFunc } // New returns a new Interface. func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) Interface { return &version{factory: f, namespace: namespace, tweakListOptions: tweakListOptions} } // Inferences returns a InferenceInformer. func (v *version) Inferences() InferenceInformer { return &inferenceInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} } ================================================ FILE: modelzetes/pkg/client/listers/modelzetes/v2alpha1/expansion_generated.go ================================================ /* Copyright 2019-2023 TensorChord Inc. Licensed under the MIT license. See LICENSE file in the project root for full license information. */ // Code generated by lister-gen. DO NOT EDIT. package v2alpha1 // InferenceListerExpansion allows custom methods to be added to // InferenceLister. type InferenceListerExpansion interface{} // InferenceNamespaceListerExpansion allows custom methods to be added to // InferenceNamespaceLister. type InferenceNamespaceListerExpansion interface{} ================================================ FILE: modelzetes/pkg/client/listers/modelzetes/v2alpha1/inference.go ================================================ /* Copyright 2019-2023 TensorChord Inc. Licensed under the MIT license. See LICENSE file in the project root for full license information. */ // Code generated by lister-gen. DO NOT EDIT. package v2alpha1 import ( v2alpha1 "github.com/tensorchord/openmodelz/modelzetes/pkg/apis/modelzetes/v2alpha1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/tools/cache" ) // InferenceLister helps list Inferences. // All objects returned here must be treated as read-only. type InferenceLister interface { // List lists all Inferences in the indexer. // Objects returned here must be treated as read-only. List(selector labels.Selector) (ret []*v2alpha1.Inference, err error) // Inferences returns an object that can list and get Inferences. Inferences(namespace string) InferenceNamespaceLister InferenceListerExpansion } // inferenceLister implements the InferenceLister interface. type inferenceLister struct { indexer cache.Indexer } // NewInferenceLister returns a new InferenceLister. func NewInferenceLister(indexer cache.Indexer) InferenceLister { return &inferenceLister{indexer: indexer} } // List lists all Inferences in the indexer. func (s *inferenceLister) List(selector labels.Selector) (ret []*v2alpha1.Inference, err error) { err = cache.ListAll(s.indexer, selector, func(m interface{}) { ret = append(ret, m.(*v2alpha1.Inference)) }) return ret, err } // Inferences returns an object that can list and get Inferences. func (s *inferenceLister) Inferences(namespace string) InferenceNamespaceLister { return inferenceNamespaceLister{indexer: s.indexer, namespace: namespace} } // InferenceNamespaceLister helps list and get Inferences. // All objects returned here must be treated as read-only. type InferenceNamespaceLister interface { // List lists all Inferences in the indexer for a given namespace. // Objects returned here must be treated as read-only. List(selector labels.Selector) (ret []*v2alpha1.Inference, err error) // Get retrieves the Inference from the indexer for a given namespace and name. // Objects returned here must be treated as read-only. Get(name string) (*v2alpha1.Inference, error) InferenceNamespaceListerExpansion } // inferenceNamespaceLister implements the InferenceNamespaceLister // interface. type inferenceNamespaceLister struct { indexer cache.Indexer namespace string } // List lists all Inferences in the indexer for a given namespace. func (s inferenceNamespaceLister) List(selector labels.Selector) (ret []*v2alpha1.Inference, err error) { err = cache.ListAllByNamespace(s.indexer, s.namespace, selector, func(m interface{}) { ret = append(ret, m.(*v2alpha1.Inference)) }) return ret, err } // Get retrieves the Inference from the indexer for a given namespace and name. func (s inferenceNamespaceLister) Get(name string) (*v2alpha1.Inference, error) { obj, exists, err := s.indexer.GetByKey(s.namespace + "/" + name) if err != nil { return nil, err } if !exists { return nil, errors.NewNotFound(v2alpha1.Resource("inference"), name) } return obj.(*v2alpha1.Inference), nil } ================================================ FILE: modelzetes/pkg/config/config.go ================================================ package config import ( "encoding/json" "errors" "time" ) type Config struct { Metrics MetricsConfig `json:"metrics,omitempty"` KubeConfig KubeConfig `json:"kube_config,omitempty"` Controller ControllerConfig `json:"controller,omitempty"` HuggingfaceProxy HuggingfaceProxyConfig `json:"huggingface_proxy,omitempty"` Probes ProbesConfig `json:"probes,omitempty"` Inference InferenceConfig `json:"inference,omitempty"` } type InferenceConfig struct { ImagePullPolicy string `json:"image_pull_policy,omitempty"` SetUpRuntimeClassNvidia bool `json:"set_up_runtime_class_nvidia,omitempty"` } type ProbesConfig struct { Startup ProbeConfig `json:"startup,omitempty"` Readiness ProbeConfig `json:"readiness,omitempty"` Liveness ProbeConfig `json:"liveness,omitempty"` } type ProbeConfig struct { InitialDelaySeconds int `json:"initial_delay_seconds,omitempty"` PeriodSeconds int `json:"period_seconds,omitempty"` TimeoutSeconds int `json:"timeout_seconds,omitempty"` } type HuggingfaceProxyConfig struct { Endpoint string `json:"endpoint,omitempty"` } type ControllerConfig struct { ThreadCount int `json:"thread_count,omitempty"` } type MetricsConfig struct { ServerPort int `json:"server_port,omitempty"` } type KubeConfig struct { Kubeconfig string `json:"kubeconfig,omitempty"` MasterURL string `json:"master_url,omitempty"` QPS int `json:"qps,omitempty"` Burst int `json:"burst,omitempty"` ResyncPeriod time.Duration `json:"resync_period,omitempty"` } func New() Config { return Config{} } func (c Config) GetString() (string, error) { bytes, err := json.Marshal(c) return string(bytes), err } func (c Config) Validate() error { if c.KubeConfig.QPS == 0 || c.KubeConfig.Burst == 0 || c.KubeConfig.ResyncPeriod == 0 { return errors.New("invalid kubeconfig") } if c.Metrics.ServerPort <= 0 { return errors.New("invalid metrics config") } if c.Controller.ThreadCount == 0 { return errors.New("invalid controller config") } if c.Inference.ImagePullPolicy == "" { return errors.New("invalid inference config") } return nil } ================================================ FILE: modelzetes/pkg/consts/consts.go ================================================ package consts const ( ResourceNvidiaGPU = "nvidia.com/gpu" LabelInferenceName = "inference" LabelInferenceNamespace = "inference-namespace" LabelBuildName = "ai.tensorchord.build" LabelName = "ai.tensorchord.name" LabelNamespace = "modelz.tensorchord.ai/namespace" LabelServerResource = "ai.tensorchord.server-resource" AnnotationBuilding = "ai.tensorchord.building" AnnotationDockerImage = "ai.tensorchord.docker.image" AnnotationControlPlaneKey = "ai.tensorchord.control-plane" ModelzAnnotationValue = "modelz" TolerationGPU = "ai.tensorchord.gpu" TolerationNvidiaGPUPresent = "nvidia.com/gpu" //OrchestrationIdentifier identifier string for provider orchestration OrchestrationIdentifier = "kubernetes" //ProviderName name of the provider ProviderName = "modelzetes" DefaultServicePrefix = "mdz-" DefaultHTTPProbePath = "/" // MaxReplicas is the maximum number of replicas that can be set for a inference. MaxReplicas = 5 ) ================================================ FILE: modelzetes/pkg/controller/annotations_test.go ================================================ package controller import ( "testing" v2alpha1 "github.com/tensorchord/openmodelz/modelzetes/pkg/apis/modelzetes/v2alpha1" ) func Test_makeAnnotations_NoKeys(t *testing.T) { annotationVal := `{"name":"","image":""}` spec := v2alpha1.Inference{ Spec: v2alpha1.InferenceSpec{}, } annotations := makeAnnotations(&spec) if _, ok := annotations["prometheus.io.scrape"]; !ok { t.Errorf("wanted annotation " + "prometheus.io.scrape" + " to be added") t.Fail() } if val, _ := annotations["prometheus.io.scrape"]; val != "false" { t.Errorf("wanted annotation " + "prometheus.io.scrape" + ` to equal "false"`) t.Fail() } if _, ok := annotations[annotationInferenceSpec]; !ok { t.Errorf("wanted annotation " + annotationInferenceSpec) t.Fail() } if val, _ := annotations[annotationInferenceSpec]; val != annotationVal { t.Errorf("Annotation " + annotationInferenceSpec + "\nwant: '" + annotationVal + "'\ngot: '" + val + "'") t.Fail() } } func Test_makeAnnotations_WithKeyAndValue(t *testing.T) { annotationVal := `{"name":"","image":"","annotations":{"key":"value","key2":"value2"}}` spec := v2alpha1.Inference{ Spec: v2alpha1.InferenceSpec{ Annotations: map[string]string{ "key": "value", "key2": "value2", }, }, } annotations := makeAnnotations(&spec) if _, ok := annotations["prometheus.io.scrape"]; !ok { t.Errorf("wanted annotation " + "prometheus.io.scrape" + " to be added") t.Fail() } if val := annotations["prometheus.io.scrape"]; val != "false" { t.Errorf("wanted annotation " + "prometheus.io.scrape" + ` to equal "false"`) t.Fail() } if _, ok := annotations[annotationInferenceSpec]; !ok { t.Errorf("wanted annotation " + annotationInferenceSpec) t.Fail() } if val := annotations[annotationInferenceSpec]; val != annotationVal { t.Errorf("Annotation " + annotationInferenceSpec + "\nwant: '" + annotationVal + "'\ngot: '" + val + "'") t.Fail() } } func Test_makeAnnotationsDoesNotModifyOriginalSpec(t *testing.T) { specAnnotations := map[string]string{ "test.foo": "bar", } function := &v2alpha1.Inference{ Spec: v2alpha1.InferenceSpec{ Name: "testfunc", Annotations: specAnnotations, }, } expectedAnnotations := map[string]string{ "prometheus.io.scrape": "false", "test.foo": "bar", annotationInferenceSpec: `{"name":"testfunc","image":"","annotations":{"test.foo":"bar"}}`, } makeAnnotations(function) annotations := makeAnnotations(function) if len(specAnnotations) != 1 { t.Errorf("length of original spec annotations has changed, expected 1, got %d", len(specAnnotations)) } if specAnnotations["test.foo"] != "bar" { t.Errorf("original spec annotation has changed") } for name, expectedValue := range expectedAnnotations { actualValue := annotations[name] if actualValue != expectedValue { t.Fatalf("incorrect annotation for '%s': \nwant %q,\ngot %q", name, expectedValue, actualValue) } } } ================================================ FILE: modelzetes/pkg/controller/controller.go ================================================ package controller import ( "context" "fmt" "strings" "time" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" kubeinformers "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1" appslisters "k8s.io/client-go/listers/apps/v1" "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/record" "k8s.io/client-go/util/workqueue" glog "k8s.io/klog" v2alpha1 "github.com/tensorchord/openmodelz/modelzetes/pkg/apis/modelzetes/v2alpha1" clientset "github.com/tensorchord/openmodelz/modelzetes/pkg/client/clientset/versioned" faasscheme "github.com/tensorchord/openmodelz/modelzetes/pkg/client/clientset/versioned/scheme" informers "github.com/tensorchord/openmodelz/modelzetes/pkg/client/informers/externalversions" listers "github.com/tensorchord/openmodelz/modelzetes/pkg/client/listers/modelzetes/v2alpha1" "github.com/tensorchord/openmodelz/modelzetes/pkg/consts" ) const ( controllerAgentName = "modelz-operator" functionPort = 8080 // SuccessSynced is used as part of the Event 'reason' when a Function is synced SuccessSynced = "Synced" // ErrResourceExists is used as part of the Event 'reason' when a Function fails // to sync due to a Deployment of the same name already existing. ErrResourceExists = "ErrResourceExists" // MessageResourceExists is the message used for Events when a resource // fails to sync due to a Deployment already existing MessageResourceExists = "Resource %q already exists and is not managed by OpenFaaS" // MessageResourceSynced is the message used for an Event fired when a Function // is synced successfully MessageResourceSynced = "Function synced successfully" ) // Controller is the controller implementation for Function resources type Controller struct { BaseDomain string // kubeclientset is a standard kubernetes clientset kubeclientset kubernetes.Interface // faasclientset is a clientset for our own API group faasclientset clientset.Interface deploymentsLister appslisters.DeploymentLister deploymentsSynced cache.InformerSynced inferenceLister listers.InferenceLister inferencesSynced cache.InformerSynced // workqueue is a rate limited work queue. This is used to queue work to be // processed instead of performing it as soon as a change happens. This // means we can ensure we only process a fixed amount of resources at a // time, and makes it easy to ensure we are never processing the same item // simultaneously in two different workers. workqueue workqueue.RateLimitingInterface // recorder is an event recorder for recording Event resources to the // Kubernetes API. recorder record.EventRecorder // OpenFaaS function factory factory FunctionFactory } // NewController returns a new OpenFaaS controller func NewController( kubeclientset kubernetes.Interface, inferenceclientset clientset.Interface, kubeInformerFactory kubeinformers.SharedInformerFactory, inferenceInformerFactory informers.SharedInformerFactory, factory FunctionFactory) *Controller { // obtain references to shared index informers for the Deployment and Function types deploymentInformer := kubeInformerFactory.Apps().V1().Deployments() inferenceInformer := inferenceInformerFactory.Tensorchord().V2alpha1().Inferences() // Create event broadcaster // Add o6s types to the default Kubernetes Scheme so Events can be // logged for faas-controller types. faasscheme.AddToScheme(scheme.Scheme) glog.V(4).Info("Creating event broadcaster") eventBroadcaster := record.NewBroadcaster() eventBroadcaster.StartLogging(glog.V(4).Infof) eventBroadcaster.StartRecordingToSink(&typedcorev1.EventSinkImpl{Interface: kubeclientset.CoreV1().Events("")}) recorder := eventBroadcaster.NewRecorder(scheme.Scheme, corev1.EventSource{Component: controllerAgentName}) controller := &Controller{ kubeclientset: kubeclientset, faasclientset: inferenceclientset, deploymentsLister: deploymentInformer.Lister(), deploymentsSynced: deploymentInformer.Informer().HasSynced, inferenceLister: inferenceInformer.Lister(), inferencesSynced: inferenceInformer.Informer().HasSynced, workqueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "Functions"), recorder: recorder, factory: factory, } glog.Info("Setting up event handlers") // Add Function (OpenFaaS CRD-entry) Informer // // Set up an event handler for when Function resources change inferenceInformer.Informer(). AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: controller.enqueueFunction, UpdateFunc: func(old, new interface{}) { controller.enqueueFunction(new) }, }) // Set up an event handler for when functions related resources like pods, deployments, replica sets // can't be materialized. This logs abnormal events like ImagePullBackOff, back-off restarting failed container, // failed to start container, oci runtime errors, etc // Enable this with -v=3 kubeInformerFactory.Core().V1().Events().Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: func(obj interface{}) { key, err := cache.MetaNamespaceKeyFunc(obj) if err == nil { event := obj.(*corev1.Event) since := time.Since(event.LastTimestamp.Time) // log abnormal events occurred in the last minute if since.Seconds() < 61 && strings.Contains(event.Type, "Warning") { glog.V(3).Infof("Abnormal event detected on %s %s: %s", event.LastTimestamp, key, event.Message) } } }, }) return controller } // Run will set up the event handlers for types we are interested in, as well // as syncing informer caches and starting workers. It will block until stopCh // is closed, at which point it will shutdown the workqueue and wait for // workers to finish processing their current work items. func (c *Controller) Run(threadiness int, stopCh <-chan struct{}) error { defer runtime.HandleCrash() defer c.workqueue.ShutDown() // Start the informer factories to begin populating the informer caches // Wait for the caches to be synced before starting workers glog.Info("Waiting for informer caches to sync") if ok := cache.WaitForCacheSync(stopCh, c.deploymentsSynced, c.inferencesSynced); !ok { return fmt.Errorf("failed to wait for caches to sync") } glog.Info("Starting workers") // Launch two workers to process Function resources for i := 0; i < threadiness; i++ { go wait.Until(c.runWorker, time.Second, stopCh) } glog.Info("Started workers") <-stopCh glog.Info("Shutting down workers") return nil } // runWorker is a long-running function that will continually call the // processNextWorkItem function in order to read and process a message on the workqueue. func (c *Controller) runWorker() { for c.processNextWorkItem() { } } // processNextWorkItem will read a single work item off the workqueue and // attempt to process it, by calling the syncHandler. func (c *Controller) processNextWorkItem() bool { obj, shutdown := c.workqueue.Get() if shutdown { return false } err := func(obj interface{}) error { defer c.workqueue.Done(obj) var key string var ok bool if key, ok = obj.(string); !ok { c.workqueue.Forget(obj) runtime.HandleError(fmt.Errorf("expected string in workqueue but got %#v", obj)) return nil } if err := c.syncHandler(key); err != nil { return fmt.Errorf("error syncing '%s': %s", key, err.Error()) } c.workqueue.Forget(obj) return nil }(obj) if err != nil { runtime.HandleError(err) return true } return true } // syncHandler compares the actual state with the desired, and attempts to // converge the two. func (c *Controller) syncHandler(key string) error { // Convert the namespace/name string into a distinct namespace and name namespace, name, err := cache.SplitMetaNamespaceKey(key) if err != nil { runtime.HandleError(fmt.Errorf("invalid resource key: %s", key)) return nil } // Get the Function resource with this namespace/name function, err := c.inferenceLister.Inferences(namespace).Get(name) if err != nil { // The Function resource may no longer exist, in which case we stop processing. if errors.IsNotFound(err) { runtime.HandleError(fmt.Errorf("function '%s' in work queue no longer exists", key)) return nil } return err } deploymentName := function.Spec.Name if deploymentName == "" { // We choose to absorb the error here as the worker would requeue the // resource otherwise. Instead, the next time the resource is updated // the resource will be queued again. runtime.HandleError(fmt.Errorf("%s: deployment name must be specified", key)) return nil } if function.Spec.Annotations != nil { if _, ok := function.Spec.Annotations[consts.AnnotationBuilding]; ok { glog.Infof("Function '%s' is still building", function.Spec.Name) return nil } } // Get the deployment with the name specified in Function.spec deployment, err := c.deploymentsLister. Deployments(function.Namespace).Get(deploymentName) // If the resource doesn't exist, we'll create it if errors.IsNotFound(err) { err = nil existingSecrets, err := c.getSecrets(function.Namespace, function.Spec.Secrets) if err != nil { return err } glog.Infof("Creating deployment for '%s'", function.Spec.Name) deployment, err = c.kubeclientset.AppsV1().Deployments(function.Namespace).Create( context.TODO(), newDeployment(function, deployment, existingSecrets, c.factory), metav1.CreateOptions{}, ) if err != nil { return err } } svcGetOptions := metav1.GetOptions{} svcName := consts.DefaultServicePrefix + deploymentName _, getSvcErr := c.kubeclientset.CoreV1().Services(function.Namespace).Get(context.TODO(), deploymentName, svcGetOptions) if errors.IsNotFound(getSvcErr) { glog.Infof("Creating ClusterIP service for '%s'", function.Spec.Name) if _, err := c.kubeclientset.CoreV1().Services(function.Namespace).Create(context.TODO(), newService(function), metav1.CreateOptions{}); err != nil { // If an error occurs during Service Create, we'll requeue the item if errors.IsAlreadyExists(err) { err = nil glog.V(2).Infof("ClusterIP service '%s' already exists. Skipping creation.", function.Spec.Name) } else { return err } } } // If an error occurs during Get/Create, we'll requeue the item so we can // attempt processing again later. This could have been caused by a // temporary network failure, or any other transient reason. if err != nil { return fmt.Errorf("transient error: %v", err) } // If the Deployment is not controlled by this Function resource, we should log // a warning to the event recorder and ret if !metav1.IsControlledBy(deployment, function) { msg := fmt.Sprintf(MessageResourceExists, deployment.Name) c.recorder.Event(function, corev1.EventTypeWarning, ErrResourceExists, msg) return fmt.Errorf(msg) } // Update the Deployment resource if the Function definition differs if deploymentNeedsUpdate(function, deployment) { glog.Infof("Updating deployment for '%s'", function.Spec.Name) existingSecrets, err := c.getSecrets(function.Namespace, function.Spec.Secrets) if err != nil { return err } deployment, err = c.kubeclientset.AppsV1().Deployments(function.Namespace).Update( context.TODO(), newDeployment(function, deployment, existingSecrets, c.factory), metav1.UpdateOptions{}, ) if err != nil { glog.Errorf("Updating deployment for '%s' failed: %v", function.Spec.Name, err) } existingService, err := c.kubeclientset.CoreV1().Services(function.Namespace).Get(context.TODO(), svcName, metav1.GetOptions{}) if err != nil { return err } existingService.Annotations = makeAnnotations(function) _, err = c.kubeclientset.CoreV1().Services(function.Namespace).Update(context.TODO(), existingService, metav1.UpdateOptions{}) if err != nil { glog.Errorf("Updating service for '%s' failed: %v", function.Spec.Name, err) } } // If an error occurs during Update, we'll requeue the item so we can // attempt processing again later. THis could have been caused by a // temporary network failure, or any other transient reason. if err != nil { return err } c.recorder.Event(function, corev1.EventTypeNormal, SuccessSynced, MessageResourceSynced) return nil } // enqueueFunction takes a Function resource and converts it into a namespace/name // string which is then put onto the work queue. This method should *not* be // passed resources of any type other than Function. func (c *Controller) enqueueFunction(obj interface{}) { var key string var err error if key, err = cache.MetaNamespaceKeyFunc(obj); err != nil { runtime.HandleError(err) return } c.workqueue.AddRateLimited(key) } // handleObject will take any resource implementing metav1.Object and attempt // to find the Function resource that 'owns' it. It does this by looking at the // objects metadata.ownerReferences field for an appropriate OwnerReference. // It then enqueues that Function resource to be processed. If the object does not // have an appropriate OwnerReference, it will simply be skipped. func (c *Controller) handleObject(obj interface{}) { var object metav1.Object var ok bool if object, ok = obj.(metav1.Object); !ok { tombstone, ok := obj.(cache.DeletedFinalStateUnknown) if !ok { runtime.HandleError(fmt.Errorf("error decoding object, invalid type")) return } object, ok = tombstone.Obj.(metav1.Object) if !ok { runtime.HandleError(fmt.Errorf("error decoding object tombstone, invalid type")) return } glog.V(4).Infof("Recovered deleted object '%s' from tombstone", object.GetName()) } glog.V(4).Infof("Processing object: %s", object.GetName()) if ownerRef := metav1.GetControllerOf(object); ownerRef != nil { // If this object is not owned by a function, we should not do anything more // with it. if ownerRef.Kind != v2alpha1.Kind { return } function, err := c.inferenceLister.Inferences( object.GetNamespace()).Get(ownerRef.Name) if err != nil { glog.Infof("Function '%s' deleted. Ignoring orphaned object '%s'", ownerRef.Name, object.GetSelfLink()) return } c.enqueueFunction(function) return } } // getSecrets queries Kubernetes for a list of secrets by name in the given k8s namespace. func (c *Controller) getSecrets(namespace string, secretNames []string) (map[string]*corev1.Secret, error) { secrets := map[string]*corev1.Secret{} for _, secretName := range secretNames { secret, err := c.kubeclientset.CoreV1(). Secrets(namespace).Get(context.TODO(), secretName, metav1.GetOptions{}) if err != nil { return secrets, err } secrets[secretName] = secret } return secrets, nil } // getReplicas returns the desired number of replicas for a function taking into account // the min replicas label, HPA, the autoscaler and scaled to zero deployments func getReplicas(inference *v2alpha1.Inference, deployment *appsv1.Deployment) *int32 { var minReplicas, maxReplicas *int32 if inference.Spec.Scaling != nil { minReplicas = inference.Spec.Scaling.MinReplicas maxReplicas = inference.Spec.Scaling.MaxReplicas } // extract current deployment replicas if specified var deploymentReplicas *int32 if deployment != nil { deploymentReplicas = deployment.Spec.Replicas } // do not set replicas if min replicas is not set // and current deployment has no replicas count if minReplicas == nil && deploymentReplicas == nil { return nil } // set replicas to min if deployment has no replicas and min replicas exists if minReplicas != nil && deploymentReplicas == nil { return minReplicas } // do not override replicas when min is not specified if minReplicas == nil && deploymentReplicas != nil { return deploymentReplicas } if minReplicas != nil && deploymentReplicas != nil { if maxReplicas == nil { // do not override HPA or OF autoscaler replicas if the value is greater than min if *deploymentReplicas >= *minReplicas { return deploymentReplicas } } else { // do not override HPA or OF autoscaler replicas if the value is between min and max if *deploymentReplicas >= *minReplicas && *deploymentReplicas <= *maxReplicas { return deploymentReplicas } else if *deploymentReplicas > *maxReplicas { return maxReplicas } } } return minReplicas } ================================================ FILE: modelzetes/pkg/controller/deployment.go ================================================ package controller import ( "encoding/json" "strconv" "strings" "github.com/google/go-cmp/cmp" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/intstr" glog "k8s.io/klog" v2alpha1 "github.com/tensorchord/openmodelz/modelzetes/pkg/apis/modelzetes/v2alpha1" "github.com/tensorchord/openmodelz/modelzetes/pkg/consts" "github.com/tensorchord/openmodelz/modelzetes/pkg/k8s" . "github.com/tensorchord/openmodelz/modelzetes/pkg/pointer" ) const ( annotationInferenceSpec = "ai.tensorchord.inference.spec" defaultPort = 8080 ) var runtimeClassNvidia = "nvidia" // newDeployment creates a new Deployment for a Function resource. It also sets // the appropriate OwnerReferences on the resource so handleObject can discover // the Function resource that 'owns' it. func newDeployment( inference *v2alpha1.Inference, existingDeployment *appsv1.Deployment, existingSecrets map[string]*corev1.Secret, factory FunctionFactory) *appsv1.Deployment { // Set replicas to 0 if the expected number of replicas is 0 replicas := getReplicas(inference, existingDeployment) envVars := makeEnvVars(inference) labels := makeLabels(inference) nodeSelector := makeNodeSelector(inference.Spec.Constraints) port := makePort(inference) probes, err := factory.MakeProbes(inference, port) if err != nil { glog.Warningf("Function %s probes parsing failed: %v", inference.Spec.Name, err) } labelMap := k8s.MakeLabelSelector(inference.Spec.Name) // Add a new env var HF_ENDPOINT if enabled. hfEnvs := factory.MakeHuggingfacePullThroughCacheEnvVar() if hfEnvs != nil { envVars = addEnvVarIfNotExists(envVars, hfEnvs.Name, hfEnvs.Value) } annotations := makeAnnotations(inference) command := makeCommand(inference) allowPrivilegeEscalation := false deploymentSpec := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: inference.Spec.Name, Annotations: annotations, Namespace: inference.Namespace, Labels: labels, OwnerReferences: []metav1.OwnerReference{ *metav1.NewControllerRef(inference, schema.GroupVersionKind{ Group: v2alpha1.SchemeGroupVersion.Group, Version: v2alpha1.SchemeGroupVersion.Version, Kind: v2alpha1.Kind, }), }, }, Spec: appsv1.DeploymentSpec{ Replicas: replicas, Strategy: appsv1.DeploymentStrategy{ Type: appsv1.RollingUpdateDeploymentStrategyType, RollingUpdate: &appsv1.RollingUpdateDeployment{ MaxUnavailable: &intstr.IntOrString{ Type: intstr.String, StrVal: "10%", }, MaxSurge: &intstr.IntOrString{ Type: intstr.String, StrVal: "10%", }, }, }, Selector: &metav1.LabelSelector{ MatchLabels: labelMap, }, RevisionHistoryLimit: Ptr(int32(5)), Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: labels, Annotations: annotations, }, Spec: corev1.PodSpec{ NodeSelector: nodeSelector, Containers: []corev1.Container{ { Name: inference.Spec.Name, Image: inference.Spec.Image, Ports: []corev1.ContainerPort{ {ContainerPort: int32(port), Protocol: corev1.ProtocolTCP}, }, Command: command, ImagePullPolicy: corev1.PullPolicy(factory.Factory.Config.ImagePullPolicy), Env: envVars, SecurityContext: &corev1.SecurityContext{ AllowPrivilegeEscalation: &allowPrivilegeEscalation, }, // TODO(xieydd): Add a function to set shm size VolumeMounts: []corev1.VolumeMount{ { Name: "dshm", MountPath: "/dev/shm", }, }, }, }, Volumes: []corev1.Volume{ { Name: "dshm", VolumeSource: corev1.VolumeSource{ EmptyDir: &corev1.EmptyDirVolumeSource{ Medium: corev1.StorageMediumMemory, }, }, }, }, }, }, }, } if probes != nil { if probes.Liveness != nil { deploymentSpec.Spec.Template.Spec.Containers[0].LivenessProbe = probes.Liveness } if probes.Readiness != nil { deploymentSpec.Spec.Template.Spec.Containers[0].ReadinessProbe = probes.Readiness } if probes.Startup != nil { deploymentSpec.Spec.Template.Spec.Containers[0].StartupProbe = probes.Startup if inference.Spec.Scaling != nil && inference.Spec.Scaling.StartupDuration != nil { // Set the failure threshold to the number of seconds in the duration. deploymentSpec.Spec.Template.Spec.Containers[0]. StartupProbe.FailureThreshold = int32( *inference.Spec.Scaling.StartupDuration / probes.Startup.PeriodSeconds) } } } if inference.Spec.Resources != nil { deploymentSpec.Spec.Template.Spec.Containers[0].Resources = *inference.Spec.Resources if q, ok := inference.Spec.Resources.Limits[consts.ResourceNvidiaGPU]; ok { if q.Value() > 0 { // If GPU is requested, add the GPU toleration. deploymentSpec.Spec.Template.Spec.Tolerations = makeTolerationGPU() if factory.Factory.Config.RuntimeClassNvidia { deploymentSpec.Spec.Template.Spec.RuntimeClassName = &runtimeClassNvidia } } else { // If GPU is not requested, set CUDA_VISIBLE_DEVICES to empty string. deploymentSpec.Spec.Template.Spec.Containers[0].Env = append( deploymentSpec.Spec.Template.Spec.Containers[0].Env, corev1.EnvVar{ Name: "CUDA_VISIBLE_DEVICES", Value: "", }, ) } } } factory.ConfigureReadOnlyRootFilesystem(inference, deploymentSpec) factory.ConfigureContainerUserID(deploymentSpec) return deploymentSpec } func makeTolerationGPU() []corev1.Toleration { res := []corev1.Toleration{ { Key: consts.TolerationGPU, Operator: corev1.TolerationOpEqual, Value: "true", }, { Key: consts.TolerationNvidiaGPUPresent, Operator: corev1.TolerationOpEqual, Value: "present", }, } return res } func makeCommand(inference *v2alpha1.Inference) []string { if inference.Spec.Command != nil { res := strings.Split(*inference.Spec.Command, " ") return res } return nil } func makeEnvVars(inference *v2alpha1.Inference) []corev1.EnvVar { envVars := []corev1.EnvVar{} if inference.Spec.EnvVars != nil { for k, v := range inference.Spec.EnvVars { envVars = append(envVars, corev1.EnvVar{ Name: k, Value: v, }) } } // Set environment variables for different frameworks. switch inference.Spec.Framework { case v2alpha1.FrameworkGradio: envVars = addEnvVarIfNotExists(envVars, "GRADIO_SERVER_NAME", "0.0.0.0") envVars = addEnvVarIfNotExists(envVars, "GRADIO_SERVER_PORT", "7860") case v2alpha1.FrameworkMosec: envVars = addEnvVarIfNotExists(envVars, "MOSEC_PORT", strconv.Itoa(defaultPort)) case v2alpha1.FrameworkStreamlit: envVars = addEnvVarIfNotExists(envVars, "STREAMLIT_SERVER_ENABLE_CORS", "false") envVars = addEnvVarIfNotExists(envVars, "STREAMLIT_SERVER_ADDRESS", "0.0.0.0") envVars = addEnvVarIfNotExists(envVars, "STREAMLIT_SERVER_ENABLE_XSRF_PROTECTION", "false") } return envVars } func addEnvVarIfNotExists(envVars []corev1.EnvVar, name, value string) []corev1.EnvVar { for _, envVar := range envVars { if envVar.Name == name { return envVars } } return append(envVars, corev1.EnvVar{ Name: name, Value: value, }) } func makeLabels(inference *v2alpha1.Inference) map[string]string { labels := map[string]string{ consts.LabelInferenceName: inference.Spec.Name, "app": inference.Spec.Name, "controller": inference.Name, } if inference.Spec.Labels != nil { for k, v := range inference.Spec.Labels { labels[k] = v } } return labels } func makePort(inference *v2alpha1.Inference) int { if inference.Spec.Port != nil { return int(*inference.Spec.Port) } return defaultPort } func makeAnnotations(inference *v2alpha1.Inference) map[string]string { annotations := make(map[string]string) // disable scraping since the watchdog doesn't expose a metrics endpoint annotations["prometheus.io.scrape"] = "false" // copy inference annotations if inference.Spec.Annotations != nil { for k, v := range inference.Spec.Annotations { annotations[k] = v } } // save inference spec in deployment annotations // used to detect changes in inference spec specJSON, err := json.Marshal(inference.Spec) if err != nil { glog.Errorf("Failed to marshal inference spec: %s", err.Error()) return annotations } annotations[annotationInferenceSpec] = string(specJSON) return annotations } func makeNodeSelector(constraints []string) map[string]string { selector := make(map[string]string) if len(constraints) > 0 { for _, constraint := range constraints { parts := strings.Split(constraint, "=") if len(parts) == 2 { selector[parts[0]] = parts[1] } } } return selector } // deploymentNeedsUpdate determines if the inference spec is different from the deployment spec func deploymentNeedsUpdate( inference *v2alpha1.Inference, deployment *appsv1.Deployment) bool { prevFnSpecJson := deployment.ObjectMeta.Annotations[annotationInferenceSpec] if prevFnSpecJson == "" { // is a new deployment or is an old deployment that is missing the annotation return true } prevFnSpec := &v2alpha1.InferenceSpec{} err := json.Unmarshal([]byte(prevFnSpecJson), prevFnSpec) if err != nil { glog.Errorf("Failed to parse previous inference spec: %s", err.Error()) return true } prevFn := v2alpha1.Inference{ Spec: *prevFnSpec, } if diff := cmp.Diff(prevFn.Spec, inference.Spec); diff != "" { glog.V(2).Infof("Change detected for %s diff\n%s", inference.Name, diff) return true } else { glog.V(3).Infof("No changes detected for %s", inference.Name) } return false } func int32p(i int32) *int32 { return &i } ================================================ FILE: modelzetes/pkg/controller/deployment_test.go ================================================ package controller import ( "strings" "testing" corev1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/fake" v2alpha1 "github.com/tensorchord/openmodelz/modelzetes/pkg/apis/modelzetes/v2alpha1" "github.com/tensorchord/openmodelz/modelzetes/pkg/consts" "github.com/tensorchord/openmodelz/modelzetes/pkg/k8s" . "github.com/tensorchord/openmodelz/modelzetes/pkg/pointer" ) var defaultK8sConfig = k8s.DeploymentConfig{ HTTPProbe: true, SetNonRootUser: true, LivenessProbe: &k8s.ProbeConfig{ PeriodSeconds: 1, TimeoutSeconds: 3, InitialDelaySeconds: 0, }, ReadinessProbe: &k8s.ProbeConfig{ PeriodSeconds: 1, TimeoutSeconds: 3, InitialDelaySeconds: 0, }, StartupProbe: &k8s.ProbeConfig{ PeriodSeconds: 1, TimeoutSeconds: 3, InitialDelaySeconds: 0, }, } func assertEnv(t *testing.T, expect map[string]string, real []v1.EnvVar) { for _, env := range real { value, exist := expect[env.Name] if exist == false || value != env.Value { t.Errorf("Environment variables contains unexpected %s:%s", env.Name, env.Value) t.Fail() } delete(expect, env.Name) } if len(expect) != 0 { t.Errorf("Environment variables should contain %v", expect) t.Fail() } } func Test_newDeployment(t *testing.T) { inference := &v2alpha1.Inference{ ObjectMeta: metav1.ObjectMeta{ Name: "kubesec", }, Spec: v2alpha1.InferenceSpec{ Name: "kubesec", Image: "docker.io/kubesec/kubesec", HTTPProbePath: Ptr("/"), Annotations: map[string]string{}, }, } factory := NewFunctionFactory(fake.NewSimpleClientset(), defaultK8sConfig) secrets := map[string]*corev1.Secret{} deployment := newDeployment(inference, nil, secrets, factory) if deployment.Spec.Template.Spec.Containers[0].ReadinessProbe.HTTPGet.Path != "/" { t.Errorf("Readiness probe should have HTTPGet handler set to %s", "/") t.Fail() } if deployment.Spec.Template.Spec.Containers[0].StartupProbe.InitialDelaySeconds != 0 { t.Errorf("Startup probe should have initial delay seconds set to %s", "0") t.Fail() } if deployment.Spec.Template.Spec.Containers[0].LivenessProbe.InitialDelaySeconds != 0 { t.Errorf("Liveness probe should have initial delay seconds set to %s", "0") t.Fail() } if *(deployment.Spec.Template.Spec.Containers[0].SecurityContext.RunAsUser) != k8s.SecurityContextUserID { t.Errorf("RunAsUser should be %v", k8s.SecurityContextUserID) t.Fail() } } func TestNewDeploymentWithStartupDurationLabel(t *testing.T) { inf := &v2alpha1.Inference{ ObjectMeta: metav1.ObjectMeta{ Name: "kubesec", }, Spec: v2alpha1.InferenceSpec{ Name: "kubesec", Image: "docker.io/kubesec/kubesec", HTTPProbePath: Ptr("/"), Annotations: map[string]string{ "prometheus.io.scrape": "true", }, Scaling: &v2alpha1.ScalingConfig{ StartupDuration: Ptr(int32(600)), }, }, } factory := NewFunctionFactory(fake.NewSimpleClientset(), k8s.DeploymentConfig{ HTTPProbe: true, SetNonRootUser: true, LivenessProbe: &k8s.ProbeConfig{ PeriodSeconds: 1, TimeoutSeconds: 3, InitialDelaySeconds: 0, }, ReadinessProbe: &k8s.ProbeConfig{ PeriodSeconds: 1, TimeoutSeconds: 3, InitialDelaySeconds: 0, }, StartupProbe: &k8s.ProbeConfig{ PeriodSeconds: 10, TimeoutSeconds: 3, InitialDelaySeconds: 0, }, }) secrets := map[string]*corev1.Secret{} expectedPeriodSeconds := int32(10) expectedFailureThreshold := int32(60) deployment := newDeployment(inf, nil, secrets, factory) if len(deployment.Spec.Template.Spec.Containers) == 0 { t.Errorf("Deployment should have at least one container") t.Fail() } if deployment.Spec.Template.Spec.Containers[0].StartupProbe == nil { t.Errorf("Deployment should have a startup probe") t.Fail() } if deployment.Spec.Template.Spec.Containers[0].StartupProbe.PeriodSeconds != expectedPeriodSeconds { t.Errorf("Startup probe should have timeout seconds set to %d", expectedPeriodSeconds) t.Fail() } if deployment.Spec.Template.Spec.Containers[0].StartupProbe.FailureThreshold != expectedFailureThreshold { t.Errorf("Startup probe should have failure threshold set to %d", expectedFailureThreshold) t.Fail() } } func Test_newDeployment_PrometheusScrape_NotOverridden(t *testing.T) { inference := &v2alpha1.Inference{ ObjectMeta: metav1.ObjectMeta{ Name: "kubesec", }, Spec: v2alpha1.InferenceSpec{ Name: "kubesec", Image: "docker.io/kubesec/kubesec", Annotations: map[string]string{ "prometheus.io.scrape": "true", }, }, } factory := NewFunctionFactory(fake.NewSimpleClientset(), k8s.DeploymentConfig{ HTTPProbe: false, SetNonRootUser: true, LivenessProbe: &k8s.ProbeConfig{ PeriodSeconds: 1, TimeoutSeconds: 3, InitialDelaySeconds: 0, }, ReadinessProbe: &k8s.ProbeConfig{ PeriodSeconds: 1, TimeoutSeconds: 3, InitialDelaySeconds: 0, }, StartupProbe: &k8s.ProbeConfig{ PeriodSeconds: 1, TimeoutSeconds: 3, InitialDelaySeconds: 0, }, }) secrets := map[string]*corev1.Secret{} deployment := newDeployment(inference, nil, secrets, factory) want := "true" if deployment.Spec.Template.Annotations["prometheus.io.scrape"] != want { t.Errorf("Annotation prometheus.io.scrape should be %s, was: %s", want, deployment.Spec.Template.Annotations["prometheus.io.scrape"]) } } func Test_newDeployment_WithZeroResource(t *testing.T) { quantity, _ := resource.ParseQuantity("0") inference := &v2alpha1.Inference{ ObjectMeta: metav1.ObjectMeta{ Name: "kubesec", }, Spec: v2alpha1.InferenceSpec{ Name: "kubesec", Image: "docker.io/kubesec/kubesec", HTTPProbePath: Ptr("/"), Annotations: map[string]string{}, Resources: &v1.ResourceRequirements{ Limits: v1.ResourceList{consts.ResourceNvidiaGPU: quantity}, }, }, } factory := NewFunctionFactory(fake.NewSimpleClientset(), defaultK8sConfig) secrets := map[string]*corev1.Secret{} deployment := newDeployment(inference, nil, secrets, factory) if deployment.Spec.Template.Spec.Containers[0].Env[0].Name != "CUDA_VISIBLE_DEVICES" { t.Errorf("CUDA_VISIBLE_DEVICES should be set to environment variables") t.Fail() } if deployment.Spec.Template.Spec.Containers[0].Env[0].Value != "" { t.Errorf("Empty value should be set to CUDA_VISIBLE_DEVICES") t.Fail() } } func Test_newDeployment_WithNonZeroResource(t *testing.T) { quantity, _ := resource.ParseQuantity("1") inference := &v2alpha1.Inference{ ObjectMeta: metav1.ObjectMeta{ Name: "kubesec", }, Spec: v2alpha1.InferenceSpec{ Name: "kubesec", Image: "docker.io/kubesec/kubesec", HTTPProbePath: Ptr("/"), Annotations: map[string]string{}, Resources: &v1.ResourceRequirements{ Limits: v1.ResourceList{consts.ResourceNvidiaGPU: quantity}, }, }, } factory := NewFunctionFactory(fake.NewSimpleClientset(), defaultK8sConfig) secrets := map[string]*corev1.Secret{} deployment := newDeployment(inference, nil, secrets, factory) if deployment.Spec.Template.Spec.Tolerations[0].Key != consts.TolerationGPU { t.Errorf("Tolerations should contain %s", consts.TolerationGPU) t.Fail() } if deployment.Spec.Template.Spec.Tolerations[1].Key != consts.TolerationNvidiaGPUPresent { t.Errorf("Tolerations should contain %s", consts.TolerationNvidiaGPUPresent) t.Fail() } } func Test_newDeployment_WithCommandsAndEnvVars(t *testing.T) { expectEnv := map[string]string{"MOCK": "TEST"} expectCommand := "python main.py" inference := &v2alpha1.Inference{ ObjectMeta: metav1.ObjectMeta{ Name: "kubesec", }, Spec: v2alpha1.InferenceSpec{ Name: "kubesec", Image: "docker.io/kubesec/kubesec", HTTPProbePath: Ptr("/"), Annotations: map[string]string{}, Command: Ptr(expectCommand), EnvVars: expectEnv, }, } factory := NewFunctionFactory(fake.NewSimpleClientset(), defaultK8sConfig) secrets := map[string]*corev1.Secret{} deployment := newDeployment(inference, nil, secrets, factory) assertEnv(t, expectEnv, deployment.Spec.Template.Spec.Containers[0].Env) if strings.Join(deployment.Spec.Template.Spec.Containers[0].Command, " ") != expectCommand { t.Errorf("Command should contain value %s", expectCommand) t.Fail() } } ================================================ FILE: modelzetes/pkg/controller/deployment_update_test.go ================================================ package controller import ( "testing" appsv1 "k8s.io/api/apps/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" v2alpha1 "github.com/tensorchord/openmodelz/modelzetes/pkg/apis/modelzetes/v2alpha1" . "github.com/tensorchord/openmodelz/modelzetes/pkg/pointer" ) func Test_Deployment_Need_Update(t *testing.T) { scenarios := []struct { name string inference *v2alpha1.Inference deploy *appsv1.Deployment expected bool }{ { "empty deployment need update", &v2alpha1.Inference{}, &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ annotationInferenceSpec: "", }, }, }, true, }, { "bad deployment need update", &v2alpha1.Inference{}, &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{annotationInferenceSpec: "bad"}, }, }, true, }, { "equal deployment doesn't need update", &v2alpha1.Inference{}, &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ annotationInferenceSpec: "{\"metadata\":{\"creationTimestamp\":null},\"spec\":{\"name\":\"\",\"image\":\"\"}}", }, }, }, false, }, { "unequal deployment need update", &v2alpha1.Inference{ Spec: v2alpha1.InferenceSpec{ Scaling: &v2alpha1.ScalingConfig{ MinReplicas: Ptr(int32(2)), }, }, }, &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ annotationInferenceSpec: "{\"metadata\":{\"creationTimestamp\":null},\"spec\":{\"name\":\"\",\"image\":\"\"}}", }, }}, true, }, } for _, s := range scenarios { t.Run(s.name, func(t *testing.T) { needUpdate := deploymentNeedsUpdate(s.inference, s.deploy) if needUpdate != s.expected { t.Errorf("incorrect judgement of need update: expected %v, got %v", s.expected, needUpdate) t.Fail() } }) } } ================================================ FILE: modelzetes/pkg/controller/factory.go ================================================ package controller import ( "github.com/tensorchord/openmodelz/agent/api/types" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes" v2alpha1 "github.com/tensorchord/openmodelz/modelzetes/pkg/apis/modelzetes/v2alpha1" "github.com/tensorchord/openmodelz/modelzetes/pkg/consts" "github.com/tensorchord/openmodelz/modelzetes/pkg/k8s" ) // FunctionFactory wraps modelzetes factory type FunctionFactory struct { Factory k8s.FunctionFactory } func NewFunctionFactory(clientset kubernetes.Interface, config k8s.DeploymentConfig) FunctionFactory { return FunctionFactory{ k8s.FunctionFactory{ Client: clientset, Config: config, }, } } func functionToResourceRequirements(in *v2alpha1.Inference) types.ResourceRequirements { resources := types.ResourceRequirements{} if in.Spec.Resources == nil { return resources } gpuLimit := in.Spec.Resources.Limits[consts.ResourceNvidiaGPU] gpuLimitPtr := &gpuLimit gpuRequest := in.Spec.Resources.Requests[consts.ResourceNvidiaGPU] gpuRequestsPtr := &gpuRequest resources = types.ResourceRequirements{ Limits: types.ResourceList{ types.ResourceCPU: types.Quantity( in.Spec.Resources.Limits.Cpu().String()), types.ResourceMemory: types.Quantity( in.Spec.Resources.Limits.Memory().String()), types.ResourceGPU: types.Quantity(gpuLimitPtr.String()), }, Requests: types.ResourceList{ types.ResourceCPU: types.Quantity( in.Spec.Resources.Requests.Cpu().String()), types.ResourceMemory: types.Quantity( in.Spec.Resources.Requests.Memory().String()), types.ResourceGPU: types.Quantity(gpuRequestsPtr.String()), }, } return resources } func (f *FunctionFactory) MakeHuggingfacePullThroughCacheEnvVar() *corev1.EnvVar { if f.Factory.Config.HuggingfacePullThroughCache { return &corev1.EnvVar{ Name: "HF_ENDPOINT", Value: f.Factory.Config.HuggingfacePullThroughCacheEndpoint, } } return nil } func (f *FunctionFactory) MakeProbes(function *v2alpha1.Inference, port int) ( *k8s.FunctionProbes, error) { // For old version inference without HTTPProbePath httpProbePath := consts.DefaultHTTPProbePath if (function.Spec.HTTPProbePath != nil) && (*function.Spec.HTTPProbePath != "") { httpProbePath = *function.Spec.HTTPProbePath } return f.Factory.MakeProbes(port, httpProbePath) } func (f *FunctionFactory) ConfigureReadOnlyRootFilesystem(function *v2alpha1.Inference, deployment *appsv1.Deployment) { f.Factory.ConfigureReadOnlyRootFilesystem(deployment) } func (f *FunctionFactory) ConfigureContainerUserID(deployment *appsv1.Deployment) { f.Factory.ConfigureContainerUserID(deployment) } ================================================ FILE: modelzetes/pkg/controller/framework_test.go ================================================ package controller import ( "strconv" "testing" v2alpha1 "github.com/tensorchord/openmodelz/modelzetes/pkg/apis/modelzetes/v2alpha1" . "github.com/tensorchord/openmodelz/modelzetes/pkg/pointer" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/fake" ) func Test_newDeployment_FrameworkGradio(t *testing.T) { expectEnv := map[string]string{"GRADIO_SERVER_NAME": "0.0.0.0", "GRADIO_SERVER_PORT": "7860"} inference := &v2alpha1.Inference{ ObjectMeta: metav1.ObjectMeta{ Name: "kubesec", }, Spec: v2alpha1.InferenceSpec{ Name: "kubesec", Image: "docker.io/kubesec/kubesec", HTTPProbePath: Ptr("/"), Annotations: map[string]string{}, Framework: v2alpha1.FrameworkGradio, }, } factory := NewFunctionFactory(fake.NewSimpleClientset(), defaultK8sConfig) secrets := map[string]*corev1.Secret{} deployment := newDeployment(inference, nil, secrets, factory) assertEnv(t, expectEnv, deployment.Spec.Template.Spec.Containers[0].Env) } func Test_newDeployment_FrameworkMosec(t *testing.T) { expectEnv := map[string]string{"MOSEC_PORT": strconv.Itoa(defaultPort)} inference := &v2alpha1.Inference{ ObjectMeta: metav1.ObjectMeta{ Name: "kubesec", }, Spec: v2alpha1.InferenceSpec{ Name: "kubesec", Image: "docker.io/kubesec/kubesec", HTTPProbePath: Ptr("/"), Annotations: map[string]string{}, Framework: v2alpha1.FrameworkMosec, }, } factory := NewFunctionFactory(fake.NewSimpleClientset(), defaultK8sConfig) secrets := map[string]*corev1.Secret{} deployment := newDeployment(inference, nil, secrets, factory) assertEnv(t, expectEnv, deployment.Spec.Template.Spec.Containers[0].Env) } func Test_newDeployment_FrameworkStreamlit(t *testing.T) { expectEnv := map[string]string{ "STREAMLIT_SERVER_ENABLE_CORS": "false", "STREAMLIT_SERVER_ADDRESS": "0.0.0.0", "STREAMLIT_SERVER_ENABLE_XSRF_PROTECTION": "false"} inference := &v2alpha1.Inference{ ObjectMeta: metav1.ObjectMeta{ Name: "kubesec", }, Spec: v2alpha1.InferenceSpec{ Name: "kubesec", Image: "docker.io/kubesec/kubesec", HTTPProbePath: Ptr("/"), Annotations: map[string]string{}, Framework: v2alpha1.FrameworkStreamlit, }, } factory := NewFunctionFactory(fake.NewSimpleClientset(), defaultK8sConfig) secrets := map[string]*corev1.Secret{} deployment := newDeployment(inference, nil, secrets, factory) assertEnv(t, expectEnv, deployment.Spec.Template.Spec.Containers[0].Env) } ================================================ FILE: modelzetes/pkg/controller/fromconfig.go ================================================ package controller import ( "errors" "fmt" kubeinformers "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/clientcmd" clientset "github.com/tensorchord/openmodelz/modelzetes/pkg/client/clientset/versioned" informers "github.com/tensorchord/openmodelz/modelzetes/pkg/client/informers/externalversions" "github.com/tensorchord/openmodelz/modelzetes/pkg/config" "github.com/tensorchord/openmodelz/modelzetes/pkg/consts" "github.com/tensorchord/openmodelz/modelzetes/pkg/k8s" ) func New(c config.Config, stopCh <-chan struct{}) (*Controller, error) { clientCmdConfig, err := clientcmd.BuildConfigFromFlags( c.KubeConfig.MasterURL, c.KubeConfig.Kubeconfig) if err != nil { return nil, fmt.Errorf("error building kubeconfig: %s", err.Error()) } clientCmdConfig.QPS = float32(c.KubeConfig.QPS) clientCmdConfig.Burst = c.KubeConfig.Burst kubeClient, err := kubernetes.NewForConfig(clientCmdConfig) if err != nil { return nil, fmt.Errorf("error building Kubernetes clientset: %s", err.Error()) } inferenceClient, err := clientset.NewForConfig(clientCmdConfig) if err != nil { return nil, fmt.Errorf("error building Inference clientset: %s", err.Error()) } deployConfig := k8s.DeploymentConfig{ HTTPProbe: true, SetNonRootUser: false, ReadinessProbe: &k8s.ProbeConfig{ InitialDelaySeconds: int32(c.Probes.Readiness.InitialDelaySeconds), TimeoutSeconds: int32(c.Probes.Readiness.TimeoutSeconds), PeriodSeconds: int32(c.Probes.Readiness.PeriodSeconds), }, LivenessProbe: &k8s.ProbeConfig{ InitialDelaySeconds: int32(c.Probes.Liveness.InitialDelaySeconds), TimeoutSeconds: int32(c.Probes.Liveness.TimeoutSeconds), PeriodSeconds: int32(c.Probes.Liveness.PeriodSeconds), }, StartupProbe: &k8s.ProbeConfig{ InitialDelaySeconds: int32(c.Probes.Startup.InitialDelaySeconds), TimeoutSeconds: int32(c.Probes.Startup.TimeoutSeconds), PeriodSeconds: int32(c.Probes.Startup.PeriodSeconds), }, ImagePullPolicy: c.Inference.ImagePullPolicy, RuntimeClassNvidia: c.Inference.SetUpRuntimeClassNvidia, ProfilesNamespace: "default", } if c.HuggingfaceProxy.Endpoint == "" { deployConfig.HuggingfacePullThroughCache = false } else { deployConfig.HuggingfacePullThroughCache = true deployConfig.HuggingfacePullThroughCacheEndpoint = c.HuggingfaceProxy.Endpoint } kubeInformerFactory := kubeinformers.NewSharedInformerFactoryWithOptions(kubeClient, c.KubeConfig.ResyncPeriod) inferenceInformerFactory := informers.NewSharedInformerFactoryWithOptions(inferenceClient, c.KubeConfig.ResyncPeriod) inferences := inferenceInformerFactory.Tensorchord().V2alpha1().Inferences() go inferences.Informer().Run(stopCh) if ok := cache.WaitForNamedCacheSync( fmt.Sprintf("%s:inferences", consts.ProviderName), stopCh, inferences.Informer().HasSynced); !ok { return nil, errors.New("failed to wait for inference caches to sync") } deployments := kubeInformerFactory.Apps().V1().Deployments() go deployments.Informer().Run(stopCh) if ok := cache.WaitForNamedCacheSync( fmt.Sprintf("%s:deployments", consts.ProviderName), stopCh, deployments.Informer().HasSynced); !ok { return nil, errors.New("failed to wait for deployment caches to sync") } controllerFactory := NewFunctionFactory(kubeClient, deployConfig) ctr := NewController( kubeClient, inferenceClient, kubeInformerFactory, inferenceInformerFactory, controllerFactory) return ctr, nil } ================================================ FILE: modelzetes/pkg/controller/replicas_test.go ================================================ package controller import ( "testing" appsv1 "k8s.io/api/apps/v1" "k8s.io/client-go/kubernetes/fake" v2alpha1 "github.com/tensorchord/openmodelz/modelzetes/pkg/apis/modelzetes/v2alpha1" "github.com/tensorchord/openmodelz/modelzetes/pkg/k8s" . "github.com/tensorchord/openmodelz/modelzetes/pkg/pointer" ) func Test_Replicas(t *testing.T) { scenarios := []struct { name string inference *v2alpha1.Inference deploy *appsv1.Deployment expected *int32 }{ { "return nil replicas when label is missing and deployment does not exist", &v2alpha1.Inference{}, nil, nil, }, { "return nil replicas when label is missing and deployment has no replicas", &v2alpha1.Inference{}, &appsv1.Deployment{}, nil, }, { "return min replicas when label is present and deployment has nil replicas", &v2alpha1.Inference{ Spec: v2alpha1.InferenceSpec{ Scaling: &v2alpha1.ScalingConfig{ MinReplicas: Ptr(int32(2)), }, }, }, &appsv1.Deployment{Spec: appsv1.DeploymentSpec{Replicas: nil}}, int32p(2), }, { "return min replicas when label is present and deployment has replicas less than min", &v2alpha1.Inference{ Spec: v2alpha1.InferenceSpec{ Scaling: &v2alpha1.ScalingConfig{ MinReplicas: Ptr(int32(2)), }, }, }, &appsv1.Deployment{Spec: appsv1.DeploymentSpec{Replicas: int32p(1)}}, int32p(2), }, { "return existing replicas when label is present and deployment has more replicas than min", &v2alpha1.Inference{ Spec: v2alpha1.InferenceSpec{ Scaling: &v2alpha1.ScalingConfig{ MinReplicas: Ptr(int32(2)), }, }, }, &appsv1.Deployment{Spec: appsv1.DeploymentSpec{Replicas: int32p(3)}}, int32p(3), }, { "return existing replicas when label is missing and deployment has replicas set by HPA", &v2alpha1.Inference{ Spec: v2alpha1.InferenceSpec{ Scaling: &v2alpha1.ScalingConfig{}, }, }, &appsv1.Deployment{Spec: appsv1.DeploymentSpec{Replicas: int32p(3)}}, int32p(3), }, { "return zero replicas when label is present and deployment has zero replicas", &v2alpha1.Inference{ Spec: v2alpha1.InferenceSpec{ Scaling: &v2alpha1.ScalingConfig{ MinReplicas: Ptr(int32(2)), }, }, }, &appsv1.Deployment{Spec: appsv1.DeploymentSpec{Replicas: int32p(0)}}, int32p(2), }, } factory := NewFunctionFactory(fake.NewSimpleClientset(), k8s.DeploymentConfig{ LivenessProbe: &k8s.ProbeConfig{}, ReadinessProbe: &k8s.ProbeConfig{}, StartupProbe: &k8s.ProbeConfig{}, }) for _, s := range scenarios { t.Run(s.name, func(t *testing.T) { deploy := newDeployment(s.inference, s.deploy, nil, factory) value := deploy.Spec.Replicas if s.expected != nil && value != nil { if *s.expected != *value { t.Errorf("incorrect replica count: expected %v, got %v", *s.expected, *value) } } else if s.expected != value { t.Errorf("incorrect replica count: expected %v, got %v", s.expected, value) } }) } } ================================================ FILE: modelzetes/pkg/controller/secrets.go ================================================ package controller import ( "fmt" v2alpha1 "github.com/tensorchord/openmodelz/modelzetes/pkg/apis/modelzetes/v2alpha1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" ) const ( secretsMountPath = "/var/openfaas/secrets" ) // UpdateSecrets will update the Deployment spec to include secrets that have been deployed // in the kubernetes cluster. For each requested secret, we inspect the type and add it to the // deployment spec as appropriate: secrets with type `SecretTypeDockercfg` are added as ImagePullSecrets // all other secrets are mounted as files in the deployments containers. func UpdateSecrets(function *v2alpha1.Inference, deployment *appsv1.Deployment, existingSecrets map[string]*corev1.Secret) error { // Add / reference pre-existing secrets within Kubernetes secretVolumeProjections := []corev1.VolumeProjection{} for _, secretName := range function.Spec.Secrets { deployedSecret, ok := existingSecrets[secretName] if !ok { return fmt.Errorf("required secret '%s' was not found in the cluster", secretName) } switch deployedSecret.Type { case corev1.SecretTypeDockercfg, corev1.SecretTypeDockerConfigJson: deployment.Spec.Template.Spec.ImagePullSecrets = append( deployment.Spec.Template.Spec.ImagePullSecrets, corev1.LocalObjectReference{ Name: secretName, }, ) default: projectedPaths := []corev1.KeyToPath{} for secretKey := range deployedSecret.Data { projectedPaths = append(projectedPaths, corev1.KeyToPath{Key: secretKey, Path: secretKey}) } projection := &corev1.SecretProjection{Items: projectedPaths} projection.Name = secretName secretProjection := corev1.VolumeProjection{ Secret: projection, } secretVolumeProjections = append(secretVolumeProjections, secretProjection) } } volumeName := fmt.Sprintf("%s-projected-secrets", function.Spec.Name) projectedSecrets := corev1.Volume{ Name: volumeName, VolumeSource: corev1.VolumeSource{ Projected: &corev1.ProjectedVolumeSource{ Sources: secretVolumeProjections, }, }, } // remove the existing secrets volume, if we can find it. The update volume will be // added below existingVolumes := removeVolume(volumeName, deployment.Spec.Template.Spec.Volumes) deployment.Spec.Template.Spec.Volumes = existingVolumes if len(secretVolumeProjections) > 0 { deployment.Spec.Template.Spec.Volumes = append(existingVolumes, projectedSecrets) } // add mount secret as a file updatedContainers := []corev1.Container{} for _, container := range deployment.Spec.Template.Spec.Containers { mount := corev1.VolumeMount{ Name: volumeName, ReadOnly: true, MountPath: secretsMountPath, } // remove the existing secrets volume mount, if we can find it. We update it later. container.VolumeMounts = removeVolumeMount(volumeName, container.VolumeMounts) if len(secretVolumeProjections) > 0 { container.VolumeMounts = append(container.VolumeMounts, mount) } updatedContainers = append(updatedContainers, container) } deployment.Spec.Template.Spec.Containers = updatedContainers return nil } // removeVolume returns a Volume slice with any volumes matching volumeName removed. // Uses the filter without allocation technique // https://github.com/golang/go/wiki/SliceTricks#filtering-without-allocating func removeVolume(volumeName string, volumes []corev1.Volume) []corev1.Volume { newVolumes := volumes[:0] for _, v := range volumes { if v.Name != volumeName { newVolumes = append(newVolumes, v) } } return newVolumes } // removeVolumeMount returns a VolumeMount slice with any mounts matching volumeName removed // Uses the filter without allocation technique // https://github.com/golang/go/wiki/SliceTricks#filtering-without-allocating func removeVolumeMount(volumeName string, mounts []corev1.VolumeMount) []corev1.VolumeMount { newMounts := mounts[:0] for _, v := range mounts { if v.Name != volumeName { newMounts = append(newMounts, v) } } return newMounts } ================================================ FILE: modelzetes/pkg/controller/secrets_test.go ================================================ package controller import ( "fmt" "testing" v2alpha1 "github.com/tensorchord/openmodelz/modelzetes/pkg/apis/modelzetes/v2alpha1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" ) func Test_UpdateSecrets_DoesNotAddVolumeIfRequestSecretsIsNil(t *testing.T) { request := &v2alpha1.Inference{ Spec: v2alpha1.InferenceSpec{ Name: "testfunc", Secrets: nil, }, } existingSecrets := map[string]*corev1.Secret{ "pullsecret": {Type: corev1.SecretTypeDockercfg}, "testsecret": {Type: corev1.SecretTypeOpaque, Data: map[string][]byte{"filename": []byte("contents")}}, } deployment := &appsv1.Deployment{ Spec: appsv1.DeploymentSpec{ Template: corev1.PodTemplateSpec{ Spec: corev1.PodSpec{ Containers: []corev1.Container{ {Name: "testfunc", Image: "alpine:latest"}, }, }, }, }, } err := UpdateSecrets(request, deployment, existingSecrets) if err != nil { t.Errorf("unexpected error %s", err.Error()) } validateEmptySecretVolumesAndMounts(t, deployment) } func Test_UpdateSecrets_DoesNotAddVolumeIfRequestSecretsIsEmpty(t *testing.T) { request := &v2alpha1.Inference{ Spec: v2alpha1.InferenceSpec{ Name: "testfunc", Secrets: []string{}, }, } existingSecrets := map[string]*corev1.Secret{ "pullsecret": {Type: corev1.SecretTypeDockercfg}, "testsecret": {Type: corev1.SecretTypeOpaque, Data: map[string][]byte{"filename": []byte("contents")}}, } deployment := &appsv1.Deployment{ Spec: appsv1.DeploymentSpec{ Template: corev1.PodTemplateSpec{ Spec: corev1.PodSpec{ Containers: []corev1.Container{ {Name: "testfunc", Image: "alpine:latest"}, }, }, }, }, } err := UpdateSecrets(request, deployment, existingSecrets) if err != nil { t.Errorf("unexpected error %s", err.Error()) } validateEmptySecretVolumesAndMounts(t, deployment) } func Test_UpdateSecrets_RemovesAllCopiesOfExitingSecretsVolumes(t *testing.T) { volumeName := "testfunc-projected-secrets" request := &v2alpha1.Inference{ Spec: v2alpha1.InferenceSpec{ Name: "testfunc", Secrets: []string{}, }, } existingSecrets := map[string]*corev1.Secret{ "pullsecret": {Type: corev1.SecretTypeDockercfg}, "testsecret": {Type: corev1.SecretTypeOpaque, Data: map[string][]byte{"filename": []byte("contents")}}, } deployment := &appsv1.Deployment{ Spec: appsv1.DeploymentSpec{ Template: corev1.PodTemplateSpec{ Spec: corev1.PodSpec{ Containers: []corev1.Container{ { Name: "testfunc", Image: "alpine:latest", VolumeMounts: []corev1.VolumeMount{ { Name: volumeName, }, { Name: volumeName, }, }, }, }, Volumes: []corev1.Volume{ { Name: volumeName, }, { Name: volumeName, }, }, }, }, }, } err := UpdateSecrets(request, deployment, existingSecrets) if err != nil { t.Errorf("unexpected error %s", err.Error()) } validateEmptySecretVolumesAndMounts(t, deployment) } func Test_UpdateSecrets_AddNewSecretVolume(t *testing.T) { request := &v2alpha1.Inference{ Spec: v2alpha1.InferenceSpec{ Name: "testfunc", Secrets: []string{"pullsecret", "testsecret"}, }, } existingSecrets := map[string]*corev1.Secret{ "pullsecret": {Type: corev1.SecretTypeDockercfg}, "testsecret": {Type: corev1.SecretTypeOpaque, Data: map[string][]byte{"filename": []byte("contents")}}, } deployment := &appsv1.Deployment{ Spec: appsv1.DeploymentSpec{ Template: corev1.PodTemplateSpec{ Spec: corev1.PodSpec{ Containers: []corev1.Container{ {Name: "testfunc", Image: "alpine:latest"}, }, }, }, }, } err := UpdateSecrets(request, deployment, existingSecrets) if err != nil { t.Errorf("unexpected error %s", err.Error()) } validateNewSecretVolumesAndMounts(t, deployment) } func Test_UpdateSecrets_ReplacesPreviousSecretMountWithNewMount(t *testing.T) { request := &v2alpha1.Inference{ Spec: v2alpha1.InferenceSpec{ Name: "testfunc", Secrets: []string{"pullsecret", "testsecret"}, }, } existingSecrets := map[string]*corev1.Secret{ "pullsecret": {Type: corev1.SecretTypeDockercfg}, "testsecret": {Type: corev1.SecretTypeOpaque, Data: map[string][]byte{"filename": []byte("contents")}}, } deployment := &appsv1.Deployment{ Spec: appsv1.DeploymentSpec{ Template: corev1.PodTemplateSpec{ Spec: corev1.PodSpec{ Containers: []corev1.Container{ {Name: "testfunc", Image: "alpine:latest"}, }, }, }, }, } err := UpdateSecrets(request, deployment, existingSecrets) if err != nil { t.Errorf("unexpected error %s", err.Error()) } // mimic the deployment already existing and deployed with the same secrets by running // UpdateSecrets twice, the first run represents the original deployment, the second run represents // retrieving the deployment from the k8s api and applying the update to it err = UpdateSecrets(request, deployment, existingSecrets) if err != nil { t.Errorf("unexpected error %s", err.Error()) } validateNewSecretVolumesAndMounts(t, deployment) } func Test_UpdateSecrets_RemovesSecretsVolumeIfRequestSecretsIsEmptyOrNil(t *testing.T) { request := &v2alpha1.Inference{ Spec: v2alpha1.InferenceSpec{ Name: "testfunc", Secrets: []string{"pullsecret", "testsecret"}, }, } existingSecrets := map[string]*corev1.Secret{ "pullsecret": {Type: corev1.SecretTypeDockercfg}, "testsecret": {Type: corev1.SecretTypeOpaque, Data: map[string][]byte{"filename": []byte("contents")}}, } deployment := &appsv1.Deployment{ Spec: appsv1.DeploymentSpec{ Template: corev1.PodTemplateSpec{ Spec: corev1.PodSpec{ Containers: []corev1.Container{ {Name: "testfunc", Image: "alpine:latest"}, }, }, }, }, } err := UpdateSecrets(request, deployment, existingSecrets) if err != nil { t.Errorf("unexpected error %s", err.Error()) } validateNewSecretVolumesAndMounts(t, deployment) request = &v2alpha1.Inference{ Spec: v2alpha1.InferenceSpec{ Name: "testfunc", Secrets: []string{}, }, } err = UpdateSecrets(request, deployment, existingSecrets) if err != nil { t.Errorf("unexpected error %s", err.Error()) } validateEmptySecretVolumesAndMounts(t, deployment) } func validateEmptySecretVolumesAndMounts(t *testing.T, deployment *appsv1.Deployment) { numVolumes := len(deployment.Spec.Template.Spec.Volumes) if numVolumes != 0 { fmt.Printf("%+v", deployment.Spec.Template.Spec.Volumes) t.Errorf("Incorrect number of volumes: expected 0, got %d", numVolumes) } c := deployment.Spec.Template.Spec.Containers[0] numVolumeMounts := len(c.VolumeMounts) if numVolumeMounts != 0 { t.Errorf("Incorrect number of volumes mounts: expected 0, got %d", numVolumeMounts) } } func validateNewSecretVolumesAndMounts(t *testing.T, deployment *appsv1.Deployment) { numVolumes := len(deployment.Spec.Template.Spec.Volumes) if numVolumes != 1 { t.Errorf("Incorrect number of volumes: expected 1, got %d", numVolumes) } volume := deployment.Spec.Template.Spec.Volumes[0] if volume.Name != "testfunc-projected-secrets" { t.Errorf("Incorrect volume name: expected \"testfunc-projected-secrets\", got \"%s\"", volume.Name) } if volume.VolumeSource.Projected == nil { t.Error("Secrets volume is not a projected volume type") } if volume.VolumeSource.Projected.Sources[0].Secret.Items[0].Key != "filename" { t.Error("Project secret not constructed correctly") } c := deployment.Spec.Template.Spec.Containers[0] numVolumeMounts := len(c.VolumeMounts) if numVolumeMounts != 1 { t.Errorf("Incorrect number of volumes mounts: expected 1, got %d", numVolumeMounts) } mount := c.VolumeMounts[0] if mount.Name != "testfunc-projected-secrets" { t.Errorf("Incorrect volume mounts: expected \"testfunc-projected-secrets\", got \"%s\"", mount.Name) } if mount.MountPath != secretsMountPath { t.Errorf("Incorrect volume mount path: expected \"%s\", got \"%s\"", secretsMountPath, mount.MountPath) } } ================================================ FILE: modelzetes/pkg/controller/service.go ================================================ package controller import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/intstr" v2alpha1 "github.com/tensorchord/openmodelz/modelzetes/pkg/apis/modelzetes/v2alpha1" "github.com/tensorchord/openmodelz/modelzetes/pkg/consts" ) // newService creates a new ClusterIP Service for a Function resource. It also sets // the appropriate OwnerReferences on the resource so handleObject can discover // the Function resource that 'owns' it. func newService(function *v2alpha1.Inference) *corev1.Service { return &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: consts.DefaultServicePrefix + function.Spec.Name, Namespace: function.Namespace, Annotations: map[string]string{"prometheus.io.scrape": "false"}, OwnerReferences: []metav1.OwnerReference{ *metav1.NewControllerRef(function, schema.GroupVersionKind{ Group: v2alpha1.SchemeGroupVersion.Group, Version: v2alpha1.SchemeGroupVersion.Version, Kind: v2alpha1.Kind, }), }, }, Spec: corev1.ServiceSpec{ Type: corev1.ServiceTypeClusterIP, Selector: map[string]string{consts.LabelInferenceName: function.Spec.Name}, Ports: []corev1.ServicePort{ { Name: "http", Protocol: corev1.ProtocolTCP, Port: functionPort, TargetPort: intstr.IntOrString{ Type: intstr.Int, IntVal: int32(makePort(function)), }, }, }, }, } } ================================================ FILE: modelzetes/pkg/controller/service_test.go ================================================ package controller import ( "strings" "testing" v2alpha1 "github.com/tensorchord/openmodelz/modelzetes/pkg/apis/modelzetes/v2alpha1" . "github.com/tensorchord/openmodelz/modelzetes/pkg/pointer" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func Test_newService(t *testing.T) { inference := &v2alpha1.Inference{ ObjectMeta: metav1.ObjectMeta{ Name: "kubesec", Namespace: "mock-space", }, Spec: v2alpha1.InferenceSpec{ Name: "kubesec", Image: "docker.io/kubesec/kubesec", HTTPProbePath: Ptr("/"), Annotations: map[string]string{}, }, } service := newService(inference) if !strings.Contains(service.ObjectMeta.Name, inference.ObjectMeta.Name) { t.Errorf("Service name %s should contains inference name %s", service.ObjectMeta.Name, inference.ObjectMeta.Name) t.Fail() } if service.ObjectMeta.Namespace != inference.ObjectMeta.Namespace { t.Errorf("Service namespace %s should be equal to inference namespace %s", service.ObjectMeta.Namespace, inference.ObjectMeta.Namespace) t.Fail() } } ================================================ FILE: modelzetes/pkg/k8s/config.go ================================================ // Copyright 2020 OpenFaaS Authors // Licensed under the MIT license. See LICENSE file in the project root for full license information. package k8s // ProbeConfig holds the deployment liveness and readiness options type ProbeConfig struct { InitialDelaySeconds int32 TimeoutSeconds int32 PeriodSeconds int32 } // DeploymentConfig holds the global deployment options type DeploymentConfig struct { HTTPProbe bool ReadinessProbe *ProbeConfig LivenessProbe *ProbeConfig StartupProbe *ProbeConfig HuggingfacePullThroughCache bool HuggingfacePullThroughCacheEndpoint string ImagePullPolicy string RuntimeClassNvidia bool // SetNonRootUser will override the function image user to ensure that it is not root. When // true, the user will set to 12000 for all functions. SetNonRootUser bool // ProfilesNamespace defines which namespace is used to look up available Profiles. ProfilesNamespace string } ================================================ FILE: modelzetes/pkg/k8s/errors.go ================================================ // Copyright 2020 OpenFaaS Authors // Licensed under the MIT license. See LICENSE file in the project root for full license information. package k8s import ( k8serrors "k8s.io/apimachinery/pkg/api/errors" ) // isNotFound tests if the error is a kubernetes API error that indicates that the object // was not found or does not exist func IsNotFound(err error) bool { return k8serrors.IsNotFound(err) || k8serrors.IsGone(err) } ================================================ FILE: modelzetes/pkg/k8s/factory.go ================================================ // Copyright 2020 OpenFaaS Authors // Licensed under the MIT license. See LICENSE file in the project root for full license information. package k8s import ( "k8s.io/client-go/kubernetes" "github.com/tensorchord/openmodelz/modelzetes/pkg/client/clientset/versioned/typed/modelzetes/v2alpha1" ) // FunctionFactory is handling Kubernetes operations to materialise functions into deployments and services type FunctionFactory struct { Client kubernetes.Interface Config DeploymentConfig } func NewFunctionFactory(clientset kubernetes.Interface, config DeploymentConfig, inferenceclientset v2alpha1.InferenceInterface) FunctionFactory { return FunctionFactory{ Client: clientset, Config: config, } } ================================================ FILE: modelzetes/pkg/k8s/factory_test.go ================================================ // Copyright 2020 OpenFaaS Authors // Licensed under the MIT license. See LICENSE file in the project root for full license information. package k8s import "k8s.io/client-go/kubernetes/fake" func mockFactory() FunctionFactory { return NewFunctionFactory(fake.NewSimpleClientset(), DeploymentConfig{ HTTPProbe: false, LivenessProbe: &ProbeConfig{ PeriodSeconds: 1, TimeoutSeconds: 3, InitialDelaySeconds: 0, }, ReadinessProbe: &ProbeConfig{ PeriodSeconds: 1, TimeoutSeconds: 3, InitialDelaySeconds: 0, }, StartupProbe: &ProbeConfig{ PeriodSeconds: 1, TimeoutSeconds: 3, InitialDelaySeconds: 0, }, }, nil) } ================================================ FILE: modelzetes/pkg/k8s/instance.go ================================================ package k8s import ( types "github.com/tensorchord/openmodelz/agent/api/types" "github.com/tensorchord/openmodelz/modelzetes/pkg/consts" v1 "k8s.io/api/core/v1" ) func MakeLabelSelector(name string) map[string]string { return map[string]string{ "app": name, } } func InstanceFromPod(pod v1.Pod) *types.InferenceDeploymentInstance { i := &types.InferenceDeploymentInstance{ Spec: types.InferenceDeploymentInstanceSpec{ Namespace: pod.Namespace, Name: pod.Name, OwnerReference: pod.Labels[consts.LabelInferenceName], }, Status: types.InferenceDeploymentInstanceStatus{ StartTime: pod.Status.StartTime.Time, Reason: pod.Status.Reason, Message: pod.Status.Message, }, } switch pod.Status.Phase { case v1.PodRunning: i.Status.Phase = types.InstancePhaseRunning case v1.PodPending: i.Status.Phase = types.InstancePhasePending for _, c := range pod.Status.Conditions { if c.Type == v1.PodScheduled && c.Status == v1.ConditionFalse { i.Status.Phase = types.InstancePhaseScheduling break } } case v1.PodFailed: i.Status.Phase = types.InstancePhaseFailed case v1.PodSucceeded: i.Status.Phase = types.InstancePhaseSucceeded case v1.PodUnknown: i.Status.Phase = types.InstancePhaseUnknown } if pod.Status.ContainerStatuses[0].Started != nil && !*pod.Status.ContainerStatuses[0].Started { i.Status.Phase = types.InstancePhaseCreating if pod.Status.ContainerStatuses[0].State.Waiting != nil { i.Status.Reason = pod.Status.ContainerStatuses[0].State.Waiting.Reason i.Status.Message = pod.Status.ContainerStatuses[0].State.Waiting.Message i.Status.Phase = types.InstancePhase( pod.Status.ContainerStatuses[0].State.Waiting.Reason) } else if pod.Status.ContainerStatuses[0].State.Running != nil { i.Status.Phase = types.InstancePhaseInitializing } } return i } ================================================ FILE: modelzetes/pkg/k8s/instance_test.go ================================================ package k8s import ( "testing" "time" "github.com/google/go-cmp/cmp" types "github.com/tensorchord/openmodelz/agent/api/types" . "github.com/tensorchord/openmodelz/modelzetes/pkg/pointer" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) var ( mock_time, _ = time.Parse("2006-01-02", "2023-08-31") ) func Test_InstanceFromPod(t *testing.T) { scenarios := []struct { name string pod v1.Pod expected types.InferenceDeploymentInstance }{ { "basic pod", v1.Pod{ Status: v1.PodStatus{ StartTime: Ptr(metav1.NewTime(mock_time)), ContainerStatuses: []v1.ContainerStatus{ {}, }, }, }, types.InferenceDeploymentInstance{ Status: types.InferenceDeploymentInstanceStatus{ StartTime: mock_time, }, }, }, { "phase running pod", v1.Pod{ Status: v1.PodStatus{ Phase: v1.PodRunning, StartTime: Ptr(metav1.NewTime(mock_time)), ContainerStatuses: []v1.ContainerStatus{ {}, }, }, }, types.InferenceDeploymentInstance{ Status: types.InferenceDeploymentInstanceStatus{ Phase: types.InstancePhaseRunning, StartTime: mock_time, }, }, }, { "phase pending pod", v1.Pod{ Status: v1.PodStatus{ Phase: v1.PodPending, StartTime: Ptr(metav1.NewTime(mock_time)), ContainerStatuses: []v1.ContainerStatus{ {}, }, }, }, types.InferenceDeploymentInstance{ Status: types.InferenceDeploymentInstanceStatus{ Phase: types.InstancePhasePending, StartTime: mock_time, }, }, }, { "phase scheduling pod", v1.Pod{ Status: v1.PodStatus{ Phase: v1.PodPending, StartTime: Ptr(metav1.NewTime(mock_time)), ContainerStatuses: []v1.ContainerStatus{ {}, }, Conditions: []v1.PodCondition{ { Type: v1.PodScheduled, Status: v1.ConditionFalse, }, }, }, }, types.InferenceDeploymentInstance{ Status: types.InferenceDeploymentInstanceStatus{ Phase: types.InstancePhaseScheduling, StartTime: mock_time, }, }, }, { "phase failed pod", v1.Pod{ Status: v1.PodStatus{ Phase: v1.PodFailed, StartTime: Ptr(metav1.NewTime(mock_time)), ContainerStatuses: []v1.ContainerStatus{ {}, }, }, }, types.InferenceDeploymentInstance{ Status: types.InferenceDeploymentInstanceStatus{ Phase: types.InstancePhaseFailed, StartTime: mock_time, }, }, }, { "phase succeed pod", v1.Pod{ Status: v1.PodStatus{ Phase: v1.PodSucceeded, StartTime: Ptr(metav1.NewTime(mock_time)), ContainerStatuses: []v1.ContainerStatus{ {}, }, }, }, types.InferenceDeploymentInstance{ Status: types.InferenceDeploymentInstanceStatus{ Phase: types.InstancePhaseSucceeded, StartTime: mock_time, }, }, }, { "phase unknown pod", v1.Pod{ Status: v1.PodStatus{ Phase: v1.PodUnknown, StartTime: Ptr(metav1.NewTime(mock_time)), ContainerStatuses: []v1.ContainerStatus{ {}, }, }, }, types.InferenceDeploymentInstance{ Status: types.InferenceDeploymentInstanceStatus{ Phase: types.InstancePhaseUnknown, StartTime: mock_time, }, }, }, { "phase creating pod", v1.Pod{ Status: v1.PodStatus{ Phase: v1.PodUnknown, StartTime: Ptr(metav1.NewTime(mock_time)), ContainerStatuses: []v1.ContainerStatus{ { Started: Ptr(false), }, }, }, }, types.InferenceDeploymentInstance{ Status: types.InferenceDeploymentInstanceStatus{ Phase: types.InstancePhaseCreating, StartTime: mock_time, }, }, }, { "phase initializing pod", v1.Pod{ Status: v1.PodStatus{ Phase: v1.PodUnknown, StartTime: Ptr(metav1.NewTime(mock_time)), ContainerStatuses: []v1.ContainerStatus{ { Started: Ptr(false), State: v1.ContainerState{ Running: Ptr(v1.ContainerStateRunning{ StartedAt: metav1.NewTime(mock_time), }), }, }, }, }, }, types.InferenceDeploymentInstance{ Status: types.InferenceDeploymentInstanceStatus{ Phase: types.InstancePhaseInitializing, StartTime: mock_time, }, }, }, { "phase waiting pod", v1.Pod{ Status: v1.PodStatus{ Phase: v1.PodUnknown, StartTime: Ptr(metav1.NewTime(mock_time)), ContainerStatuses: []v1.ContainerStatus{ { Started: Ptr(false), State: v1.ContainerState{ Waiting: Ptr(v1.ContainerStateWaiting{ Reason: "mock-reason", Message: "mock-message", }), }, }, }, }, }, types.InferenceDeploymentInstance{ Status: types.InferenceDeploymentInstanceStatus{ Phase: types.InstancePhase("mock-reason"), Reason: "mock-reason", Message: "mock-message", StartTime: mock_time, }, }, }, } for _, s := range scenarios { t.Run(s.name, func(t *testing.T) { instance := InstanceFromPod(s.pod) if diff := cmp.Diff(s.expected, *instance); diff != "" { t.Errorf("Create instance from pod: expected %v, got %v", s.expected, instance) t.Fail() } }) } } ================================================ FILE: modelzetes/pkg/k8s/log.go ================================================ // Copyright 2020 OpenFaaS Author(s) // Licensed under the MIT license. See LICENSE file in the project root for full license information. package k8s import ( "context" "log" "strings" "time" "github.com/tensorchord/openmodelz/agent/api/types" "k8s.io/client-go/kubernetes" ) // LogRequestor implements the Requestor interface for k8s type LogRequestor struct { client kubernetes.Interface functionNamespace string } // NewLogRequestor returns a new logs.Requestor that uses kail to select and follow pod logs func NewLogRequestor(client kubernetes.Interface, functionNamespace string) *LogRequestor { return &LogRequestor{ client: client, functionNamespace: functionNamespace, } } // Query implements the actual Swarm logs request logic for the Requestor interface // This implementation ignores the r.Limit value because the OF-Provider already handles server side // line limits. func (l LogRequestor) Query(ctx context.Context, r types.LogRequest) (<-chan types.Message, error) { ns := l.functionNamespace if len(r.Namespace) > 0 && strings.ToLower(r.Namespace) != "kube-system" { ns = r.Namespace } var since *time.Time if r.Since != "" { buf, err := time.Parse(time.RFC3339, r.Since) if err != nil { return nil, err } since = &buf } logStream, err := GetLogs(ctx, l.client, r.Name, ns, int64(r.Tail), since, r.Follow) if err != nil { log.Printf("LogRequestor: get logs failed: %s\n", err) return nil, err } msgStream := make(chan types.Message, LogBufferSize) go func() { defer close(msgStream) // here we depend on the fact that logStream will close when the context is cancelled, // this ensures that the go routine will resolve for msg := range logStream { msgStream <- types.Message{ Timestamp: msg.Timestamp, Text: msg.Text, Name: msg.FunctionName, Instance: msg.PodName, Namespace: msg.Namespace, } } }() return msgStream, nil } ================================================ FILE: modelzetes/pkg/k8s/logs.go ================================================ package k8s import ( "bufio" "bytes" "context" "io" "log" "strings" "time" "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/informers" "k8s.io/client-go/informers/internalinterfaces" "k8s.io/client-go/kubernetes" v1 "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/client-go/tools/cache" "github.com/tensorchord/openmodelz/modelzetes/pkg/consts" ) const ( // podInformerResync is the period between cache syncs in the pod informer podInformerResync = 5 * time.Second // defaultLogSince is the fallback log stream history defaultLogSince = 5 * time.Minute // LogBufferSize number of log messages that may be buffered LogBufferSize = 500 * 2 ) // Log is the object which will be used together with the template to generate // the output. type Log struct { // Text is the log message itself Text string `json:"text"` // Namespace of the pod Namespace string `json:"namespace"` // PodName of the instance PodName string `json:"podName"` // FunctionName of the pod FunctionName string `json:"FunctionName"` // Timestamp of the message Timestamp time.Time `json:"timestamp"` } // GetLogs returns a channel of logs for the given function func GetLogs(ctx context.Context, client kubernetes.Interface, functionName, namespace string, tail int64, since *time.Time, follow bool) (<-chan Log, error) { added, err := startFunctionPodInformer(ctx, client, functionName, namespace) if err != nil { return nil, err } logs := make(chan Log, LogBufferSize) go func() { var watching uint defer close(logs) finished := make(chan error) for { select { case <-ctx.Done(): return case <-finished: watching-- if watching == 0 && !follow { return } case p := <-added: watching++ go func() { finished <- podLogs(ctx, client.CoreV1().Pods(namespace), p, functionName, namespace, tail, since, follow, logs) }() } } }() return logs, nil } // podLogs returns a stream of logs lines from the specified pod func podLogs(ctx context.Context, i v1.PodInterface, pod, container, namespace string, tail int64, since *time.Time, follow bool, dst chan<- Log) error { log.Printf("Logger: starting log stream for %s\n", pod) defer log.Printf("Logger: stopping log stream for %s\n", pod) opts := &corev1.PodLogOptions{ Follow: follow, Timestamps: true, Container: container, } if tail > 0 { opts.TailLines = &tail } if opts.TailLines == nil || since != nil { opts.SinceSeconds = parseSince(since) } stream, err := i.GetLogs(pod, opts).Stream(context.TODO()) if err != nil { return err } defer stream.Close() done := make(chan error) go func() { reader := bufio.NewReader(stream) for { line, err := reader.ReadBytes('\n') if err != nil { done <- err return } msg, ts := extractTimestampAndMsg(string(bytes.Trim(line, "\x00"))) dst <- Log{ Timestamp: ts, Text: msg, PodName: pod, FunctionName: container, Namespace: namespace, } } }() select { case <-ctx.Done(): return ctx.Err() case err := <-done: if err != io.EOF { return err } return nil } } func extractTimestampAndMsg(logText string) (string, time.Time) { // first 32 characters is the k8s timestamp parts := strings.SplitN(logText, " ", 2) ts, err := time.Parse(time.RFC3339Nano, parts[0]) if err != nil { log.Printf("error: invalid timestamp '%s'\n", parts[0]) return "", time.Time{} } if len(parts) == 2 { return parts[1], ts } return "", ts } // parseSince returns the time.Duration of the requested Since value _or_ 5 minutes func parseSince(r *time.Time) *int64 { var since int64 if r == nil || r.IsZero() { since = int64(defaultLogSince.Seconds()) return &since } since = int64(time.Since(*r).Seconds()) return &since } // startFunctionPodInformer will gather the list of existing Pods for the function, it will then watch // and watch for newly added or deleted function instances. func startFunctionPodInformer(ctx context.Context, client kubernetes.Interface, functionName, namespace string) (<-chan string, error) { functionSelector := &metav1.LabelSelector{ MatchLabels: map[string]string{consts.LabelInferenceName: functionName}, } selector, err := metav1.LabelSelectorAsSelector(functionSelector) if err != nil { err = errors.Wrap(err, "unable to build function selector") log.Printf("PodInformer: %s", err) return nil, err } log.Printf("PodInformer: starting informer for %s in: %s\n", selector.String(), namespace) factory := informers.NewFilteredSharedInformerFactory( client, podInformerResync, namespace, withLabels(selector.String()), ) podInformer := factory.Core().V1().Pods() podsResp, err := client.CoreV1().Pods(namespace).List(context.TODO(), metav1.ListOptions{LabelSelector: selector.String()}) if err != nil { log.Printf("PodInformer: %s", err) return nil, err } pods := podsResp.Items if len(pods) == 0 { err = errors.New("no matching instances found") log.Printf("PodInformer: %s", err) return nil, k8serrors.NewNotFound(corev1.Resource("pods"), selector.String()) } // prepare channel with enough space for the current instance set added := make(chan string, len(pods)) podInformer.Informer().AddEventHandler(&podLoggerEventHandler{ added: added, }) // will add existing pods to the chan and then listen for any new pods go podInformer.Informer().Run(ctx.Done()) go func() { <-ctx.Done() close(added) }() return added, nil } func withLabels(selector string) internalinterfaces.TweakListOptionsFunc { return func(opts *metav1.ListOptions) { opts.LabelSelector = selector } } type podLoggerEventHandler struct { cache.ResourceEventHandler added chan<- string deleted chan<- string } func (h *podLoggerEventHandler) OnAdd(obj interface{}, isInitialList bool) { pod := obj.(*corev1.Pod) log.Printf("PodInformer: adding instance: %s", pod.Name) h.added <- pod.Name } func (h *podLoggerEventHandler) OnUpdate(oldObj, newObj interface{}) { // purposefully empty, we don't need to do anything for logs on update } func (h *podLoggerEventHandler) OnDelete(obj interface{}) { // this may not be needed, the log stream Reader _should_ close on its own without // us needing to watch and close it // pod := obj.(*corev1.Pod) // h.deleted <- pod.Name } ================================================ FILE: modelzetes/pkg/k8s/probes.go ================================================ // Copyright 2020 OpenFaaS Authors // Licensed under the MIT license. See LICENSE file in the project root for full license information. package k8s import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/intstr" ) type FunctionProbes struct { Liveness *corev1.Probe Readiness *corev1.Probe Startup *corev1.Probe } // MakeProbes returns the liveness and readiness probes // by default the health check runs `cat /tmp/.lock` every ten seconds func (f *FunctionFactory) MakeProbes(port int, httpProbePath string) (*FunctionProbes, error) { var handler corev1.ProbeHandler if f.Config.HTTPProbe { handler = corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ Path: httpProbePath, Port: intstr.IntOrString{ Type: intstr.Int, IntVal: int32(port), }, }, } } else { return nil, nil } probes := FunctionProbes{} probes.Readiness = &corev1.Probe{ ProbeHandler: handler, InitialDelaySeconds: f.Config.ReadinessProbe.InitialDelaySeconds, TimeoutSeconds: int32(f.Config.ReadinessProbe.TimeoutSeconds), PeriodSeconds: int32(f.Config.ReadinessProbe.PeriodSeconds), SuccessThreshold: 1, FailureThreshold: 3, } probes.Liveness = &corev1.Probe{ ProbeHandler: handler, InitialDelaySeconds: f.Config.LivenessProbe.InitialDelaySeconds, TimeoutSeconds: int32(f.Config.LivenessProbe.TimeoutSeconds), PeriodSeconds: int32(f.Config.LivenessProbe.PeriodSeconds), SuccessThreshold: 1, FailureThreshold: 3, } probes.Startup = &corev1.Probe{ ProbeHandler: handler, InitialDelaySeconds: f.Config.StartupProbe.InitialDelaySeconds, TimeoutSeconds: int32(f.Config.StartupProbe.TimeoutSeconds), PeriodSeconds: int32(f.Config.StartupProbe.PeriodSeconds), SuccessThreshold: 1, // Set failure threshold to 30 to allow for slow-starting inferences. FailureThreshold: 30, } return &probes, nil } ================================================ FILE: modelzetes/pkg/k8s/probes_test.go ================================================ // Copyright 2020 OpenFaaS Authors // Licensed under the MIT license. See LICENSE file in the project root for full license information. package k8s import ( "testing" ) func Test_makeProbes_useHTTPProbe(t *testing.T) { f := mockFactory() f.Config.HTTPProbe = true probes, err := f.MakeProbes(8080, "/") if err != nil { t.Fatal(err) } if probes.Readiness.HTTPGet == nil { t.Errorf("Readiness probe should have had HTTPGet handler") t.Fail() } if probes.Liveness.HTTPGet == nil { t.Errorf("Liveness probe should have had HTTPGet handler") t.Fail() } } func Test_makeProbes_useCustomDurationHTTPProbe(t *testing.T) { f := mockFactory() f.Config.HTTPProbe = true f.Config.LivenessProbe = &ProbeConfig{ PeriodSeconds: 1, TimeoutSeconds: 3, InitialDelaySeconds: 0, } f.Config.ReadinessProbe = &ProbeConfig{ PeriodSeconds: 1, TimeoutSeconds: 3, InitialDelaySeconds: 0, } f.Config.StartupProbe = &ProbeConfig{ PeriodSeconds: 1, TimeoutSeconds: 3, InitialDelaySeconds: 0, } customDelay := "0" probes, err := f.MakeProbes(8080, "/") if err != nil { t.Fatal(err) } if probes.Readiness.HTTPGet == nil { t.Errorf("Readiness probe should have had HTTPGet handler") t.Fail() } if probes.Readiness.InitialDelaySeconds != 0 { t.Errorf("Readiness probe should have initial delay seconds set to %s", customDelay) t.Fail() } if probes.Liveness.HTTPGet == nil { t.Errorf("Liveness probe should have had HTTPGet handler") t.Fail() } if probes.Liveness.InitialDelaySeconds != 0 { t.Errorf("Readiness probe should have had HTTPGet handler set to %s", customDelay) t.Fail() } } ================================================ FILE: modelzetes/pkg/k8s/proxy.go ================================================ // Copyright (c) Alex Ellis 2017. All rights reserved. // Copyright 2020 OpenFaaS Author(s) // Licensed under the MIT license. See LICENSE file in the project root for full license information. package k8s import ( "fmt" "math/rand" "net/url" "strings" "sync" "github.com/tensorchord/openmodelz/modelzetes/pkg/consts" corelister "k8s.io/client-go/listers/core/v1" ) // watchdogPort for the OpenFaaS function watchdog const watchdogPort = 8080 func NewFunctionLookup(ns string, lister corelister.EndpointsLister) *FunctionLookup { return &FunctionLookup{ DefaultNamespace: ns, EndpointLister: lister, Listers: map[string]corelister.EndpointsNamespaceLister{}, lock: sync.RWMutex{}, } } type FunctionLookup struct { DefaultNamespace string EndpointLister corelister.EndpointsLister Listers map[string]corelister.EndpointsNamespaceLister lock sync.RWMutex } func (f *FunctionLookup) GetLister(ns string) corelister.EndpointsNamespaceLister { f.lock.RLock() defer f.lock.RUnlock() return f.Listers[ns] } func (f *FunctionLookup) SetLister(ns string, lister corelister.EndpointsNamespaceLister) { f.lock.Lock() defer f.lock.Unlock() f.Listers[ns] = lister } func getNamespace(name, defaultNamespace string) string { namespace := defaultNamespace if strings.Contains(name, ".") { namespace = name[strings.LastIndexAny(name, ".")+1:] } return namespace } func (l *FunctionLookup) Resolve(name string) (url.URL, error) { functionName := name namespace := getNamespace(name, l.DefaultNamespace) if err := l.verifyNamespace(namespace); err != nil { return url.URL{}, err } if strings.Contains(name, ".") { functionName = strings.TrimSuffix(name, "."+namespace) } nsEndpointLister := l.GetLister(namespace) if nsEndpointLister == nil { l.SetLister(namespace, l.EndpointLister.Endpoints(namespace)) nsEndpointLister = l.GetLister(namespace) } svcName := consts.DefaultServicePrefix + functionName svc, err := nsEndpointLister.Get(svcName) if err != nil { return url.URL{}, fmt.Errorf("error listing \"%s.%s\": %s", svcName, namespace, err.Error()) } if len(svc.Subsets) == 0 { return url.URL{}, fmt.Errorf("no subsets available for \"%s.%s\"", svcName, namespace) } all := len(svc.Subsets[0].Addresses) if len(svc.Subsets[0].Addresses) == 0 { return url.URL{}, fmt.Errorf("no addresses in subset for \"%s.%s\"", svcName, namespace) } target := rand.Intn(all) serviceIP := svc.Subsets[0].Addresses[target].IP servicePort := svc.Subsets[0].Ports[target].Port urlStr := fmt.Sprintf("http://%s:%d", serviceIP, servicePort) urlRes, err := url.Parse(urlStr) if err != nil { return url.URL{}, err } return *urlRes, nil } func (l *FunctionLookup) verifyNamespace(name string) error { if name != "kube-system" { return nil } // ToDo use global namespace parse and validation return fmt.Errorf("namespace not allowed") } ================================================ FILE: modelzetes/pkg/k8s/proxy_test.go ================================================ package k8s import ( "fmt" "strings" "testing" corelister "k8s.io/client-go/listers/core/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/labels" ) type FakeLister struct { } func (f FakeLister) List(selector labels.Selector) (ret []*corev1.Endpoints, err error) { return nil, nil } func (f FakeLister) Endpoints(namespace string) corelister.EndpointsNamespaceLister { return FakeNSLister{} } type FakeNSLister struct { } func (f FakeNSLister) List(selector labels.Selector) (ret []*corev1.Endpoints, err error) { return nil, nil } func (f FakeNSLister) Get(name string) (*corev1.Endpoints, error) { // make sure that we only send the function name to the lister if strings.Contains(name, ".") { return nil, fmt.Errorf("can not look up function name with a dot!") } ep := corev1.Endpoints{ Subsets: []corev1.EndpointSubset{{ Addresses: []corev1.EndpointAddress{{IP: "127.0.0.1"}}, Ports: []corev1.EndpointPort{{Port: 8080}}, }}, } return &ep, nil } func Test_FunctionLookup(t *testing.T) { lister := FakeLister{} resolver := NewFunctionLookup("testDefault", lister) cases := []struct { name string funcName string expError string expUrl string }{ { name: "function without namespace uses default namespace", funcName: "testfunc", expUrl: "http://127.0.0.1:8080", }, { name: "function with namespace uses the given namespace", funcName: "testfunc.othernamespace", expUrl: "http://127.0.0.1:8080", }, { name: "url parse errors are returned", funcName: "testfunc.kube-system", expError: "namespace not allowed", }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { url, err := resolver.Resolve(tc.funcName) if tc.expError == "" && err != nil { t.Fatalf("expected no error, got %s", err) } if tc.expError != "" && (err == nil || !strings.Contains(err.Error(), tc.expError)) { t.Fatalf("expected %s, got %s", tc.expError, err) } if url.String() != tc.expUrl { t.Fatalf("expected url %s, got %s", tc.expUrl, url.String()) } }) } } ================================================ FILE: modelzetes/pkg/k8s/secrets.go ================================================ // Copyright 2020 OpenFaaS Authors // Licensed under the MIT license. See LICENSE file in the project root for full license information. package k8s import ( "context" "fmt" "log" "sort" "strings" "github.com/pkg/errors" types "github.com/tensorchord/openmodelz/agent/api/types" "github.com/tensorchord/openmodelz/modelzetes/pkg/apis/modelzetes/v2alpha1" appsv1 "k8s.io/api/apps/v1" apiv1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" typedV1 "k8s.io/client-go/kubernetes/typed/core/v1" ) const ( secretsMountPath = "/var/modelz/secrets" secretLabel = "app.kubernetes.io/managed-by" secretLabelValue = "modelz" secretsProjectVolumeNameTmpl = "projected-secrets" ) // SecretsClient exposes the standardized CRUD behaviors for Kubernetes secrets. These methods // will ensure that the secrets are structured and labelled correctly for use by the modelz system. type SecretsClient interface { // List returns a list of available function secrets. Only the names are returned // to ensure we do not accidentally read or print the sensitive values during // read operations. List(namespace string) (names []string, err error) // Create adds a new secret, with the appropriate labels and structure to be // used as a function secret. Create(secret types.Secret) error // Replace updates the value of a function secret Replace(secret types.Secret) error // Delete removes a function secret Delete(name string, namespace string) error // GetSecrets queries Kubernetes for a list of secrets by name in the given k8s namespace. // This should only be used if you need access to the actual secret structure/value. Specifically, // inside the FunctionFactory. GetSecrets(namespace string, secretNames []string) (map[string]*apiv1.Secret, error) } // SecretInterfacer exposes the SecretInterface getter for the k8s client. // This is implemented by the CoreV1Interface() interface in the Kubernetes client. // The SecretsClient only needs this one interface, but needs to be able to set the // namespaces when the interface is instantiated, meaning, we need the Getter and not the // SecretInterface itself. type SecretInterfacer interface { // Secrets returns a SecretInterface scoped to the specified namespace Secrets(namespace string) typedV1.SecretInterface } type secretClient struct { kube SecretInterfacer } // NewSecretsClient constructs a new SecretsClient using the provided Kubernetes client. func NewSecretsClient(kube kubernetes.Interface) SecretsClient { return &secretClient{ kube: kube.CoreV1(), } } func (c secretClient) List(namespace string) (names []string, err error) { res, err := c.kube.Secrets(namespace).List(context.TODO(), c.selector()) if err != nil { log.Printf("failed to list secrets in %s: %v\n", namespace, err) return nil, err } names = make([]string, len(res.Items)) for idx, item := range res.Items { // this is safe because size of names matches res.Items exactly names[idx] = item.Name } return names, nil } func (c secretClient) Create(secret types.Secret) error { err := c.validateSecret(secret) if err != nil { return err } req := &apiv1.Secret{ Type: apiv1.SecretTypeOpaque, ObjectMeta: metav1.ObjectMeta{ Name: secret.Name, Namespace: secret.Namespace, Labels: map[string]string{ secretLabel: secretLabelValue, }, }, } req.Data = c.getValidSecretData(secret) _, err = c.kube.Secrets(secret.Namespace).Create(context.TODO(), req, metav1.CreateOptions{}) if err != nil { log.Printf("failed to create secret %s.%s: %v\n", secret.Name, secret.Namespace, err) return err } log.Printf("created secret %s.%s\n", secret.Name, secret.Namespace) return nil } func (c secretClient) Replace(secret types.Secret) error { err := c.validateSecret(secret) if err != nil { return err } kube := c.kube.Secrets(secret.Namespace) found, err := kube.Get(context.TODO(), secret.Name, metav1.GetOptions{}) if err != nil { log.Printf("can not retrieve secret for update %s.%s: %v\n", secret.Name, secret.Namespace, err) return err } found.Data = c.getValidSecretData(secret) _, err = kube.Update(context.TODO(), found, metav1.UpdateOptions{}) if err != nil { log.Printf("can not update secret %s.%s: %v\n", secret.Name, secret.Namespace, err) return err } return nil } func (c secretClient) Delete(namespace string, name string) error { err := c.kube.Secrets(namespace).Delete(context.TODO(), name, metav1.DeleteOptions{}) if err != nil { log.Printf("can not delete %s.%s: %v\n", name, namespace, err) } return err } func (c secretClient) GetSecrets(namespace string, secretNames []string) (map[string]*apiv1.Secret, error) { kube := c.kube.Secrets(namespace) opts := metav1.GetOptions{} secrets := map[string]*apiv1.Secret{} for _, secretName := range secretNames { secret, err := kube.Get(context.TODO(), secretName, opts) if err != nil { return nil, err } secrets[secretName] = secret } return secrets, nil } func (c secretClient) selector() metav1.ListOptions { return metav1.ListOptions{ LabelSelector: fmt.Sprintf("%s=%s", secretLabel, secretLabelValue), } } func (c secretClient) validateSecret(secret types.Secret) error { if strings.TrimSpace(secret.Namespace) == "" { return errors.New("namespace may not be empty") } if strings.TrimSpace(secret.Name) == "" { return errors.New("name may not be empty") } return nil } func (c secretClient) getValidSecretData(secret types.Secret) map[string][]byte { if len(secret.RawValue) > 0 { return map[string][]byte{ secret.Name: secret.RawValue, } } return map[string][]byte{ secret.Name: []byte(secret.Value), } } // ConfigureSecrets will update the Deployment spec to include secrets that have been deployed // in the kubernetes cluster. For each requested secret, we inspect the type and add it to the // deployment spec as appropriate: secrets with type `SecretTypeDockercfg/SecretTypeDockerjson` // are added as ImagePullSecrets all other secrets are mounted as files in the deployments containers. func (f *FunctionFactory) ConfigureSecrets(request v2alpha1.Inference, deployment *appsv1.Deployment, existingSecrets map[string]*apiv1.Secret) error { // Add / reference pre-existing secrets within Kubernetes secretVolumeProjections := []apiv1.VolumeProjection{} for _, secretName := range request.Spec.Secrets { deployedSecret, ok := existingSecrets[secretName] if !ok { return fmt.Errorf("required secret '%s' was not found in the cluster", secretName) } switch deployedSecret.Type { case apiv1.SecretTypeDockercfg, apiv1.SecretTypeDockerConfigJson: deployment.Spec.Template.Spec.ImagePullSecrets = append( deployment.Spec.Template.Spec.ImagePullSecrets, apiv1.LocalObjectReference{ Name: secretName, }, ) default: projectedPaths := []apiv1.KeyToPath{} for secretKey := range deployedSecret.Data { projectedPaths = append(projectedPaths, apiv1.KeyToPath{Key: secretKey, Path: secretKey}) } projection := &apiv1.SecretProjection{Items: projectedPaths} projection.Name = secretName secretProjection := apiv1.VolumeProjection{ Secret: projection, } secretVolumeProjections = append(secretVolumeProjections, secretProjection) } } volumeName := secretsProjectVolumeNameTmpl projectedSecrets := apiv1.Volume{ Name: volumeName, VolumeSource: apiv1.VolumeSource{ Projected: &apiv1.ProjectedVolumeSource{ Sources: secretVolumeProjections, }, }, } // remove the existing secrets volume, if we can find it. The update volume will be // added below existingVolumes := removeVolume(volumeName, deployment.Spec.Template.Spec.Volumes) deployment.Spec.Template.Spec.Volumes = existingVolumes if len(secretVolumeProjections) > 0 { deployment.Spec.Template.Spec.Volumes = append(existingVolumes, projectedSecrets) } // add mount secret as a file updatedContainers := []apiv1.Container{} for _, container := range deployment.Spec.Template.Spec.Containers { mount := apiv1.VolumeMount{ Name: volumeName, ReadOnly: true, MountPath: secretsMountPath, } // remove the existing secrets volume mount, if we can find it. We update it later. container.VolumeMounts = removeVolumeMount(volumeName, container.VolumeMounts) if len(secretVolumeProjections) > 0 { container.VolumeMounts = append(container.VolumeMounts, mount) } updatedContainers = append(updatedContainers, container) } deployment.Spec.Template.Spec.Containers = updatedContainers return nil } // ReadFunctionSecretsSpec parses the name of the required function secrets. This is the inverse of ConfigureSecrets. func ReadFunctionSecretsSpec(item appsv1.Deployment) []string { secrets := []string{} for _, s := range item.Spec.Template.Spec.ImagePullSecrets { secrets = append(secrets, s.Name) } volumeName := secretsProjectVolumeNameTmpl var sourceSecrets []apiv1.VolumeProjection for _, v := range item.Spec.Template.Spec.Volumes { if v.Name == volumeName { sourceSecrets = v.Projected.Sources break } } for _, s := range sourceSecrets { if s.Secret == nil { continue } secrets = append(secrets, s.Secret.Name) } sort.Strings(secrets) return secrets } ================================================ FILE: modelzetes/pkg/k8s/secrets_factory_test.go ================================================ // Copyright 2020 OpenFaaS Author(s) // Licensed under the MIT license. See LICENSE file in the project root for full license information. package k8s import ( "fmt" "testing" "github.com/tensorchord/openmodelz/modelzetes/pkg/apis/modelzetes/v2alpha1" appsv1 "k8s.io/api/apps/v1" apiv1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func Test_ReadFunctionSecretsSpec(t *testing.T) { f := mockFactory() existingSecrets := map[string]*apiv1.Secret{ "pullsecret": {Type: apiv1.SecretTypeDockercfg}, "testsecret": {Type: apiv1.SecretTypeOpaque, Data: map[string][]byte{"filename": []byte("contents")}}, } functionDep := appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{Name: "testfunc"}, Spec: appsv1.DeploymentSpec{ Template: apiv1.PodTemplateSpec{ Spec: apiv1.PodSpec{ Containers: []apiv1.Container{ {Name: "testfunc", Image: "alpine:latest"}, }, }, }, }, } cases := []struct { name string req v2alpha1.Inference deployment appsv1.Deployment expected []string }{ { name: "empty secrets, returns empty slice", req: v2alpha1.Inference{ Spec: v2alpha1.InferenceSpec{ Secrets: []string{}, }, }, deployment: functionDep, expected: []string{}, }, { name: "detects and extracts image pull secret", req: v2alpha1.Inference{ Spec: v2alpha1.InferenceSpec{ Secrets: []string{"pullsecret"}, }, }, deployment: functionDep, expected: []string{"pullsecret"}, }, { name: "detects and extracts projected generic secret", req: v2alpha1.Inference{ Spec: v2alpha1.InferenceSpec{ Secrets: []string{"testsecret"}, }, }, deployment: functionDep, expected: []string{"testsecret"}, }, { name: "detects and extracts both pull secrets and projected generic secret, result is sorted", req: v2alpha1.Inference{ Spec: v2alpha1.InferenceSpec{ Secrets: []string{"pullsecret", "testsecret"}, }, }, deployment: functionDep, expected: []string{"pullsecret", "testsecret"}, }, } for _, tc := range cases { err := f.ConfigureSecrets(tc.req, &tc.deployment, existingSecrets) if err != nil { t.Fatalf("unexpected error result: got %q", err) } parsedSecrets := ReadFunctionSecretsSpec(tc.deployment) if len(tc.expected) != len(parsedSecrets) { t.Fatalf("incorrect secret count, expected: %v, got: %v", tc.expected, parsedSecrets) } for idx, expected := range tc.expected { value := parsedSecrets[idx] if expected != value { t.Fatalf("incorrect secret in idx %d, expected: %q, got: %q", idx, expected, value) } } } } func Test_FunctionFactory_ConfigureSecrets(t *testing.T) { f := mockFactory() existingSecrets := map[string]*apiv1.Secret{ "pullsecret": {Type: apiv1.SecretTypeDockercfg}, "testsecret": {Type: apiv1.SecretTypeOpaque, Data: map[string][]byte{"filename": []byte("contents")}}, } basicDeployment := appsv1.Deployment{ Spec: appsv1.DeploymentSpec{ Template: apiv1.PodTemplateSpec{ Spec: apiv1.PodSpec{ Containers: []apiv1.Container{ {Name: "testfunc", Image: "alpine:latest"}, }, }, }, }, } volumeName := "projected-secrets" withExistingSecret := appsv1.Deployment{ Spec: appsv1.DeploymentSpec{ Template: apiv1.PodTemplateSpec{ Spec: apiv1.PodSpec{ Containers: []apiv1.Container{ { Name: "testfunc", Image: "alpine:latest", VolumeMounts: []apiv1.VolumeMount{ { Name: volumeName, }, { Name: volumeName, }, }, }, }, Volumes: []apiv1.Volume{ { Name: volumeName, }, { Name: volumeName, }, }, }, }, }, } cases := []struct { name string req v2alpha1.Inference deployment appsv1.Deployment validator func(t *testing.T, deployment *appsv1.Deployment) err error }{ { name: "does not add volume if request secrets is nil", req: v2alpha1.Inference{ Spec: v2alpha1.InferenceSpec{}, }, deployment: basicDeployment, validator: validateEmptySecretVolumesAndMounts, }, { name: "does not add volume if request secrets is nil", req: v2alpha1.Inference{ Spec: v2alpha1.InferenceSpec{ Secrets: []string{}, }, }, deployment: basicDeployment, validator: validateEmptySecretVolumesAndMounts, }, { name: "removes all copies of exiting secrets volumes", req: v2alpha1.Inference{ Spec: v2alpha1.InferenceSpec{ Secrets: []string{}, }, }, deployment: withExistingSecret, validator: validateEmptySecretVolumesAndMounts, }, { name: "add new secret volume", req: v2alpha1.Inference{ Spec: v2alpha1.InferenceSpec{ Secrets: []string{"pullsecret", "testsecret"}, }, }, deployment: basicDeployment, validator: validateNewSecretVolumesAndMounts, }, { name: "replaces previous secret mount with new mount", req: v2alpha1.Inference{ Spec: v2alpha1.InferenceSpec{ Secrets: []string{"pullsecret", "testsecret"}, }, }, deployment: withExistingSecret, validator: validateNewSecretVolumesAndMounts, }, { name: "removes secrets volume if request secrets is empty or nil", req: v2alpha1.Inference{ Spec: v2alpha1.InferenceSpec{ Secrets: []string{}, }, }, deployment: withExistingSecret, validator: validateEmptySecretVolumesAndMounts, }, } for _, tc := range cases { err := f.ConfigureSecrets(tc.req, &tc.deployment, existingSecrets) if err != tc.err { t.Errorf("unexpected error result: got %v, expected %v", err, tc.err) } tc.validator(t, &tc.deployment) } } func validateEmptySecretVolumesAndMounts(t *testing.T, deployment *appsv1.Deployment) { numVolumes := len(deployment.Spec.Template.Spec.Volumes) if numVolumes != 0 { fmt.Printf("%+v", deployment.Spec.Template.Spec.Volumes) t.Errorf("Incorrect number of volumes: expected 0, got %d", numVolumes) } c := deployment.Spec.Template.Spec.Containers[0] numVolumeMounts := len(c.VolumeMounts) if numVolumeMounts != 0 { t.Errorf("Incorrect number of volumes mounts: expected 0, got %d", numVolumeMounts) } } func validateNewSecretVolumesAndMounts(t *testing.T, deployment *appsv1.Deployment) { numVolumes := len(deployment.Spec.Template.Spec.Volumes) if numVolumes != 1 { t.Errorf("Incorrect number of volumes: expected 1, got %d", numVolumes) } volume := deployment.Spec.Template.Spec.Volumes[0] if volume.Name != "projected-secrets" { t.Errorf("Incorrect volume name: expected \"projected-secrets\", got \"%s\"", volume.Name) } if volume.VolumeSource.Projected == nil { t.Error("Secrets volume is not a projected volume type") } if volume.VolumeSource.Projected.Sources[0].Secret.Items[0].Key != "filename" { t.Error("Project secret not constructed correctly") } c := deployment.Spec.Template.Spec.Containers[0] numVolumeMounts := len(c.VolumeMounts) if numVolumeMounts != 1 { t.Errorf("Incorrect number of volumes mounts: expected 1, got %d", numVolumeMounts) } mount := c.VolumeMounts[0] if mount.Name != "projected-secrets" { t.Errorf("Incorrect volume mounts: expected \"projected-secrets\", got \"%s\"", mount.Name) } if mount.MountPath != secretsMountPath { t.Errorf("Incorrect volume mount path: expected \"%s\", got \"%s\"", secretsMountPath, mount.MountPath) } } ================================================ FILE: modelzetes/pkg/k8s/securityContext.go ================================================ // Copyright 2020 OpenFaaS Authors // Licensed under the MIT license. See LICENSE file in the project root for full license information. package k8s import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" ) // nonRootFunctionuserID is the user id that is set when DeployHandlerConfig.SetNonRootUser is true. // value >10000 per the suggestion from https://kubesec.io/basics/containers-securitycontext-runasuser/ const SecurityContextUserID = int64(12000) // ConfigureContainerUserID sets the UID to 12000 for the function Container. Defaults to user // specified in image metadata if `SetNonRootUser` is `false`. Root == 0. func (f *FunctionFactory) ConfigureContainerUserID(deployment *appsv1.Deployment) { userID := SecurityContextUserID var functionUser *int64 if f.Config.SetNonRootUser { functionUser = &userID } if deployment.Spec.Template.Spec.Containers[0].SecurityContext == nil { deployment.Spec.Template.Spec.Containers[0].SecurityContext = &corev1.SecurityContext{} } deployment.Spec.Template.Spec.Containers[0].SecurityContext.RunAsUser = functionUser } // ConfigureReadOnlyRootFilesystem will create or update the required settings and mounts to ensure // that the ReadOnlyRootFilesystem setting works as expected, meaning: // 1. when ReadOnlyRootFilesystem is true, the security context of the container will have ReadOnlyRootFilesystem also // marked as true and a new `/tmp` folder mount will be added to the deployment spec // 2. when ReadOnlyRootFilesystem is false, the security context of the container will also have ReadOnlyRootFilesystem set // to false and there will be no mount for the `/tmp` folder // // This method is safe for both create and update operations. func (f *FunctionFactory) ConfigureReadOnlyRootFilesystem(deployment *appsv1.Deployment) { readonly := false if deployment.Spec.Template.Spec.Containers[0].SecurityContext != nil { deployment.Spec.Template.Spec.Containers[0].SecurityContext.ReadOnlyRootFilesystem = &readonly } else { deployment.Spec.Template.Spec.Containers[0].SecurityContext = &corev1.SecurityContext{ ReadOnlyRootFilesystem: &readonly, } } existingVolumes := removeVolume("temp", deployment.Spec.Template.Spec.Volumes) deployment.Spec.Template.Spec.Volumes = existingVolumes existingMounts := removeVolumeMount("temp", deployment.Spec.Template.Spec.Containers[0].VolumeMounts) deployment.Spec.Template.Spec.Containers[0].VolumeMounts = existingMounts } ================================================ FILE: modelzetes/pkg/k8s/securityContext_test.go ================================================ // Copyright 2020 OpenFaaS Authors // Licensed under the MIT license. See LICENSE file in the project root for full license information. package k8s import ( "testing" appsv1 "k8s.io/api/apps/v1" apiv1 "k8s.io/api/core/v1" ) func readOnlyRootDisabled(t *testing.T, deployment *appsv1.Deployment) { if len(deployment.Spec.Template.Spec.Volumes) != 0 { t.Error("Volumes should be empty if ReadOnlyRootFilesystem is false") } if len(deployment.Spec.Template.Spec.Containers[0].VolumeMounts) != 0 { t.Error("VolumeMounts should be empty if ReadOnlyRootFilesystem is false") } functionContatiner := deployment.Spec.Template.Spec.Containers[0] if functionContatiner.SecurityContext != nil { if *functionContatiner.SecurityContext.ReadOnlyRootFilesystem != false { t.Error("ReadOnlyRootFilesystem should be false on the container SecurityContext") } } } func Test_configureReadOnlyRootFilesystem_Disabled_To_Disabled(t *testing.T) { f := mockFactory() deployment := &appsv1.Deployment{ Spec: appsv1.DeploymentSpec{ Template: apiv1.PodTemplateSpec{ Spec: apiv1.PodSpec{ Containers: []apiv1.Container{ {Name: "testfunc", Image: "alpine:latest"}, }, }, }, }, } f.ConfigureReadOnlyRootFilesystem(deployment) readOnlyRootDisabled(t, deployment) } ================================================ FILE: modelzetes/pkg/k8s/utils.go ================================================ // Copyright 2020 OpenFaaS Authors // Licensed under the MIT license. See LICENSE file in the project root for full license information. package k8s import ( corev1 "k8s.io/api/core/v1" ) // removeVolume returns a Volume slice with any volumes matching volumeName removed. // Uses the filter without allocation technique // https://github.com/golang/go/wiki/SliceTricks#filtering-without-allocating func removeVolume(volumeName string, volumes []corev1.Volume) []corev1.Volume { if volumes == nil { return []corev1.Volume{} } newVolumes := volumes[:0] for _, v := range volumes { if v.Name != volumeName { newVolumes = append(newVolumes, v) } } return newVolumes } // removeVolumeMount returns a VolumeMount slice with any mounts matching volumeName removed // Uses the filter without allocation technique // https://github.com/golang/go/wiki/SliceTricks#filtering-without-allocating func removeVolumeMount(volumeName string, mounts []corev1.VolumeMount) []corev1.VolumeMount { if mounts == nil { return []corev1.VolumeMount{} } newMounts := mounts[:0] for _, v := range mounts { if v.Name != volumeName { newMounts = append(newMounts, v) } } return newMounts } ================================================ FILE: modelzetes/pkg/pointer/ptr.go ================================================ package util func Ptr[T any](v T) *T { return &v } func PtrCopy[T any](v T) *T { n := new(T) *n = v return n } ================================================ FILE: modelzetes/pkg/signals/signal.go ================================================ package signals import ( "os" "os/signal" ) var onlyOneSignalHandler = make(chan struct{}) // SetupSignalHandler registered for SIGTERM and SIGINT. A stop channel is returned // which is closed on one of these signals. If a second signal is caught, the program // is terminated with exit code 1. func SetupSignalHandler() (stopCh <-chan struct{}) { close(onlyOneSignalHandler) // panics when called twice stop := make(chan struct{}) c := make(chan os.Signal, 2) signal.Notify(c, shutdownSignals...) go func() { <-c close(stop) <-c os.Exit(1) // second signal. Exit directly. }() return stop } ================================================ FILE: modelzetes/pkg/signals/signal_posix.go ================================================ //go:build !windows // +build !windows package signals import ( "os" "syscall" ) var shutdownSignals = []os.Signal{os.Interrupt, syscall.SIGTERM} ================================================ FILE: modelzetes/pkg/signals/signal_windows.go ================================================ package signals import ( "os" ) var shutdownSignals = []os.Signal{os.Interrupt} ================================================ FILE: modelzetes/pkg/version/version.go ================================================ /* Copyright The TensorChord Inc. Copyright The BuildKit Authors. Copyright The containerd Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package version import ( "fmt" "regexp" "runtime" "strings" "sync" ) var ( // Package is filled at linking time Package = "github.com/tensorchord/openmodelz/modelzetes" // Revision is filled with the VCS (e.g. git) revision being used to build // the program at linking time. Revision = "" version = "0.0.0+unknown" buildDate = "1970-01-01T00:00:00Z" // output from `date -u +'%Y-%m-%dT%H:%M:%SZ'` gitCommit = "" // output from `git rev-parse HEAD` gitTag = "" // output from `git describe --exact-match --tags HEAD` (if clean tree state) gitTreeState = "" // determined from `git status --porcelain`. either 'clean' or 'dirty' developmentFlag = "false" ) // Version contains envd version information type Version struct { Version string BuildDate string GitCommit string GitTag string GitTreeState string GoVersion string Compiler string Platform string } func (v Version) String() string { return v.Version } // SetGitTagForE2ETest sets the gitTag for test purpose. func SetGitTagForE2ETest(tag string) { gitTag = tag } // GetEnvdVersion gets Envd version information func GetEnvdVersion() string { var versionStr string if gitCommit != "" && gitTag != "" && gitTreeState == "clean" && developmentFlag == "false" { // if we have a clean tree state and the current commit is tagged, // this is an official release. versionStr = gitTag } else { // otherwise formulate a version string based on as much metadata // information we have available. if strings.HasPrefix(version, "v") { versionStr = version } else { versionStr = "v" + version } if len(gitCommit) >= 7 { versionStr += "+" + gitCommit[0:7] if gitTreeState != "clean" { versionStr += ".dirty" } } else { versionStr += "+unknown" } } return versionStr } // GetVersion returns the version information func GetVersion() Version { return Version{ Version: GetEnvdVersion(), BuildDate: buildDate, GitCommit: gitCommit, GitTag: gitTag, GitTreeState: gitTreeState, GoVersion: runtime.Version(), Compiler: runtime.Compiler, Platform: fmt.Sprintf("%s/%s", runtime.GOOS, runtime.GOARCH), } } var ( reRelease *regexp.Regexp reDev *regexp.Regexp reOnce sync.Once ) func UserAgent() string { version := GetVersion().String() reOnce.Do(func() { reRelease = regexp.MustCompile(`^(v[0-9]+\.[0-9]+)\.[0-9]+$`) reDev = regexp.MustCompile(`^(v[0-9]+\.[0-9]+)\.[0-9]+`) }) if matches := reRelease.FindAllStringSubmatch(version, 1); len(matches) > 0 { version = matches[0][1] } else if matches := reDev.FindAllStringSubmatch(version, 1); len(matches) > 0 { version = matches[0][1] + "-dev" } return "envd/" + version } ================================================ FILE: modelzetes/vendor.go ================================================ //go:build vendor package main // This file exists to trick "go mod vendor" to include "main" packages. // It is not expected to build, the build tag above is only to prevent this // file from being included in builds. import ( _ "k8s.io/code-generator/cmd/client-gen" _ "k8s.io/code-generator/cmd/deepcopy-gen" _ "k8s.io/code-generator/cmd/defaulter-gen" _ "k8s.io/code-generator/cmd/informer-gen" _ "k8s.io/code-generator/cmd/lister-gen" _ "k8s.io/code-generator/cmd/openapi-gen" ) func main() {} ================================================ FILE: pyproject.toml ================================================ [project] name = "openmodelz" description = "Simplify machine learning deployment for any environments." readme = "README.md" authors = [ {name = "TensorChord", email = "modelz@tensorchord.ai"}, ] license = {text = "Apache-2.0"} keywords = ["machine learning", "deep learning", "model serving"] dynamic = ["version"] requires-python = ">=2.7" classifiers = [ "Environment :: GPU", "Intended Audience :: Developers", "License :: OSI Approved :: Apache Software License", "Programming Language :: Python :: 2", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Topic :: Scientific/Engineering :: Artificial Intelligence", "Topic :: Software Development :: Libraries :: Python Modules", "Topic :: Software Development :: Build Tools", ] [project.urls] homepage = "https://modelz.ai/" documentation = "https://docs.open.modelz.ai/" repository = "https://github.com/tensorchord/openmodelz" changelog = "https://github.com/tensorchord/openmodelz/releases" [tool.cibuildwheel] build-frontend = "build" archs = ["auto64"] skip = "pp*" # skip pypy before-all = "" environment = { PIP_NO_CLEAN="yes" } before-build = "ls -la mdz/bin" # help to debug [project.optional-dependencies] [project.scripts] [build-system] requires = ["setuptools>=45", "wheel", "setuptools_scm"] build-backend = "setuptools.build_meta" [tool.setuptools_scm] write_to = "mdz/_version.py" ================================================ FILE: setup.py ================================================ import os import subprocess import shlex from wheel.bdist_wheel import bdist_wheel from setuptools import setup, find_packages, Extension from setuptools.command.build_ext import build_ext from setuptools_scm import get_version with open("README.md", "r", encoding="utf-8") as f: readme = f.read() class bdist_wheel_universal(bdist_wheel): def get_tag(self): *_, plat = super().get_tag() return "py2.py3", "none", plat def build_if_not_exist(): if os.path.isfile("mdz/bin/mdz"): return version = get_version() print(f"build mdz from source ({version})") errno = subprocess.call(shlex.split( f"make build-release GIT_TAG={version}" ), cwd="mdz") assert errno == 0, f"mdz build failed with code {errno}" class ModelzExtension(Extension): """A custom extension to define the OpenModelz extension.""" class ModelzBuildExt(build_ext): def build_extension(self, ext: Extension) -> None: if not isinstance(ext, ModelzExtension): return super().build_extension(ext) build_if_not_exist() setup( name="openmodelz", use_scm_version=True, description="Simplify machine learning deployment for any environments.", long_description=readme, long_description_content_type="text/markdown", url="https://github.com/tensorchord/openmodelz", license="Apache License 2.0", author="TensorChord", author_email="modelz@tensorchord.ai", packages=find_packages("mdz"), include_package_data=True, data_files=[("bin", ["mdz/bin/mdz"])], zip_safe=False, ext_modules=[ ModelzExtension(name="mdz", sources=["mdz/*"]), ], cmdclass=dict( build_ext=ModelzBuildExt, bdist_wheel=bdist_wheel_universal, ), ) ================================================ FILE: typos.toml ================================================ [files] extend-exclude = ["CHANGELOG.md", "go.mod", "go.sum"] [default.extend-words] requestor = "requestor" ba = "ba"