Repository: whywaita/myshoes Branch: master Commit: d20faef23bff Files: 93 Total size: 350.2 KB Directory structure: gitextract_98bm4mpl/ ├── .github/ │ └── workflows/ │ ├── build-docker-sha.yaml │ ├── release.yaml │ └── test.yaml ├── .gitignore ├── .goreleaser.yml ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── api/ │ ├── myshoes/ │ │ ├── README.md │ │ ├── client.go │ │ ├── http.go │ │ └── target.go │ ├── proto/ │ │ └── myshoes.proto │ └── proto.go/ │ ├── myshoes.pb.go │ └── myshoes_grpc.pb.go ├── cmd/ │ ├── server/ │ │ └── cmd.go │ └── shoes-tester/ │ └── main.go ├── docs/ │ ├── 01_01_for_admin_setup.md │ ├── 01_02_for_admin_tips.md │ ├── 02_01_for_user_setup.md │ ├── 03_how-to-develop-shoes.md │ └── assets/ │ └── myshoes.service ├── go.mod ├── go.sum ├── internal/ │ ├── testutils/ │ │ ├── mysql.go │ │ ├── testutils.go │ │ └── web.go │ └── util/ │ └── util.go └── pkg/ ├── config/ │ ├── config.go │ └── init.go ├── datastore/ │ ├── github.go │ ├── interface.go │ ├── memory/ │ │ └── memory.go │ ├── mysql/ │ │ ├── job.go │ │ ├── job_test.go │ │ ├── lock.go │ │ ├── mysql.go │ │ ├── mysql_test.go │ │ ├── runner.go │ │ ├── runner_test.go │ │ ├── schema.sql │ │ ├── target.go │ │ └── target_test.go │ └── resource_type.go ├── docker/ │ └── ratelimit.go ├── gh/ │ ├── github.go │ ├── github_test.go │ ├── installation.go │ ├── jwt.go │ ├── jwt_test.go │ ├── label.go │ ├── metrics.go │ ├── metrics_test.go │ ├── ratelimit.go │ ├── runner.go │ ├── scope.go │ ├── token_registration.go │ ├── webhook.go │ ├── workflow_job.go │ └── workflow_run.go ├── logger/ │ └── logger.go ├── metric/ │ ├── collector.go │ ├── scrape_datastore.go │ ├── scrape_github.go │ ├── scrape_memory.go │ └── webhook.go ├── runner/ │ ├── metrics.go │ ├── runner.go │ ├── runner_delete.go │ ├── runner_delete_ephemeral.go │ ├── runner_delete_once.go │ ├── token_update.go │ └── util.go ├── shoes/ │ └── shoes.go ├── starter/ │ ├── README.md │ ├── error.go │ ├── metric.go │ ├── metrics.go │ ├── safety/ │ │ ├── README.md │ │ ├── safety.go │ │ └── unlimited/ │ │ └── unlimited.go │ ├── scripts/ │ │ └── RunnerService.js │ ├── scripts.go │ └── starter.go └── web/ ├── config.go ├── http.go ├── http_test.go ├── metrics.go ├── target.go ├── target_create.go ├── target_test.go └── webhook.go ================================================ FILE CONTENTS ================================================ ================================================ FILE: .github/workflows/build-docker-sha.yaml ================================================ name: Build Docker image (sha) on: push: branches: - "**" workflow_dispatch: jobs: docker-build-sha: runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3.7.0 - uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 - name: Cache Docker layers uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: /tmp/.buildx-cache key: ${{ runner.os }}-buildx-${{ github.sha }} restore-keys: | ${{ runner.os }}-buildx- - name: Login to GitHub Container Registry uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5.10.0 id: meta with: images: ghcr.io/${{ github.repository_owner }}/myshoes tags: | type=sha - name: Build container image uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 with: push: true tags: ${{ steps.meta.outputs.tags }} ================================================ FILE: .github/workflows/release.yaml ================================================ name: release on: push: tags: - "v[0-9]+.[0-9]+.[0-9]+" jobs: goreleaser: runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: fetch-depth: 0 - name: Setup Go uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # v6.1.0 with: go-version-file: 'go.mod' - name: Run GoReleaser uses: goreleaser/goreleaser-action@e435ccd777264be153ace6237001ef4d979d3a7a # v6.4.0 with: version: latest args: release --clean env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} docker: runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3.7.0 - uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 - name: Login to GitHub Container Registry uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5.10.0 id: meta with: images: ghcr.io/whywaita/myshoes tags: | type=raw,value=latest type=semver,pattern={{raw}} type=sha - name: Build container image uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 with: push: true tags: ${{ steps.meta.outputs.tags }} ================================================ FILE: .github/workflows/test.yaml ================================================ name: test on: push: branches: - "**" pull_request: workflow_dispatch: jobs: test: runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: os: - ubuntu-latest steps: - name: checkout uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: fetch-depth: 1 - name: setup go uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # v6.1.0 with: go-version-file: 'go.mod' - name: lint run: | go install honnef.co/go/tools/cmd/staticcheck@latest staticcheck ./... - name: vet run: | go vet ./... - name: test run: | make test docker-build-test: runs-on: ubuntu-latest steps: - name: checkout uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: fetch-depth: 1 - name: Build container image uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 with: push: false tags: ${{ steps.meta.outputs.tags }} ================================================ FILE: .gitignore ================================================ # Created by https://www.toptal.com/developers/gitignore/api/macos,intellij,go # Edit at https://www.toptal.com/developers/gitignore?templates=macos,intellij,go ### Go ### # Binaries for programs and plugins *.exe *.exe~ *.dll *.so *.dylib # Test binary, built with `go test -c` *.test # Output of the go coverage tool, specifically when used with LiteIDE *.out # Dependency directories (remove the comment below to include it) # vendor/ ### Go Patch ### /vendor/ /Godeps/ ### Intellij ### # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 # User-specific stuff .idea/**/workspace.xml .idea/**/tasks.xml .idea/**/usage.statistics.xml .idea/**/dictionaries .idea/**/shelf # Generated files .idea/**/contentModel.xml # Sensitive or high-churn files .idea/**/dataSources/ .idea/**/dataSources.ids .idea/**/dataSources.local.xml .idea/**/sqlDataSources.xml .idea/**/dynamic.xml .idea/**/uiDesigner.xml .idea/**/dbnavigator.xml # Gradle .idea/**/gradle.xml .idea/**/libraries # Gradle and Maven with auto-import # When using Gradle or Maven with auto-import, you should exclude module files, # since they will be recreated, and may cause churn. Uncomment if using # auto-import. # .idea/artifacts # .idea/compiler.xml # .idea/jarRepositories.xml # .idea/modules.xml # .idea/*.iml # .idea/modules # *.iml # *.ipr # CMake cmake-build-*/ # Mongo Explorer plugin .idea/**/mongoSettings.xml # File-based project format *.iws # IntelliJ out/ # mpeltonen/sbt-idea plugin .idea_modules/ # JIRA plugin atlassian-ide-plugin.xml # Cursive Clojure plugin .idea/replstate.xml # Crashlytics plugin (for Android Studio and IntelliJ) com_crashlytics_export_strings.xml crashlytics.properties crashlytics-build.properties fabric.properties # Editor-based Rest Client .idea/httpRequests # Android studio 3.1+ serialized cache file .idea/caches/build_file_checksums.ser ### Intellij Patch ### # Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721 # *.iml # modules.xml # .idea/misc.xml # *.ipr # Sonarlint plugin # https://plugins.jetbrains.com/plugin/7973-sonarlint .idea/**/sonarlint/ # SonarQube Plugin # https://plugins.jetbrains.com/plugin/7238-sonarqube-community-plugin .idea/**/sonarIssues.xml # Markdown Navigator plugin # https://plugins.jetbrains.com/plugin/7896-markdown-navigator-enhanced .idea/**/markdown-navigator.xml .idea/**/markdown-navigator-enh.xml .idea/**/markdown-navigator/ # Cache file creation bug # See https://youtrack.jetbrains.com/issue/JBR-2257 .idea/$CACHE_FILE$ # CodeStream plugin # https://plugins.jetbrains.com/plugin/12206-codestream .idea/codestream.xml ### macOS ### # General .DS_Store .AppleDouble .LSOverride # Icon must end with two \r Icon # Thumbnails ._* # Files that might appear in the root of a volume .DocumentRevisions-V100 .fseventsd .Spotlight-V100 .TemporaryItems .Trashes .VolumeIcon.icns .com.apple.timemachine.donotpresent # Directories potentially created on remote AFP share .AppleDB .AppleDesktop Network Trash Folder Temporary Items .apdisk # End of https://www.toptal.com/developers/gitignore/api/macos,intellij,go /myshoes* ================================================ FILE: .goreleaser.yml ================================================ builds: - main: ./cmd/server/cmd.go goos: - linux goarch: - amd64 - arm64 ================================================ FILE: Dockerfile ================================================ FROM golang:1.25 AS builder WORKDIR /go/src/github.com/whywaita/myshoes RUN go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.34.2 RUN go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.5.1 RUN apt-get update -y \ && apt-get install -y protobuf-compiler ENV CGO_ENABLED=0 ENV GOOS=linux ENV GOARCH=amd64 COPY . . RUN make build-linux FROM alpine RUN apk update \ && apk update RUN apk add --no-cache ca-certificates \ && update-ca-certificates 2>/dev/null || true COPY --from=builder /go/src/github.com/whywaita/myshoes/myshoes-linux-amd64 /app CMD ["/app"] ================================================ FILE: LICENSE ================================================ MIT License Copyright (c) 2021 whywaita Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: Makefile ================================================ .PHONY: help .DEFAULT_GOAL := help CURRENT_REVISION = $(shell git rev-parse --short HEAD) BUILD_LDFLAGS = "-X main.revision=$(CURRENT_REVISION)" help: @grep -E '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' build: ## Build All go generate ./... make build-proto go build -o myshoes -ldflags $(BUILD_LDFLAGS) cmd/server/cmd.go build-linux: ## Build for Linux go generate ./... make build-proto GOOS=linux GOARCH=amd64 go build -o myshoes-linux-amd64 -ldflags $(BUILD_LDFLAGS) cmd/server/cmd.go build-proto: ## Build proto file mkdir -p tmp/proto-go rm -rf api/proto.go protoc -I=api/proto/ --go_out=tmp/proto-go/ --go-grpc_out=tmp/proto-go/ api/proto/**.proto mv tmp/proto-go/github.com/whywaita/myshoes/api/proto.go api/ rm -rf tmp test: ## Exec test go test -v ./... ================================================ FILE: README.md ================================================ # myshoes: Auto scaling self-hosted runner for GitHub Actions ![](./docs/assets/img/myshoes_logo_yoko_colorA.png) [![awesome-runners](https://img.shields.io/badge/listed%20on-awesome--runners-blue.svg)](https://github.com/jonico/awesome-runners) [![Go Reference](https://pkg.go.dev/badge/github.com/whywaita/myshoes.svg)](https://pkg.go.dev/github.com/whywaita/myshoes) [![test](https://github.com/whywaita/myshoes/actions/workflows/test.yaml/badge.svg)](https://github.com/whywaita/myshoes/actions/workflows/test.yaml) [![MIT License](http://img.shields.io/badge/license-MIT-blue.svg?style=flat)](LICENSE) [![Go Report Card](https://goreportcard.com/badge/github.com/whywaita/myshoes)](https://goreportcard.com/report/github.com/whywaita/myshoes) Auto scaling self-hosted runner :runner: (like GitHub-hosted) for GitHub Actions! ## Features - Auto-scaling and runner with your cloud-provider - your infrastructure (private cloud, homelab...) - [LXD](https://linuxcontainers.org): [shoes-lxd](https://github.com/whywaita/myshoes-providers/tree/master/shoes-lxd) - [OpenStack](https://www.openstack.org): [shoes-openstack](https://github.com/whywaita/myshoes-providers/tree/master/shoes-openstack) - a low-cost instance in public cloud - [AWS EC2 Spot Instances](https://aws.amazon.com/ec2/spot): [shoes-aws](https://github.com/whywaita/myshoes-providers/tree/master/shoes-aws) - [GCP Preemptible VM instances](https://cloud.google.com/compute/docs/instances/preemptible): shoes-gcp (not yet) - using special hardware - Graphics Processing Unit (GPU) - Field Programmable Gate Array (FPGA) - And more in [whywaita/myshoes-providers](https://github.com/whywaita/myshoes-providers) ## Setup (only once) Please see [Documents](./docs). ## How to contribute 1. Fork it 1. Clone original repository `git clone https://github.com/whywaita/myshoes` 1. Add remote your repository `git remote add your-name https://github.com/${your-name}/myshoes` 1. Create your feature branch `git switch -c my-new-feature` 1. Commit your changes `git commit -am 'Add some feature'` 1. Push to the branch `git push your-name my-new-feature` 1. Create new Pull Request ## Publications ### Talk - [Development myshoes and Provide Cycloud-hosted runner -- GitHub Actions with your shoes. (en)](https://www.slideshare.net/whywaita/development-myshoes-and-provide-cycloudhosted-runner-github-actions-with-your-shoes) - [Development OSS CI/CD platform in CyberAgent (ja)](https://www.slideshare.net/whywaita/cyberagent-oss-cicd-myshoes-cicd2021) ================================================ FILE: api/myshoes/README.md ================================================ # myshoes-sdk-go The Go SDK for myshoes ## Usage ```go package main import ( "context" "fmt" "io" "log" "net/http" "github.com/whywaita/myshoes/api/myshoes" ) func main() { // Set customized HTTP Client customHTTPClient := http.DefaultClient // Set customized logger customLogger := log.New(io.Discard, "", log.LstdFlags) client, err := myshoes.NewClient("https://example.com", customHTTPClient, customLogger) if err != nil { // ... } targets, err := client.ListTarget(context.Background()) if err != nil { // ... } fmt.Println(targets) } ``` ================================================ FILE: api/myshoes/client.go ================================================ package myshoes import ( "context" "fmt" "io" "log" "net/http" "net/url" "path" "strings" ) // Client is a client for myshoes type Client struct { HTTPClient http.Client URL *url.URL UserAgent string Logger *log.Logger } const ( defaultUserAgent = "myshoes-sdk-go" ) // NewClient create a Client func NewClient(endpoint string, client *http.Client, logger *log.Logger) (*Client, error) { u, err := url.Parse(endpoint) if err != nil { return nil, fmt.Errorf("failed to parse endpoint: %w", err) } httpClient := client if httpClient == nil { httpClient = http.DefaultClient } l := logger if l == nil { return &Client{ HTTPClient: *httpClient, URL: u, // Default is discard logger Logger: log.New(io.Discard, "", log.LstdFlags), }, nil } return &Client{ HTTPClient: *httpClient, URL: u, Logger: l, }, nil } func (c *Client) newRequest(ctx context.Context, method, spath string, body io.Reader) (*http.Request, error) { u := *c.URL u.Path = path.Join(c.URL.Path, spath) req, err := http.NewRequestWithContext(ctx, method, u.String(), body) if err != nil { return nil, fmt.Errorf("failed to create a new HTTP request: %w", err) } ua := c.UserAgent if strings.EqualFold(ua, "") { ua = defaultUserAgent } req.Header.Set("User-Agent", ua) req.Header.Set("Content-Type", "application/json") return req, nil } // Error values var ( errCreateRequest = "failed to create request: %w" errRequest = "failed to request: %w" errDecodeBody = "failed to decodeBody: %w" ) ================================================ FILE: api/myshoes/http.go ================================================ package myshoes import ( "encoding/json" "fmt" "io" "net/http" "github.com/whywaita/myshoes/pkg/web" ) func decodeBody(resp *http.Response, out interface{}) error { defer resp.Body.Close() body, err := io.ReadAll(resp.Body) if err != nil { return fmt.Errorf("failed to io.ReadAll(resp.Body): %w", err) } if err := json.Unmarshal(body, out); err != nil { return fmt.Errorf("failed to json.Unmarshal() (out: %s): %w", body, err) } return nil } func decodeErrorBody(resp *http.Response) error { var e web.ErrorResponse if err := decodeBody(resp, &e); err != nil { return fmt.Errorf(errDecodeBody, err) } return fmt.Errorf("%s", e.Error) } func (c *Client) request(req *http.Request, out interface{}) error { c.Logger.Printf("Do request: %+v", req) resp, err := c.HTTPClient.Do(req) if err != nil { return fmt.Errorf("failed to do HTTP request: %w", err) } switch { case resp.StatusCode == http.StatusNoContent: return nil case resp.StatusCode >= 400: return decodeErrorBody(resp) } if err := decodeBody(resp, out); err != nil { return fmt.Errorf(errDecodeBody, err) } return nil } ================================================ FILE: api/myshoes/target.go ================================================ package myshoes import ( "bytes" "context" "encoding/json" "fmt" "net/http" "github.com/whywaita/myshoes/pkg/web" ) // CreateTarget create a target func (c *Client) CreateTarget(ctx context.Context, param web.TargetCreateParam) (*web.UserTarget, error) { spath := "/target" jb, err := json.Marshal(param) if err != nil { return nil, fmt.Errorf("failed to json.Marshal: %w", err) } req, err := c.newRequest(ctx, http.MethodPost, spath, bytes.NewBuffer(jb)) if err != nil { return nil, fmt.Errorf(errCreateRequest, err) } var target web.UserTarget if err := c.request(req, &target); err != nil { return nil, fmt.Errorf(errRequest, err) } return &target, nil } // GetTarget get a target func (c *Client) GetTarget(ctx context.Context, targetID string) (*web.UserTarget, error) { spath := fmt.Sprintf("/target/%s", targetID) req, err := c.newRequest(ctx, http.MethodGet, spath, nil) if err != nil { return nil, fmt.Errorf(errCreateRequest, err) } var target web.UserTarget if err := c.request(req, &target); err != nil { return nil, fmt.Errorf(errRequest, err) } return &target, nil } // UpdateTarget update a target func (c *Client) UpdateTarget(ctx context.Context, targetID string, param web.TargetCreateParam) (*web.UserTarget, error) { spath := fmt.Sprintf("/target/%s", targetID) jb, err := json.Marshal(param) if err != nil { return nil, fmt.Errorf("failed to json.Marshal: %w", err) } req, err := c.newRequest(ctx, http.MethodPost, spath, bytes.NewBuffer(jb)) if err != nil { return nil, fmt.Errorf(errCreateRequest, err) } var target web.UserTarget if err := c.request(req, &target); err != nil { return nil, fmt.Errorf(errRequest, err) } return &target, nil } // DeleteTarget delete a target func (c *Client) DeleteTarget(ctx context.Context, targetID string) error { spath := fmt.Sprintf("/target/%s", targetID) req, err := c.newRequest(ctx, http.MethodDelete, spath, nil) if err != nil { return fmt.Errorf(errCreateRequest, err) } var i interface{} // this endpoint return N/A if err := c.request(req, &i); err != nil { return fmt.Errorf(errRequest, err) } return nil } // ListTarget get a list of target func (c *Client) ListTarget(ctx context.Context) ([]web.UserTarget, error) { spath := "/target" req, err := c.newRequest(ctx, http.MethodGet, spath, nil) if err != nil { return nil, fmt.Errorf(errCreateRequest, err) } var targets []web.UserTarget if err := c.request(req, &targets); err != nil { return nil, fmt.Errorf(errRequest, err) } return targets, nil } ================================================ FILE: api/proto/myshoes.proto ================================================ syntax = "proto3"; package whywaita.myshoes; option go_package = "github.com/whywaita/myshoes/api/proto.go"; service Shoes { rpc AddInstance(AddInstanceRequest) returns (AddInstanceResponse) {} rpc DeleteInstance(DeleteInstanceRequest) returns (DeleteInstanceResponse) {} } enum ResourceType { Unknown = 0; Nano = 1; Micro = 2; Small = 3; Medium = 4; Large = 5; XLarge = 6; XLarge2 = 7; XLarge3 = 8; XLarge4 = 9; } message AddInstanceRequest { string runner_name = 1; string setup_script = 2; ResourceType resource_type = 3; repeated string labels = 4; } message AddInstanceResponse { string cloud_id = 1; string shoes_type = 2; string ip_address = 3; ResourceType resource_type = 4; } message DeleteInstanceRequest { string cloud_id = 1; repeated string labels = 2; } message DeleteInstanceResponse {} ================================================ FILE: api/proto.go/myshoes.pb.go ================================================ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.36.11 // protoc v5.29.3 // source: myshoes.proto package proto_go import ( protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" sync "sync" unsafe "unsafe" ) const ( // Verify that this generated code is sufficiently up-to-date. _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) // Verify that runtime/protoimpl is sufficiently up-to-date. _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) type ResourceType int32 const ( ResourceType_Unknown ResourceType = 0 ResourceType_Nano ResourceType = 1 ResourceType_Micro ResourceType = 2 ResourceType_Small ResourceType = 3 ResourceType_Medium ResourceType = 4 ResourceType_Large ResourceType = 5 ResourceType_XLarge ResourceType = 6 ResourceType_XLarge2 ResourceType = 7 ResourceType_XLarge3 ResourceType = 8 ResourceType_XLarge4 ResourceType = 9 ) // Enum value maps for ResourceType. var ( ResourceType_name = map[int32]string{ 0: "Unknown", 1: "Nano", 2: "Micro", 3: "Small", 4: "Medium", 5: "Large", 6: "XLarge", 7: "XLarge2", 8: "XLarge3", 9: "XLarge4", } ResourceType_value = map[string]int32{ "Unknown": 0, "Nano": 1, "Micro": 2, "Small": 3, "Medium": 4, "Large": 5, "XLarge": 6, "XLarge2": 7, "XLarge3": 8, "XLarge4": 9, } ) func (x ResourceType) Enum() *ResourceType { p := new(ResourceType) *p = x return p } func (x ResourceType) String() string { return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) } func (ResourceType) Descriptor() protoreflect.EnumDescriptor { return file_myshoes_proto_enumTypes[0].Descriptor() } func (ResourceType) Type() protoreflect.EnumType { return &file_myshoes_proto_enumTypes[0] } func (x ResourceType) Number() protoreflect.EnumNumber { return protoreflect.EnumNumber(x) } // Deprecated: Use ResourceType.Descriptor instead. func (ResourceType) EnumDescriptor() ([]byte, []int) { return file_myshoes_proto_rawDescGZIP(), []int{0} } type AddInstanceRequest struct { state protoimpl.MessageState `protogen:"open.v1"` RunnerName string `protobuf:"bytes,1,opt,name=runner_name,json=runnerName,proto3" json:"runner_name,omitempty"` SetupScript string `protobuf:"bytes,2,opt,name=setup_script,json=setupScript,proto3" json:"setup_script,omitempty"` ResourceType ResourceType `protobuf:"varint,3,opt,name=resource_type,json=resourceType,proto3,enum=whywaita.myshoes.ResourceType" json:"resource_type,omitempty"` Labels []string `protobuf:"bytes,4,rep,name=labels,proto3" json:"labels,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } func (x *AddInstanceRequest) Reset() { *x = AddInstanceRequest{} mi := &file_myshoes_proto_msgTypes[0] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } func (x *AddInstanceRequest) String() string { return protoimpl.X.MessageStringOf(x) } func (*AddInstanceRequest) ProtoMessage() {} func (x *AddInstanceRequest) ProtoReflect() protoreflect.Message { mi := &file_myshoes_proto_msgTypes[0] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use AddInstanceRequest.ProtoReflect.Descriptor instead. func (*AddInstanceRequest) Descriptor() ([]byte, []int) { return file_myshoes_proto_rawDescGZIP(), []int{0} } func (x *AddInstanceRequest) GetRunnerName() string { if x != nil { return x.RunnerName } return "" } func (x *AddInstanceRequest) GetSetupScript() string { if x != nil { return x.SetupScript } return "" } func (x *AddInstanceRequest) GetResourceType() ResourceType { if x != nil { return x.ResourceType } return ResourceType_Unknown } func (x *AddInstanceRequest) GetLabels() []string { if x != nil { return x.Labels } return nil } type AddInstanceResponse struct { state protoimpl.MessageState `protogen:"open.v1"` CloudId string `protobuf:"bytes,1,opt,name=cloud_id,json=cloudId,proto3" json:"cloud_id,omitempty"` ShoesType string `protobuf:"bytes,2,opt,name=shoes_type,json=shoesType,proto3" json:"shoes_type,omitempty"` IpAddress string `protobuf:"bytes,3,opt,name=ip_address,json=ipAddress,proto3" json:"ip_address,omitempty"` ResourceType ResourceType `protobuf:"varint,4,opt,name=resource_type,json=resourceType,proto3,enum=whywaita.myshoes.ResourceType" json:"resource_type,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } func (x *AddInstanceResponse) Reset() { *x = AddInstanceResponse{} mi := &file_myshoes_proto_msgTypes[1] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } func (x *AddInstanceResponse) String() string { return protoimpl.X.MessageStringOf(x) } func (*AddInstanceResponse) ProtoMessage() {} func (x *AddInstanceResponse) ProtoReflect() protoreflect.Message { mi := &file_myshoes_proto_msgTypes[1] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use AddInstanceResponse.ProtoReflect.Descriptor instead. func (*AddInstanceResponse) Descriptor() ([]byte, []int) { return file_myshoes_proto_rawDescGZIP(), []int{1} } func (x *AddInstanceResponse) GetCloudId() string { if x != nil { return x.CloudId } return "" } func (x *AddInstanceResponse) GetShoesType() string { if x != nil { return x.ShoesType } return "" } func (x *AddInstanceResponse) GetIpAddress() string { if x != nil { return x.IpAddress } return "" } func (x *AddInstanceResponse) GetResourceType() ResourceType { if x != nil { return x.ResourceType } return ResourceType_Unknown } type DeleteInstanceRequest struct { state protoimpl.MessageState `protogen:"open.v1"` CloudId string `protobuf:"bytes,1,opt,name=cloud_id,json=cloudId,proto3" json:"cloud_id,omitempty"` Labels []string `protobuf:"bytes,2,rep,name=labels,proto3" json:"labels,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } func (x *DeleteInstanceRequest) Reset() { *x = DeleteInstanceRequest{} mi := &file_myshoes_proto_msgTypes[2] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } func (x *DeleteInstanceRequest) String() string { return protoimpl.X.MessageStringOf(x) } func (*DeleteInstanceRequest) ProtoMessage() {} func (x *DeleteInstanceRequest) ProtoReflect() protoreflect.Message { mi := &file_myshoes_proto_msgTypes[2] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use DeleteInstanceRequest.ProtoReflect.Descriptor instead. func (*DeleteInstanceRequest) Descriptor() ([]byte, []int) { return file_myshoes_proto_rawDescGZIP(), []int{2} } func (x *DeleteInstanceRequest) GetCloudId() string { if x != nil { return x.CloudId } return "" } func (x *DeleteInstanceRequest) GetLabels() []string { if x != nil { return x.Labels } return nil } type DeleteInstanceResponse struct { state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } func (x *DeleteInstanceResponse) Reset() { *x = DeleteInstanceResponse{} mi := &file_myshoes_proto_msgTypes[3] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } func (x *DeleteInstanceResponse) String() string { return protoimpl.X.MessageStringOf(x) } func (*DeleteInstanceResponse) ProtoMessage() {} func (x *DeleteInstanceResponse) ProtoReflect() protoreflect.Message { mi := &file_myshoes_proto_msgTypes[3] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use DeleteInstanceResponse.ProtoReflect.Descriptor instead. func (*DeleteInstanceResponse) Descriptor() ([]byte, []int) { return file_myshoes_proto_rawDescGZIP(), []int{3} } var File_myshoes_proto protoreflect.FileDescriptor const file_myshoes_proto_rawDesc = "" + "\n" + "\rmyshoes.proto\x12\x10whywaita.myshoes\"\xb5\x01\n" + "\x12AddInstanceRequest\x12\x1f\n" + "\vrunner_name\x18\x01 \x01(\tR\n" + "runnerName\x12!\n" + "\fsetup_script\x18\x02 \x01(\tR\vsetupScript\x12C\n" + "\rresource_type\x18\x03 \x01(\x0e2\x1e.whywaita.myshoes.ResourceTypeR\fresourceType\x12\x16\n" + "\x06labels\x18\x04 \x03(\tR\x06labels\"\xb3\x01\n" + "\x13AddInstanceResponse\x12\x19\n" + "\bcloud_id\x18\x01 \x01(\tR\acloudId\x12\x1d\n" + "\n" + "shoes_type\x18\x02 \x01(\tR\tshoesType\x12\x1d\n" + "\n" + "ip_address\x18\x03 \x01(\tR\tipAddress\x12C\n" + "\rresource_type\x18\x04 \x01(\x0e2\x1e.whywaita.myshoes.ResourceTypeR\fresourceType\"J\n" + "\x15DeleteInstanceRequest\x12\x19\n" + "\bcloud_id\x18\x01 \x01(\tR\acloudId\x12\x16\n" + "\x06labels\x18\x02 \x03(\tR\x06labels\"\x18\n" + "\x16DeleteInstanceResponse*\x85\x01\n" + "\fResourceType\x12\v\n" + "\aUnknown\x10\x00\x12\b\n" + "\x04Nano\x10\x01\x12\t\n" + "\x05Micro\x10\x02\x12\t\n" + "\x05Small\x10\x03\x12\n" + "\n" + "\x06Medium\x10\x04\x12\t\n" + "\x05Large\x10\x05\x12\n" + "\n" + "\x06XLarge\x10\x06\x12\v\n" + "\aXLarge2\x10\a\x12\v\n" + "\aXLarge3\x10\b\x12\v\n" + "\aXLarge4\x10\t2\xcc\x01\n" + "\x05Shoes\x12\\\n" + "\vAddInstance\x12$.whywaita.myshoes.AddInstanceRequest\x1a%.whywaita.myshoes.AddInstanceResponse\"\x00\x12e\n" + "\x0eDeleteInstance\x12'.whywaita.myshoes.DeleteInstanceRequest\x1a(.whywaita.myshoes.DeleteInstanceResponse\"\x00B*Z(github.com/whywaita/myshoes/api/proto.gob\x06proto3" var ( file_myshoes_proto_rawDescOnce sync.Once file_myshoes_proto_rawDescData []byte ) func file_myshoes_proto_rawDescGZIP() []byte { file_myshoes_proto_rawDescOnce.Do(func() { file_myshoes_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_myshoes_proto_rawDesc), len(file_myshoes_proto_rawDesc))) }) return file_myshoes_proto_rawDescData } var file_myshoes_proto_enumTypes = make([]protoimpl.EnumInfo, 1) var file_myshoes_proto_msgTypes = make([]protoimpl.MessageInfo, 4) var file_myshoes_proto_goTypes = []any{ (ResourceType)(0), // 0: whywaita.myshoes.ResourceType (*AddInstanceRequest)(nil), // 1: whywaita.myshoes.AddInstanceRequest (*AddInstanceResponse)(nil), // 2: whywaita.myshoes.AddInstanceResponse (*DeleteInstanceRequest)(nil), // 3: whywaita.myshoes.DeleteInstanceRequest (*DeleteInstanceResponse)(nil), // 4: whywaita.myshoes.DeleteInstanceResponse } var file_myshoes_proto_depIdxs = []int32{ 0, // 0: whywaita.myshoes.AddInstanceRequest.resource_type:type_name -> whywaita.myshoes.ResourceType 0, // 1: whywaita.myshoes.AddInstanceResponse.resource_type:type_name -> whywaita.myshoes.ResourceType 1, // 2: whywaita.myshoes.Shoes.AddInstance:input_type -> whywaita.myshoes.AddInstanceRequest 3, // 3: whywaita.myshoes.Shoes.DeleteInstance:input_type -> whywaita.myshoes.DeleteInstanceRequest 2, // 4: whywaita.myshoes.Shoes.AddInstance:output_type -> whywaita.myshoes.AddInstanceResponse 4, // 5: whywaita.myshoes.Shoes.DeleteInstance:output_type -> whywaita.myshoes.DeleteInstanceResponse 4, // [4:6] is the sub-list for method output_type 2, // [2:4] is the sub-list for method input_type 2, // [2:2] is the sub-list for extension type_name 2, // [2:2] is the sub-list for extension extendee 0, // [0:2] is the sub-list for field type_name } func init() { file_myshoes_proto_init() } func file_myshoes_proto_init() { if File_myshoes_proto != nil { return } type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_myshoes_proto_rawDesc), len(file_myshoes_proto_rawDesc)), NumEnums: 1, NumMessages: 4, NumExtensions: 0, NumServices: 1, }, GoTypes: file_myshoes_proto_goTypes, DependencyIndexes: file_myshoes_proto_depIdxs, EnumInfos: file_myshoes_proto_enumTypes, MessageInfos: file_myshoes_proto_msgTypes, }.Build() File_myshoes_proto = out.File file_myshoes_proto_goTypes = nil file_myshoes_proto_depIdxs = nil } ================================================ FILE: api/proto.go/myshoes_grpc.pb.go ================================================ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.6.0 // - protoc v5.29.3 // source: myshoes.proto package proto_go import ( context "context" grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" status "google.golang.org/grpc/status" ) // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. // Requires gRPC-Go v1.64.0 or later. const _ = grpc.SupportPackageIsVersion9 const ( Shoes_AddInstance_FullMethodName = "/whywaita.myshoes.Shoes/AddInstance" Shoes_DeleteInstance_FullMethodName = "/whywaita.myshoes.Shoes/DeleteInstance" ) // ShoesClient is the client API for Shoes service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. type ShoesClient interface { AddInstance(ctx context.Context, in *AddInstanceRequest, opts ...grpc.CallOption) (*AddInstanceResponse, error) DeleteInstance(ctx context.Context, in *DeleteInstanceRequest, opts ...grpc.CallOption) (*DeleteInstanceResponse, error) } type shoesClient struct { cc grpc.ClientConnInterface } func NewShoesClient(cc grpc.ClientConnInterface) ShoesClient { return &shoesClient{cc} } func (c *shoesClient) AddInstance(ctx context.Context, in *AddInstanceRequest, opts ...grpc.CallOption) (*AddInstanceResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(AddInstanceResponse) err := c.cc.Invoke(ctx, Shoes_AddInstance_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } return out, nil } func (c *shoesClient) DeleteInstance(ctx context.Context, in *DeleteInstanceRequest, opts ...grpc.CallOption) (*DeleteInstanceResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(DeleteInstanceResponse) err := c.cc.Invoke(ctx, Shoes_DeleteInstance_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } return out, nil } // ShoesServer is the server API for Shoes service. // All implementations must embed UnimplementedShoesServer // for forward compatibility. type ShoesServer interface { AddInstance(context.Context, *AddInstanceRequest) (*AddInstanceResponse, error) DeleteInstance(context.Context, *DeleteInstanceRequest) (*DeleteInstanceResponse, error) mustEmbedUnimplementedShoesServer() } // UnimplementedShoesServer must be embedded to have // forward compatible implementations. // // NOTE: this should be embedded by value instead of pointer to avoid a nil // pointer dereference when methods are called. type UnimplementedShoesServer struct{} func (UnimplementedShoesServer) AddInstance(context.Context, *AddInstanceRequest) (*AddInstanceResponse, error) { return nil, status.Error(codes.Unimplemented, "method AddInstance not implemented") } func (UnimplementedShoesServer) DeleteInstance(context.Context, *DeleteInstanceRequest) (*DeleteInstanceResponse, error) { return nil, status.Error(codes.Unimplemented, "method DeleteInstance not implemented") } func (UnimplementedShoesServer) mustEmbedUnimplementedShoesServer() {} func (UnimplementedShoesServer) testEmbeddedByValue() {} // UnsafeShoesServer may be embedded to opt out of forward compatibility for this service. // Use of this interface is not recommended, as added methods to ShoesServer will // result in compilation errors. type UnsafeShoesServer interface { mustEmbedUnimplementedShoesServer() } func RegisterShoesServer(s grpc.ServiceRegistrar, srv ShoesServer) { // If the following call panics, it indicates UnimplementedShoesServer was // embedded by pointer and is nil. This will cause panics if an // unimplemented method is ever invoked, so we test this at initialization // time to prevent it from happening at runtime later due to I/O. if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { t.testEmbeddedByValue() } s.RegisterService(&Shoes_ServiceDesc, srv) } func _Shoes_AddInstance_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(AddInstanceRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(ShoesServer).AddInstance(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: Shoes_AddInstance_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ShoesServer).AddInstance(ctx, req.(*AddInstanceRequest)) } return interceptor(ctx, in, info, handler) } func _Shoes_DeleteInstance_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(DeleteInstanceRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(ShoesServer).DeleteInstance(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: Shoes_DeleteInstance_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ShoesServer).DeleteInstance(ctx, req.(*DeleteInstanceRequest)) } return interceptor(ctx, in, info, handler) } // Shoes_ServiceDesc is the grpc.ServiceDesc for Shoes service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) var Shoes_ServiceDesc = grpc.ServiceDesc{ ServiceName: "whywaita.myshoes.Shoes", HandlerType: (*ShoesServer)(nil), Methods: []grpc.MethodDesc{ { MethodName: "AddInstance", Handler: _Shoes_AddInstance_Handler, }, { MethodName: "DeleteInstance", Handler: _Shoes_DeleteInstance_Handler, }, }, Streams: []grpc.StreamDesc{}, Metadata: "myshoes.proto", } ================================================ FILE: cmd/server/cmd.go ================================================ package main import ( "context" "fmt" "log" "net/http" "runtime" "strings" "time" "github.com/whywaita/myshoes/pkg/config" "github.com/whywaita/myshoes/pkg/datastore" "github.com/whywaita/myshoes/pkg/datastore/mysql" "github.com/whywaita/myshoes/pkg/gh" "github.com/whywaita/myshoes/pkg/logger" "github.com/whywaita/myshoes/pkg/runner" "github.com/whywaita/myshoes/pkg/starter" "github.com/whywaita/myshoes/pkg/starter/safety/unlimited" "github.com/whywaita/myshoes/pkg/web" "golang.org/x/sync/errgroup" ) func init() { config.Load() mysqlURL := config.LoadMySQLURL() config.Config.MySQLDSN = mysqlURL if err := gh.InitializeCache(config.Config.GitHub.AppID, config.Config.GitHub.PEMByte); err != nil { log.Panicf("failed to create a cache: %+v", err) } } func main() { runtime.SetBlockProfileRate(1) runtime.SetMutexProfileFraction(1) go func() { log.Fatal(http.ListenAndServe("localhost:6060", nil)) }() myshoes, err := newShoes() if err != nil { log.Fatalln(err) } if err := myshoes.Run(); err != nil { log.Fatalln(err) } } type myShoes struct { ds datastore.Datastore start *starter.Starter run *runner.Manager } // newShoes create myshoes. func newShoes() (*myShoes, error) { notifyEnqueueCh := make(chan struct{}, 1) ds, err := mysql.New(config.Config.MySQLDSN, notifyEnqueueCh) if err != nil { return nil, fmt.Errorf("failed to mysql.New: %w", err) } unlimit := unlimited.Unlimited{} s := starter.New(ds, unlimit, config.Config.RunnerVersion, notifyEnqueueCh) manager := runner.New(ds, config.Config.RunnerVersion) return &myShoes{ ds: ds, start: s, run: manager, }, nil } // Run start services. func (m *myShoes) Run() error { eg, ctx := errgroup.WithContext(context.Background()) for { logger.Logf(false, "start getting lock...") isLocked, err := m.ds.IsLocked(ctx) if err != nil { return fmt.Errorf("failed to check lock: %w", err) } if strings.EqualFold(isLocked, datastore.IsNotLocked) { if err := m.ds.GetLock(ctx); err != nil { return fmt.Errorf("failed to get lock: %w", err) } logger.Logf(false, "get lock successfully!") break } time.Sleep(time.Second) } eg.Go(func() error { if err := web.Serve(ctx, m.ds); err != nil { logger.Logf(false, "failed to web.Serve: %+v", err) return fmt.Errorf("failed to serve: %w", err) } return nil }) eg.Go(func() error { if err := m.start.Loop(ctx); err != nil { logger.Logf(false, "failed to starter manager: %+v", err) return fmt.Errorf("failed to starter loop: %w", err) } return nil }) eg.Go(func() error { if err := m.run.Loop(ctx); err != nil { logger.Logf(false, "failed to runner manager: %+v", err) return fmt.Errorf("failed to runner loop: %w", err) } return nil }) if err := eg.Wait(); err != nil { return fmt.Errorf("failed to wait errgroup: %w", err) } return nil } ================================================ FILE: cmd/shoes-tester/main.go ================================================ package main import ( "context" "crypto/x509" "encoding/json" "encoding/pem" "flag" "fmt" "os" "os/exec" "strconv" "strings" "github.com/hashicorp/go-plugin" "github.com/whywaita/myshoes/pkg/config" "github.com/whywaita/myshoes/pkg/datastore" "github.com/whywaita/myshoes/pkg/gh" "github.com/whywaita/myshoes/pkg/shoes" "github.com/whywaita/myshoes/pkg/starter" ) func main() { if len(os.Args) < 2 { printUsage() os.Exit(1) } subcommand := os.Args[1] switch subcommand { case "add": if err := runAdd(os.Args[2:]); err != nil { fmt.Fprintf(os.Stderr, "Error: %v\n", err) os.Exit(1) } case "delete": if err := runDelete(os.Args[2:]); err != nil { fmt.Fprintf(os.Stderr, "Error: %v\n", err) os.Exit(1) } default: fmt.Fprintf(os.Stderr, "Unknown subcommand: %s\n", subcommand) printUsage() os.Exit(1) } } func printUsage() { fmt.Fprintf(os.Stderr, `Usage: shoes-tester [options] Commands: add Add an instance delete Delete an instance Run 'shoes-tester --help' for more information on a command. `) } type addFlags struct { pluginPath string runnerName string resourceType string labels string setupScript string generateScript bool scope string githubAppID string githubPrivateKeyPath string runnerVersion string runnerUser string runnerBaseDirectory string githubURL string jsonOutput bool } type deleteFlags struct { pluginPath string cloudID string labels string jsonOutput bool } func runAdd(args []string) error { fs := flag.NewFlagSet("add", flag.ExitOnError) flags := &addFlags{} fs.StringVar(&flags.pluginPath, "plugin", "", "Path to shoes-provider binary (required)") fs.StringVar(&flags.runnerName, "runner-name", "", "Runner name (required)") fs.StringVar(&flags.resourceType, "resource-type", "nano", "Resource type (nano|micro|small|medium|large|xlarge|2xlarge|3xlarge|4xlarge)") fs.StringVar(&flags.labels, "labels", "", "Comma-separated labels") fs.StringVar(&flags.setupScript, "setup-script", "", "Setup script (simple mode)") fs.BoolVar(&flags.generateScript, "generate-script", false, "Generate setup script (script generation mode)") fs.StringVar(&flags.scope, "scope", "", "Repository (owner/repo) or Organization (script generation mode)") fs.StringVar(&flags.githubAppID, "github-app-id", os.Getenv("GITHUB_APP_ID"), "GitHub App ID (script generation mode)") fs.StringVar(&flags.githubPrivateKeyPath, "github-private-key-path", os.Getenv("GITHUB_PRIVATE_KEY_PATH"), "GitHub App private key path (script generation mode)") fs.StringVar(&flags.runnerVersion, "runner-version", "latest", "Runner version (script generation mode)") fs.StringVar(&flags.runnerUser, "runner-user", "runner", "Runner user (script generation mode)") fs.StringVar(&flags.runnerBaseDirectory, "runner-base-directory", "/tmp", "Runner base directory (script generation mode)") fs.StringVar(&flags.githubURL, "github-url", "", "GitHub Enterprise Server URL (script generation mode)") fs.BoolVar(&flags.jsonOutput, "json", false, "Output in JSON format") fs.Parse(args) if flags.pluginPath == "" { return fmt.Errorf("--plugin is required") } if flags.runnerName == "" { return fmt.Errorf("--runner-name is required") } if flags.generateScript { if flags.scope == "" { return fmt.Errorf("--scope is required for script generation mode") } if flags.githubAppID == "" { return fmt.Errorf("--github-app-id is required for script generation mode") } if flags.githubPrivateKeyPath == "" { return fmt.Errorf("--github-private-key-path is required for script generation mode") } } ctx := context.Background() var setupScript string if flags.generateScript { script, err := generateSetupScript(ctx, flags) if err != nil { return fmt.Errorf("failed to generate setup script: %w", err) } setupScript = script } else { setupScript = flags.setupScript } resourceType := datastore.UnmarshalResourceTypeString(flags.resourceType) if resourceType == datastore.ResourceTypeUnknown && flags.resourceType != "unknown" { return fmt.Errorf("invalid resource type: %s", flags.resourceType) } labels := parseLabels(flags.labels) client, teardown, err := getClientWithPath(flags.pluginPath) if err != nil { return fmt.Errorf("failed to get plugin client: %w", err) } defer teardown() cloudID, ipAddress, shoesType, actualResourceType, err := client.AddInstance(ctx, flags.runnerName, setupScript, resourceType, labels) if err != nil { return fmt.Errorf("failed to add instance: %w", err) } if flags.jsonOutput { output := map[string]string{ "cloud_id": cloudID, "ip_address": ipAddress, "shoes_type": shoesType, "resource_type": actualResourceType.String(), } data, err := json.Marshal(output) if err != nil { return fmt.Errorf("failed to marshal JSON: %w", err) } fmt.Println(string(data)) } else { fmt.Printf("AddInstance succeeded:\n") fmt.Printf(" Cloud ID: %s\n", cloudID) fmt.Printf(" Shoes Type: %s\n", shoesType) fmt.Printf(" IP Address: %s\n", ipAddress) fmt.Printf(" Resource Type: %s\n", actualResourceType.String()) } return nil } func runDelete(args []string) error { fs := flag.NewFlagSet("delete", flag.ExitOnError) flags := &deleteFlags{} fs.StringVar(&flags.pluginPath, "plugin", "", "Path to shoes-provider binary (required)") fs.StringVar(&flags.cloudID, "cloud-id", "", "Cloud ID (required)") fs.StringVar(&flags.labels, "labels", "", "Comma-separated labels") fs.BoolVar(&flags.jsonOutput, "json", false, "Output in JSON format") fs.Parse(args) if flags.pluginPath == "" { return fmt.Errorf("--plugin is required") } if flags.cloudID == "" { return fmt.Errorf("--cloud-id is required") } ctx := context.Background() labels := parseLabels(flags.labels) client, teardown, err := getClientWithPath(flags.pluginPath) if err != nil { return fmt.Errorf("failed to get plugin client: %w", err) } defer teardown() if err := client.DeleteInstance(ctx, flags.cloudID, labels); err != nil { return fmt.Errorf("failed to delete instance: %w", err) } if flags.jsonOutput { output := map[string]string{ "status": "success", } data, err := json.Marshal(output) if err != nil { return fmt.Errorf("failed to marshal JSON: %w", err) } fmt.Println(string(data)) } else { fmt.Printf("DeleteInstance succeeded\n") } return nil } func getClientWithPath(pluginPath string) (shoes.Client, func(), error) { handshake := plugin.HandshakeConfig{ ProtocolVersion: 1, MagicCookieKey: "SHOES_PLUGIN_MAGIC_COOKIE", MagicCookieValue: "are_you_a_shoes?", } pluginMap := map[string]plugin.Plugin{ "shoes_grpc": &shoes.Plugin{}, } client := plugin.NewClient(&plugin.ClientConfig{ HandshakeConfig: handshake, Plugins: pluginMap, Cmd: exec.Command(pluginPath), Managed: true, Stderr: os.Stderr, SyncStdout: os.Stdout, SyncStderr: os.Stderr, AllowedProtocols: []plugin.Protocol{plugin.ProtocolGRPC}, }) rpcClient, err := client.Client() if err != nil { return nil, nil, fmt.Errorf("failed to get shoes client: %w", err) } raw, err := rpcClient.Dispense("shoes_grpc") if err != nil { return nil, nil, fmt.Errorf("failed to shoes client instance: %w", err) } return raw.(shoes.Client), client.Kill, nil } func generateSetupScript(ctx context.Context, flags *addFlags) (string, error) { keyBytes, err := os.ReadFile(flags.githubPrivateKeyPath) if err != nil { return "", fmt.Errorf("failed to read private key: %w", err) } appID, err := strconv.ParseInt(flags.githubAppID, 10, 64) if err != nil { return "", fmt.Errorf("failed to parse github-app-id: %w", err) } block, _ := pem.Decode(keyBytes) if block == nil { return "", fmt.Errorf("failed to decode PEM block from private key") } privateKey, err := x509.ParsePKCS1PrivateKey(block.Bytes) if err != nil { return "", fmt.Errorf("failed to parse private key: %w", err) } config.Config.GitHub.AppID = appID config.Config.GitHub.PEMByte = keyBytes config.Config.GitHub.PEM = privateKey if flags.githubURL == "" { config.Config.GitHubURL = "https://github.com" } else { config.Config.GitHubURL = flags.githubURL } config.Config.RunnerUser = flags.runnerUser config.Config.RunnerBaseDirectory = flags.runnerBaseDirectory if err := gh.InitializeCache(appID, keyBytes); err != nil { return "", fmt.Errorf("failed to initialize GitHub client cache: %w", err) } s := starter.New(nil, nil, flags.runnerVersion, nil) return s.GetSetupScript(ctx, flags.scope, flags.runnerName) } func parseLabels(labels string) []string { if labels == "" { return []string{} } parts := strings.Split(labels, ",") result := make([]string, 0, len(parts)) for _, p := range parts { trimmed := strings.TrimSpace(p) if trimmed != "" { result = append(result, trimmed) } } return result } ================================================ FILE: docs/01_01_for_admin_setup.md ================================================ # Setup myshoes daemon ## Goal - Start myshoes daemon ## Prepare - The network connectivity to myshoes server. - The webhook endpoint from github.com **OR** your GitHub Enterprise Server (`POST /github/events`). - REST API from your workspace (`GET, POST, DELETE /target`). - You decide platform for runner and shoes-provider - The official shoes-provider topic is [myshoes-provider](https://github.com/search?q=topic%3Amyshoes-provider). - You can implement and use your private shoes-provider. Please check [how-to-develop-shoes.md](./03_how-to-develop-shoes.md). ## Word definition - `your_shoes_host`: The endpoint of serving myshoes. - e.g.) `https://myshoes.example.com` ## Setup Please prepare a few things first. ### Machine image for runner - Virtual Machine Image on your cloud provider. - installed a some commands. - required: curl (1) - optional: jq (1), docker (1) - optional, but **STRONG RECOMMEND INSTALLING BEFORE** (please read known issue) - put latest runner tar.gz to `/usr/local/etc` [optional] - optional, but **STRONG RECOMMEND INSTALLING BEFORE** (please read known issue) For example is [here](https://github.com/whywaita/myshoes-providers/tree/master/shoes-lxd/images). (packer file) ### Create GitHub Apps #### Configure values - GitHub App Name: any text - Homepage URL: any text ##### Webhook - Webhook URL: `${your_shoes_host}/github/events` - Webhook secret: any text ##### Repository permissions - Actions: Read-only - Administration: Read & write - Checks: Read-only ##### Organization permissions - Self-hosted runners: Read & write ##### Subscribe to events - Check `Workflow job` ### Download private key - download from GitHub or upload private key from your machine. ### Running ```bash $ make build $ ./myshoes ``` A config variables can set from environment values. - `PORT` - default: 8080 - Listen port for myshoes. - GitHub Apps information - required - `GITHUB_APP_ID` - `GITHUB_APP_SECRET` (if you set `Webhook secret` for your GitHub App) - `GITHUB_PRIVATE_KEY_BASE64` - base64 encoded private key from GitHub Apps - `$ cat privatekey.pem | base64 -w 0` - `MYSQL_URL` - required - DataSource Name, ex) `username:password@tcp(localhost:3306)/myshoes` - if `MYSQL_USER`, `MYSQL_PASSWORD`, `MYSQL_HOST`, `MYSQL_PORT`, `MYSQL_DATABASE` all are set, this env value are ignored. - `MYSQL_USER`, `MYSQL_PASSWORD`, `MYSQL_HOST`, `MYSQL_PORT`, `MYSQL_DATABASE` - optional - If all environment variables are set, mysql_url is constructed and loaded in the following way, then `MYSQL_URL` env will be ignored. - example) `${MYSQL_USER}:${MYSQL_PASSWORD}@tcp(${MYSQL_HOST}:${MYSQL_PORT})/${MYSQL_DATABASE}` - `PLUGIN` - required - set path of myshoes-provider binary. - example) `./shoes-mock` `https://example.com/shoes-mock` `https://github.com/whywaita/myshoes-providers/releases/download/v0.1.0/shoes-lxd-linux-amd64` - `PLUGIN_OUTPUT` - default: `.` - set path of directory that contains myshoes-provider binary. - `GITHUB_URL` - default: `https://github.com` - The URL of GitHub Enterprise Server. - Please contain schema. - `RUNNER_VERSION` - default: `latest` - Use the latest version in starting job - The version of `actions/runner` - example) `v2.302.1`, `latest` - `RUNNER_USER` - default: `runner` - set linux username that executes runner. you need to set exist user. - DO NOT set root. It can't run GitHub Actions runner in root permission. - Example: `ubuntu` - `PROVIDE_DOCKER_HUB_METRICS` - default: `false` - set `true` if you want to provide rate-limit metrics for Docker Hub. - If you're not anonymous user, you need to set `DOCKER_HUB_USERNAME` and `DOCKER_HUB_PASSWORD`. - `DOCKER_HUB_USERNAME` - default: `` (empty) - set Docker Hub username for pulling Docker image. (Use for provide rate-limit metrics) - `DOCKER_HUB_PASSWORD` - default: `` (empty) - set Docker Hub password for pulling Docker image. (Use for provide rate-limit metrics) For tuning values - `DEBUG` - default: false - show debugging log - `STRICT` - default: true - set strict mode - `MODE_WEBHOOK_TYPE` - default: `workflow_job` (use receive `workflow_job` event) - Set type of webhook from GitHub - option: `check_run` - `MAX_CONNECTIONS_TO_BACKEND` - default: 50 - The number of max connections to shoes-provider - `MAX_CONCURRENCY_DELETING` - default: 1 - The number of max concurrency of deleting and more some env values from [shoes provider](https://github.com/search?q=topic%3Amyshoes-provider). ================================================ FILE: docs/01_02_for_admin_tips.md ================================================ # Tips for myshoes admin ## Job management hooks for self-hosted runners You can use job management hooks for self-hosted runners. Please set script file to your runner image. - `ACTIONS_RUNNER_HOOK_JOB_STARTED`: `/myshoes-actions-runner-hook-job-started.sh` - `ACTIONS_RUNNER_HOOK_JOB_COMPLETED`: `/myshoes-actions-runner-hook-job-completed.sh` ================================================ FILE: docs/02_01_for_user_setup.md ================================================ # Setup (only once) ## Goal - Start provision runner ## Prepare - Get GitHub Apps's Public page from myshoes admin. - e.g.) `https://github.com/apps/` or `/github-apps/` - Get Endpoint for myshoes from myshoes admin. - e.g.) `/target` ## Repository or Organization setup ### Install GitHub Apps Please open GitHub Apps's Public page and install GitHub Apps to Organization or repository. ![](./assets/img/02_01_githubapps_publicpage.png) ![](./assets/img/02_01_githubapps_installpage.png) ### Register target to myshoes you need to register a target that repository or organization. - `scope`: set target scope for an auto-scaling runner. - Repository example: `octocat/hello-worlds` - Organization example: `octocat` - `resource_type`: set instance size for a runner. - We will describe later. - Please teach it from myshoes admin. Example (create a target): ```bash $ curl -XPOST -d '{"scope": "octocat/hello-world", "resource_type": "micro"}' ${your_shoes_host}/target ``` You can check registered targets. ```bash curl -XGET ${your_shoes_host}/target | jq . [ { "id": "477f6073-90d1-47d8-958f-4707cea61e8d", "scope": "octocat", "token_expired_at": "2006-01-02T15:04:05Z", "resource_type": "micro", "provider_url": "", "status": "active", "status_description": "", "created_at": "2006-01-02T15:04:05Z", "updated_at": "2006-01-02T15:04:05Z" } ] ``` #### Switch `resource_type` You can set `resource_type` in target. So myshoes switch size of instance. For example, - In organization scope (`octocat`), want to set small size runner as a `nano`. - But specific repository (`octocat/huge-repository`), want to set big size runner as a `4xlarge`. So please configure it. ```bash $ curl -XPOST -d '{"scope": "octocat", "resource_type": "nano"}' ${your_shoes_host}/target $ curl -XPOST -d '{"scope": "octocat/huge-repository", "resource_type": "4xlarge"}' ${your_shoes_host}/target $ curl -XGET ${your_shoes_host}/target | jq . [ { "id": "477f6073-90d1-47d8-958f-4707cea61e8d", "scope": "octocat", "token_expired_at": "2006-01-02T15:04:05Z", "resource_type": "nano", "provider_url": "", "status": "active", "status_description": "", "created_at": "2006-01-02T15:04:05Z", "updated_at": "2006-01-02T15:04:05Z" }, { "id": "3775e3b6-08e0-4abc-830d-fd5325397de0", "scope": "octocat/huge-repository", "token_expired_at": "2006-01-02T15:04:05Z", "resource_type": "4xlarge", "provider_url": "", "status": "active", "status_description": "", "created_at": "2006-01-02T15:04:05Z", "updated_at": "2006-01-02T15:04:05Z" } ] ``` In this configuration, myshoes will create under it. - In `octocat/normal-repository`, will create `nano` - In `octocat/normal-repository2`, will create `nano` - In `octocat/huge-repository`, will create `4xlarge` ### Create an offline runner (only use `check_run` mode) GitHub Actions need offline runner if queueing job. Please create an offline runner in the target repository. https://docs.github.com/en/free-pro-team@latest/actions/hosting-your-own-runners/adding-self-hosted-runners Please delete a runner after registered. After that, You can use [cycloud-io/refresh-runner-action](https://github.com/cycloud-io/refresh-runner-action) for automation. ### Let's go using your shoes! Let's execute your jobs! :runner::runner::runner: ================================================ FILE: docs/03_how-to-develop-shoes.md ================================================ # How to develop shoes provider ## TL;DR - implement to gRPC server - `shoes`, `health`, `stdio` - define resource type in your shoes provider's flavor. ## gRPC server shoes provider use [hashicorp/go-plugin](https://github.com/hashicorp/go-plugin). you need to register three Service. if you use a golang in development, you can use `pkg/pluginutils/setup.go`. please check `plugins/shoes-mock`. There are mock shoes provider. ### shoes server `shoes` is gRPC server. you need to implement two funtion. - `AddInstance` - `DeleteInstance` please check `api/proto/myshoes.proto`. ### health `health` is [grpc-ecosystem/grpc-health-probe](https://github.com/grpc-ecosystem/grpc-health-probe). ### stdio `stdio` is standard I/O service. this service communicate plugin binary's standard I/O. ## Resource type myshoes defined some machine type. you need to map machine spec for your resource type. - nano - micro - small - medium - large - xlarge - 2xlarge - 3xlarge - 4xlarge ## Testing shoes provider `shoes-tester` is a CLI tool for testing shoes provider without running myshoes server. ### Build ```bash go build -o shoes-tester ./cmd/shoes-tester ``` ### Usage #### Add instance Simple mode (with setup script): ```bash ./shoes-tester add \ --plugin ./path/to/your-shoes-provider \ --runner-name test-runner \ --resource-type nano \ --labels "label1,label2" \ --setup-script "#!/bin/bash\necho 'setup'" ``` Script generation mode (generate setup script automatically): ```bash ./shoes-tester add \ --plugin ./path/to/your-shoes-provider \ --runner-name test-runner \ --resource-type nano \ --generate-script \ --scope owner/repo \ --github-app-id 123456 \ --github-private-key-path /path/to/key.pem \ --runner-version latest ``` #### Delete instance ```bash ./shoes-tester delete \ --plugin ./path/to/your-shoes-provider \ --cloud-id your-cloud-id \ --labels "label1,label2" ``` #### Options Add command: - `--plugin`: Path to shoes-provider binary (required) - `--runner-name`: Runner name (required) - `--resource-type`: Resource type (default: nano) - `--labels`: Comma-separated labels - `--setup-script`: Setup script (simple mode) - `--generate-script`: Generate setup script automatically (script generation mode) - `--scope`: Repository (owner/repo) or Organization (script generation mode) - `--github-app-id`: GitHub App ID (script generation mode) - `--github-private-key-path`: GitHub App private key path (script generation mode) - `--runner-version`: Runner version (default: latest, script generation mode) - `--runner-user`: Runner user (default: runner, script generation mode) - `--runner-base-directory`: Runner base directory (default: /tmp, script generation mode) - `--github-url`: GitHub Enterprise Server URL (script generation mode) - `--json`: Output in JSON format Delete command: - `--plugin`: Path to shoes-provider binary (required) - `--cloud-id`: Cloud ID (required) - `--labels`: Comma-separated labels - `--json`: Output in JSON format ================================================ FILE: docs/assets/myshoes.service ================================================ [Unit] Description=myshoes is Auto scaling self-hosted runner :runner: (like GitHub-hosted) for GitHub Actions After=network.target [Service] User=root EnvironmentFile=/etc/default/myshoes ExecStart=/usr/local/bin/myshoes Restart=always [Install] WantedBy=multi-user.target ================================================ FILE: go.mod ================================================ module github.com/whywaita/myshoes go 1.25 require ( github.com/bradleyfalzon/ghinstallation/v2 v2.17.0 github.com/go-sql-driver/mysql v1.9.3 github.com/golang-jwt/jwt/v4 v4.5.2 github.com/google/go-cmp v0.7.0 github.com/google/go-github/v80 v80.0.0 github.com/hashicorp/go-plugin v1.7.0 github.com/hashicorp/go-version v1.8.0 github.com/jmoiron/sqlx v1.4.0 github.com/m4ns0ur/httpcache v0.0.0-20200426190423-1040e2e8823f github.com/ory/dockertest/v3 v3.12.0 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/prometheus/client_golang v1.23.2 github.com/r3labs/diff/v2 v2.15.1 github.com/satori/go.uuid v1.2.0 goji.io v2.0.2+incompatible golang.org/x/oauth2 v0.32.0 golang.org/x/sync v0.18.0 google.golang.org/grpc v1.77.0 google.golang.org/protobuf v1.36.10 ) require ( dario.cat/mergo v1.0.2 // indirect filippo.io/edwards25519 v1.1.0 // indirect github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/containerd/continuity v0.4.5 // indirect github.com/containerd/errdefs v1.0.0 // indirect github.com/containerd/errdefs/pkg v0.3.0 // indirect github.com/distribution/reference v0.6.0 // indirect github.com/docker/cli v29.1.2+incompatible // indirect github.com/docker/go-connections v0.6.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/fatih/color v1.18.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-viper/mapstructure/v2 v2.4.0 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/go-github/v75 v75.0.0 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/hashicorp/go-hclog v1.6.3 // indirect github.com/hashicorp/yamux v0.1.2 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/moby/api v1.52.0 // indirect github.com/moby/moby/client v0.2.1 // indirect github.com/moby/sys/user v0.4.0 // indirect github.com/moby/term v0.5.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/oklog/run v1.2.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.1 // indirect github.com/opencontainers/runc v1.2.3 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.66.1 // indirect github.com/prometheus/procfs v0.16.1 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/vmihailenco/msgpack v4.0.4+incompatible // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/xeipuuv/gojsonschema v1.2.0 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 // indirect go.opentelemetry.io/otel v1.38.0 // indirect go.opentelemetry.io/otel/metric v1.38.0 // indirect go.opentelemetry.io/otel/trace v1.38.0 // indirect go.yaml.in/yaml/v2 v2.4.2 // indirect golang.org/x/net v0.47.0 // indirect golang.org/x/sys v0.38.0 // indirect golang.org/x/text v0.31.0 // indirect google.golang.org/appengine v1.6.8 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) ================================================ FILE: go.sum ================================================ dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg= github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw= github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bradleyfalzon/ghinstallation/v2 v2.17.0 h1:SmbUK/GxpAspRjSQbB6ARvH+ArzlNzTtHydNyXUQ6zg= github.com/bradleyfalzon/ghinstallation/v2 v2.17.0/go.mod h1:vuD/xvJT9Y+ZVZRv4HQ42cMyPFIYqpc7AbB4Gvt/DlY= github.com/bufbuild/protocompile v0.14.1 h1:iA73zAf/fyljNjQKwYzUHD6AD4R8KMasmwa/FBatYVw= github.com/bufbuild/protocompile v0.14.1/go.mod h1:ppVdAIhbr2H8asPk6k4pY7t9zB1OU5DoEw9xY/FUi1c= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/containerd/continuity v0.4.5 h1:ZRoN1sXq9u7V6QoHMcVWGhOwDFqZ4B9i5H6un1Wh0x4= github.com/containerd/continuity v0.4.5/go.mod h1:/lNJvtJKUQStBzpVQ1+rasXO1LAWtUQssk28EZvJ3nE= github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s= github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/docker/cli v29.1.2+incompatible h1:s4QI7drXpIo78OM+CwuthPsO5kCf8cpNsck5PsLVTH8= github.com/docker/cli v29.1.2+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94= github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1aweo= github.com/go-sql-driver/mysql v1.9.3/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU= github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI= github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/go-github/v75 v75.0.0 h1:k7q8Bvg+W5KxRl9Tjq16a9XEgVY1pwuiG5sIL7435Ic= github.com/google/go-github/v75 v75.0.0/go.mod h1:H3LUJEA1TCrzuUqtdAQniBNwuKiQIqdGKgBo1/M/uqI= github.com/google/go-github/v80 v80.0.0 h1:BTyk3QOHekrk5VF+jIGz1TNEsmeoQG9K/UWaaP+EWQs= github.com/google/go-github/v80 v80.0.0/go.mod h1:pRo4AIMdHW83HNMGfNysgSAv0vmu+/pkY8nZO9FT9Yo= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= github.com/hashicorp/go-plugin v1.7.0 h1:YghfQH/0QmPNc/AZMTFE3ac8fipZyZECHdDPshfk+mA= github.com/hashicorp/go-plugin v1.7.0/go.mod h1:BExt6KEaIYx804z8k4gRzRLEvxKVb+kn0NMcihqOqb8= github.com/hashicorp/go-version v1.8.0 h1:KAkNb1HAiZd1ukkxDFGmokVZe1Xy9HG6NUp+bPle2i4= github.com/hashicorp/go-version v1.8.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/yamux v0.1.2 h1:XtB8kyFOyHXYVFnwT5C3+Bdo8gArse7j2AQ0DA0Uey8= github.com/hashicorp/yamux v0.1.2/go.mod h1:C+zze2n6e/7wshOZep2A70/aQU6QBRWJO/G6FT1wIns= github.com/jhump/protoreflect v1.17.0 h1:qOEr613fac2lOuTgWN4tPAtLL7fUSbuJL5X5XumQh94= github.com/jhump/protoreflect v1.17.0/go.mod h1:h9+vUUL38jiBzck8ck+6G/aeMX8Z4QUY/NiJPwPNi+8= github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o= github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/m4ns0ur/httpcache v0.0.0-20200426190423-1040e2e8823f h1:MBcrTbmCf7CZa9yAwcB7ArveQb9TPVy4zFnQGz/LiUU= github.com/m4ns0ur/httpcache v0.0.0-20200426190423-1040e2e8823f/go.mod h1:UawoqorwkpZ58qWiL+nVJM0Po7FrzAdCxYVh9GgTTaA= github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/moby/api v1.52.0 h1:00BtlJY4MXkkt84WhUZPRqt5TvPbgig2FZvTbe3igYg= github.com/moby/moby/api v1.52.0/go.mod h1:8mb+ReTlisw4pS6BRzCMts5M49W5M7bKt1cJy/YbAqc= github.com/moby/moby/client v0.2.1 h1:1Grh1552mvv6i+sYOdY+xKKVTvzJegcVMhuXocyDz/k= github.com/moby/moby/client v0.2.1/go.mod h1:O+/tw5d4a1Ha/ZA/tPxIZJapJRUS6LNZ1wiVRxYHyUE= github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs= github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs= github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ= github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/oklog/run v1.2.0 h1:O8x3yXwah4A73hJdlrwo/2X6J62gE5qTMusH0dvz60E= github.com/oklog/run v1.2.0/go.mod h1:mgDbKRSwPhJfesJ4PntqFUbKQRZ50NgmZTSPlFA0YFk= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= github.com/opencontainers/runc v1.2.3 h1:fxE7amCzfZflJO2lHXf4y/y8M1BoAqp+FVmG19oYB80= github.com/opencontainers/runc v1.2.3/go.mod h1:nSxcWUydXrsBZVYNSkTjoQ/N6rcyTtn+1SD5D4+kRIM= github.com/ory/dockertest/v3 v3.12.0 h1:3oV9d0sDzlSQfHtIaB5k6ghUCVMVLpAY8hwrqoCyRCw= github.com/ory/dockertest/v3 v3.12.0/go.mod h1:aKNDTva3cp8dwOWwb9cWuX84aH5akkxXRvO7KCwWVjE= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs= github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= github.com/r3labs/diff/v2 v2.15.1 h1:EOrVqPUzi+njlumoqJwiS/TgGgmZo83619FNDB9xQUg= github.com/r3labs/diff/v2 v2.15.1/go.mod h1:I8noH9Fc2fjSaMxqF3G2lhDdC0b+JXCfyx85tWFM9kc= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/vmihailenco/msgpack v4.0.4+incompatible h1:dSLoQfGFAo3F6OoNhwUmLwVgaUXK79GlxNBwueZn0xI= github.com/vmihailenco/msgpack v4.0.4+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 h1:RbKq8BG0FI8OiXhBfcRtqqHcZcka+gU3cskNuf05R18= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0/go.mod h1:h06DGIukJOevXaj/xrNjhi/2098RZzcLTbc0jDAUbsg= go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM= go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA= go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI= go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E= go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg= go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM= go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA= go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE= go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= goji.io v2.0.2+incompatible h1:uIssv/elbKRLznFUy3Xj4+2Mz/qKhek/9aZQDUMae7c= goji.io v2.0.2+incompatible/go.mod h1:sbqFwrtqZACxLBTQcdgVjFh54yGVCvwq8+w49MVMMIk= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= golang.org/x/oauth2 v0.32.0 h1:jsCblLleRMDrxMN29H3z/k1KliIvpLgCkE6R8FXXNgY= golang.org/x/oauth2 v0.32.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 h1:gRkg/vSppuSQoDjxyiGfN4Upv/h/DQmIR10ZU8dh4Ww= google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk= google.golang.org/grpc v1.77.0 h1:wVVY6/8cGA6vvffn+wWK5ToddbgdU3d8MNENr4evgXM= google.golang.org/grpc v1.77.0/go.mod h1:z0BY1iVj0q8E1uSQCjL9cppRj+gnZjzDnzV0dHhrNig= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= pgregory.net/rapid v1.2.0 h1:keKAYRcjm+e1F0oAuU5F5+YPAWcyxNNRK2wud503Gnk= pgregory.net/rapid v1.2.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04= ================================================ FILE: internal/testutils/mysql.go ================================================ package testutils import ( "fmt" "log" "os" "path" "runtime" "strings" "github.com/go-sql-driver/mysql" "github.com/jmoiron/sqlx" "github.com/whywaita/myshoes/pkg/datastore" ) const schemaDirRelativePathFormat = "%s/../../pkg/datastore/mysql/%s" func execSchema(fpath string) { b, err := os.ReadFile(fpath) if err != nil { log.Fatalf("schema reading error: %v", err) } queries := strings.Split(string(b), ";") for _, query := range queries[:len(queries)-1] { _, err = testDB.Exec(query) if err != nil { log.Fatalf("exec schema error: %v, query: %s", err, query) } } } func createTablesIfNotExist() { _, pwd, _, _ := runtime.Caller(0) schemaPath := fmt.Sprintf(schemaDirRelativePathFormat, path.Dir(pwd), "schema.sql") execSchema(schemaPath) } func truncateTables() { rows, err := testDB.Query("SHOW TABLES") if err != nil { log.Fatalf("show tables error: %#v", err) } defer rows.Close() for rows.Next() { var tableName string err = rows.Scan(&tableName) if err != nil { log.Fatalf("show table error: %#v", err) continue } cmds := []string{ "SET FOREIGN_KEY_CHECKS = 0", fmt.Sprintf("TRUNCATE %s", tableName), "SET FOREIGN_KEY_CHECKS = 1", } for _, cmd := range cmds { _, err := testDB.Exec(cmd) if err != nil { mysqlErr, ok := err.(*mysql.MySQLError) if ok { if mysqlErr.Number == 0xde2 { // is rejected continue } } else { log.Fatalf("truncate error: %#v", err) continue } } } } } // GetTestDatastore return pointer of datastore func GetTestDatastore() (datastore.Datastore, func()) { if testDatastore == nil { panic("datastore is not initialized yet") } return testDatastore, func() { truncateTables() } } // GetTestDB return pointer of testDB func GetTestDB() (*sqlx.DB, func()) { if testDB == nil { panic("testDB is not initialized yet") } return testDB, func() { truncateTables() } } ================================================ FILE: internal/testutils/testutils.go ================================================ package testutils import ( "fmt" "log" "net/http/httptest" "testing" "github.com/whywaita/myshoes/pkg/datastore" "github.com/whywaita/myshoes/pkg/datastore/mysql" "github.com/whywaita/myshoes/pkg/web" "github.com/jmoiron/sqlx" "github.com/ory/dockertest/v3" ) const ( mysqlRootPassword = "secret" ) var ( testDB *sqlx.DB testDatastore datastore.Datastore testURL string ) // IntegrationTestRunner is all integration test func IntegrationTestRunner(m *testing.M) int { // uses a sensible default on windows (tcp/http) and linux/osx (socket) pool, err := dockertest.NewPool("") if err != nil { log.Fatalf("Could not connect to docker: %s", err) } // pulls an image, creates a container based on it and runs it resource, err := pool.Run("mysql", "8.0", []string{"MYSQL_ROOT_PASSWORD=" + mysqlRootPassword}) if err != nil { log.Fatalf("Could not start resource: %s", err) } // exponential backoff-retry, because the application in the container might not be ready to accept connections yet if err := pool.Retry(func() error { var err error dsn := fmt.Sprintf("root:%s@(localhost:%s)/mysql", mysqlRootPassword, resource.GetPort("3306/tcp")) testDatastore, err = mysql.New(dsn, make(chan<- struct{})) if err != nil { log.Fatalf("failed to create datastore instance: %s", err) } testDB, err = sqlx.Open("mysql", fmt.Sprintf("root:%s@(localhost:%s)/mysql?parseTime=true&loc=UTC", mysqlRootPassword, resource.GetPort("3306/tcp"))) if err != nil { return err } return testDB.Ping() }); err != nil { log.Fatalf("Could not connect to docker: %s", err) } createTablesIfNotExist() //SetupDefaultFixtures() mux := web.NewMux(testDatastore) ts := httptest.NewServer(mux) testURL = ts.URL code := m.Run() ts.Close() truncateTables() // You can't defer this because os.Exit doesn't care for defer if err := pool.Purge(resource); err != nil { log.Fatalf("Could not purge resource: %s", err) } return code } ================================================ FILE: internal/testutils/web.go ================================================ package testutils // GetTestURL return url of httptest.Server func GetTestURL() string { if testURL == "" { panic("testURL is not initialized yet") } return testURL } ================================================ FILE: internal/util/util.go ================================================ package util import ( "math/rand/v2" "time" ) // CalcRetryTime is caliculate retry time by exponential backoff and jitter func CalcRetryTime(count int) time.Duration { if count == 0 { return 0 } backoff := 1 << count jitter := time.Duration(rand.IntN(1000)) * time.Millisecond return time.Duration(backoff)*time.Second + jitter } ================================================ FILE: pkg/config/config.go ================================================ package config import ( "crypto/rsa" "strings" ) // Config is config value var Config Conf // Conf is type of Config type Conf struct { GitHub GitHubApp MySQLDSN string Port int ShoesPluginPath string ShoesPluginOutputPath string RunnerUser string RunnerBaseDirectory string Debug bool Strict bool // check to registered runner before delete job ModeWebhookType ModeWebhookType MaxConnectionsToBackend int64 MaxConcurrencyDeleting int64 GitHubURL string RunnerVersion string DockerHubCredential DockerHubCredential ProvideDockerHubMetrics bool } // DockerHubCredential is type of config value type DockerHubCredential struct { Username string Password string } // GitHubApp is type of config value type GitHubApp struct { AppID int64 AppSecret []byte PEMByte []byte PEM *rsa.PrivateKey } // Config Environment keys const ( EnvGitHubAppID = "GITHUB_APP_ID" EnvGitHubAppSecret = "GITHUB_APP_SECRET" EnvGitHubAppPrivateKeyBase64 = "GITHUB_PRIVATE_KEY_BASE64" EnvMySQLHost = "MYSQL_HOST" EnvMySQLPort = "MYSQL_PORT" EnvMySQLUser = "MYSQL_USER" EnvMySQLPassword = "MYSQL_PASSWORD" EnvMySQLDatabase = "MYSQL_DATABASE" EnvMySQLURL = "MYSQL_URL" EnvPort = "PORT" EnvShoesPluginPath = "PLUGIN" EnvShoesPluginOutputPath = "PLUGIN_OUTPUT" EnvRunnerUser = "RUNNER_USER" EnvRunnerBaseDirectory = "RUNNER_BASE_DIRECTORY" EnvDebug = "DEBUG" EnvStrict = "STRICT" EnvModeWebhookType = "MODE_WEBHOOK_TYPE" EnvMaxConnectionsToBackend = "MAX_CONNECTIONS_TO_BACKEND" EnvMaxConcurrencyDeleting = "MAX_CONCURRENCY_DELETING" EnvGitHubURL = "GITHUB_URL" EnvRunnerVersion = "RUNNER_VERSION" EnvDockerHubUsername = "DOCKER_HUB_USERNAME" EnvDockerHubPassword = "DOCKER_HUB_PASSWORD" EnvProvideDockerHubMetrics = "PROVIDE_DOCKER_HUB_METRICS" ) // ModeWebhookType is type value for GitHub webhook type ModeWebhookType int const ( // ModeWebhookTypeUnknown is unknown ModeWebhookTypeUnknown ModeWebhookType = iota // ModeWebhookTypeCheckRun is check_run ModeWebhookTypeCheckRun // ModeWebhookTypeWorkflowJob is workflow_job ModeWebhookTypeWorkflowJob ) // String is implementation of fmt.Stringer func (mwt ModeWebhookType) String() string { unknown := "unknown" switch mwt { case ModeWebhookTypeUnknown: return unknown case ModeWebhookTypeCheckRun: return "check_run" case ModeWebhookTypeWorkflowJob: return "workflow_job" } return unknown } // Equal check in and value func (mwt ModeWebhookType) Equal(in string) bool { return strings.EqualFold(in, mwt.String()) } func marshalModeWebhookType(in string) ModeWebhookType { switch in { case "check_run": return ModeWebhookTypeCheckRun case "workflow_job": return ModeWebhookTypeWorkflowJob } return ModeWebhookTypeUnknown } // IsGHES return myshoes for GitHub Enterprise Server func (c Conf) IsGHES() bool { return !strings.EqualFold(c.GitHubURL, "https://github.com") } ================================================ FILE: pkg/config/init.go ================================================ package config import ( "crypto/x509" "encoding/base64" "encoding/pem" "fmt" "io" "log" "net/http" "net/url" "os" "path/filepath" "strconv" "strings" "github.com/hashicorp/go-version" ) // Load load config from environment func Load() { c := LoadWithDefault() ga := LoadGitHubApps() c.GitHub = *ga pluginPath := LoadPluginPath() c.ShoesPluginPath = pluginPath Config = c } // LoadWithDefault load only value that has default value func LoadWithDefault() Conf { var c Conf p := "8080" if os.Getenv(EnvPort) != "" { p = os.Getenv(EnvPort) } pp, err := strconv.Atoi(p) if err != nil { log.Panicf("failed to parse PORT: %+v", err) } c.Port = pp runnerUser := "runner" if os.Getenv(EnvRunnerUser) != "" { runnerUser = os.Getenv(EnvRunnerUser) } c.RunnerUser = runnerUser c.RunnerBaseDirectory = "/tmp" if os.Getenv(EnvRunnerBaseDirectory) != "" { c.RunnerBaseDirectory = os.Getenv(EnvRunnerBaseDirectory) log.Printf("use runner base directory is %s\n", c.RunnerBaseDirectory) } c.Debug = false if os.Getenv(EnvDebug) == "true" { c.Debug = true } c.Strict = true if os.Getenv(EnvStrict) == "false" { c.Strict = false } c.ModeWebhookType = ModeWebhookTypeWorkflowJob if os.Getenv(EnvModeWebhookType) != "" { mwt := marshalModeWebhookType(os.Getenv(EnvModeWebhookType)) if mwt == ModeWebhookTypeUnknown { log.Panicf("%s is invalid webhook type", os.Getenv(EnvModeWebhookType)) } if mwt == ModeWebhookTypeCheckRun { log.Println("WARNING: check_run is deprecated mode and will delete it. Please use workflow_job") } c.ModeWebhookType = mwt } c.ProvideDockerHubMetrics = false if os.Getenv(EnvProvideDockerHubMetrics) == "true" { c.ProvideDockerHubMetrics = true } c.DockerHubCredential = DockerHubCredential{} if c.ProvideDockerHubMetrics { if os.Getenv(EnvDockerHubUsername) != "" && os.Getenv(EnvDockerHubPassword) != "" { c.DockerHubCredential.Username = os.Getenv(EnvDockerHubUsername) c.DockerHubCredential.Password = os.Getenv(EnvDockerHubPassword) } else { log.Println("WARNING: Providing Docker Hub metrics is enabled, but DOCKER_HUB_USERNAME and DOCKER_HUB_PASSWORD are not set. Providing Docker Hub metrics with anonymous user mode") } } else { log.Println("Docker Hub metrics is disabled") } c.MaxConnectionsToBackend = 50 if os.Getenv(EnvMaxConnectionsToBackend) != "" { numberPB, err := strconv.ParseInt(os.Getenv(EnvMaxConnectionsToBackend), 10, 64) if err != nil { log.Panicf("failed to convert int64 %s: %+v", EnvMaxConnectionsToBackend, err) } c.MaxConnectionsToBackend = numberPB } c.MaxConcurrencyDeleting = 1 if os.Getenv(EnvMaxConcurrencyDeleting) != "" { numberCD, err := strconv.ParseInt(os.Getenv(EnvMaxConcurrencyDeleting), 10, 64) if err != nil { log.Panicf("failed to convert int64 %s: %+v", EnvMaxConcurrencyDeleting, err) } c.MaxConcurrencyDeleting = numberCD } c.GitHubURL = "https://github.com" if os.Getenv(EnvGitHubURL) != "" { u, err := url.Parse(os.Getenv(EnvGitHubURL)) if err != nil { log.Panicf("failed to parse URL %s: %+v", os.Getenv(EnvGitHubURL), err) } if strings.EqualFold(u.Scheme, "") { log.Panicf("%s must has scheme (value: %s)", EnvGitHubURL, os.Getenv(EnvGitHubURL)) } if strings.EqualFold(u.Host, "") { log.Panicf("%s must has host (value: %s)", EnvGitHubURL, os.Getenv(EnvGitHubURL)) } c.GitHubURL = os.Getenv(EnvGitHubURL) } if os.Getenv(EnvRunnerVersion) == "" { c.RunnerVersion = "latest" } else { // valid value: "latest" or "vX.XXX.X" switch os.Getenv(EnvRunnerVersion) { case "latest": c.RunnerVersion = "latest" default: _, err := version.NewVersion(os.Getenv(EnvRunnerVersion)) if err != nil { log.Panicf("failed to parse input runner version: %+v", err) } c.RunnerVersion = os.Getenv(EnvRunnerVersion) } } c.ShoesPluginOutputPath = "." if os.Getenv(EnvShoesPluginOutputPath) != "" { c.ShoesPluginOutputPath = os.Getenv(EnvShoesPluginOutputPath) } Config = c return c } // LoadGitHubApps load config for GitHub Apps func LoadGitHubApps() *GitHubApp { var ga GitHubApp appID, err := strconv.ParseInt(os.Getenv(EnvGitHubAppID), 10, 64) if err != nil { log.Panicf("failed to parse %s: %+v", EnvGitHubAppID, err) } ga.AppID = appID pemBase64ed := os.Getenv(EnvGitHubAppPrivateKeyBase64) if pemBase64ed == "" { log.Panicf("%s must be set", EnvGitHubAppPrivateKeyBase64) } pemByte, err := base64.StdEncoding.DecodeString(pemBase64ed) if err != nil { log.Panicf("failed to decode base64 %s: %+v", EnvGitHubAppPrivateKeyBase64, err) } ga.PEMByte = pemByte block, _ := pem.Decode(pemByte) if block == nil { log.Panicf("%s is invalid format, please input private key ", EnvGitHubAppPrivateKeyBase64) } key, err := x509.ParsePKCS1PrivateKey(block.Bytes) if err != nil { log.Panicf("%s is invalid format, failed to parse private key: %+v", EnvGitHubAppPrivateKeyBase64, err) } ga.PEM = key appSecret := os.Getenv(EnvGitHubAppSecret) if appSecret == "" { log.Panicf("%s must be set", EnvGitHubAppSecret) } ga.AppSecret = []byte(appSecret) return &ga } // LoadMySQLURL load MySQL URL from environment func LoadMySQLURL() string { mysqlHost, ok_Host := os.LookupEnv(EnvMySQLHost) mysqlPort, ok_Port := os.LookupEnv(EnvMySQLPort) mysqlUser, ok_User := os.LookupEnv(EnvMySQLUser) mysqlPassword, ok_Password := os.LookupEnv(EnvMySQLPassword) mysqlDatabase, ok_Database := os.LookupEnv(EnvMySQLDatabase) if ok_Host && ok_Port && ok_User && ok_Password && ok_Database { mysqlURL := fmt.Sprintf("%s:%s@tcp(%s:%s)/%s", mysqlUser, mysqlPassword, mysqlHost, mysqlPort, mysqlDatabase) log.Println("load MySQL URL from environment variables MYSQL_USER, MYSQL_PASSWORD, MYSQL_HOST, MYSQL_PORT, MYSQL_DATABASE, not MYSQL_URL") return mysqlURL } mysqlURL := os.Getenv(EnvMySQLURL) if mysqlURL == "" { log.Panicf("%s must be set", EnvMySQLURL) } return mysqlURL } // LoadPluginPath load plugin path from environment func LoadPluginPath() string { pluginPath := os.Getenv(EnvShoesPluginPath) if pluginPath == "" { log.Panicf("%s must be set", EnvShoesPluginPath) } fp, err := fetch(pluginPath) if err != nil { log.Panicf("failed to fetch plugin binary: %+v", err) } absPath, err := checkBinary(fp) if err != nil { log.Panicf("failed to check plugin binary: %+v", err) } log.Printf("use plugin path is %s\n", absPath) return absPath } func checkBinary(p string) (string, error) { f, err := os.ReadFile(p) if err != nil { return "", fmt.Errorf("failed to open file: %w", err) } // check binary type mineType := http.DetectContentType(f) if !strings.EqualFold(mineType, "application/octet-stream") { return "", fmt.Errorf("invalid file type (correct: application/octet-stream got: %s)", mineType) } // need permission of execute if err := os.Chmod(p, 0777); err != nil { return "", fmt.Errorf("failed to chmod: %w", err) } if filepath.IsAbs(p) { return p, nil } apath, err := filepath.Abs(p) if err != nil { return "", fmt.Errorf("failed to get abs: %w", err) } return apath, nil } // fetch retrieve plugin binaries. // return saved file path. func fetch(p string) (string, error) { _, err := os.Stat(p) if err == nil { // this is file path! return p, nil } u, err := url.Parse(p) if err != nil { return "", fmt.Errorf("failed to parse input url: %w", err) } switch u.Scheme { case "http", "https": return fetchHTTP(u) default: return "", fmt.Errorf("unsupported fetch schema (scheme: %s)", u.Scheme) } } // fetchHTTP fetch plugin binary over HTTP(s). // save to current directory. func fetchHTTP(u *url.URL) (string, error) { log.Printf("fetch plugin binary from %s\n", u.String()) dir := Config.ShoesPluginOutputPath if strings.EqualFold(dir, ".") { pwd, err := os.Getwd() if err != nil { return "", fmt.Errorf("failed to working directory: %w", err) } dir = pwd } p := strings.Split(u.Path, "/") fileName := p[len(p)-1] fp := filepath.Join(dir, fileName) f, err := os.Create(fp) if err != nil { return "", fmt.Errorf("failed to create os file: %w", err) } defer f.Close() resp, err := http.Get(u.String()) if err != nil { return "", fmt.Errorf("failed to get config via HTTP(S): %w", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return "", fmt.Errorf("failed to get config via HTTP(S): status code is not 200 (status code: %d)", resp.StatusCode) } if _, err := io.Copy(f, resp.Body); err != nil { return "", fmt.Errorf("failed to write file (path: %s): %w", fp, err) } return fp, nil } ================================================ FILE: pkg/datastore/github.go ================================================ package datastore import ( "context" "fmt" "net/url" "sort" "strings" "sync" "time" "github.com/whywaita/myshoes/pkg/logger" "github.com/google/go-github/v80/github" "github.com/whywaita/myshoes/pkg/gh" ) // NewClientInstallationByRepo create a client of GitHub using installation ID from repo name func NewClientInstallationByRepo(ctx context.Context, ds Datastore, repo string) (*github.Client, *Target, error) { target, err := SearchRepo(ctx, ds, repo) if err != nil { return nil, nil, fmt.Errorf("failed to search repository: %w", err) } installationID, err := gh.IsInstalledGitHubApp(ctx, target.Scope) if err != nil { return nil, nil, fmt.Errorf("failed to get installation ID: %w", err) } client, err := gh.NewClientInstallation(installationID) if err != nil { return nil, nil, fmt.Errorf("failed to create client: %w", err) } return client, target, nil } // PendingWorkflowRunWithTarget is struct for pending workflow run type PendingWorkflowRunWithTarget struct { Target *Target WorkflowRun *github.WorkflowRun } // GetPendingWorkflowRunByRecentRepositories get pending workflow runs by recent active repositories func GetPendingWorkflowRunByRecentRepositories(ctx context.Context, ds Datastore) ([]PendingWorkflowRunWithTarget, error) { pendingRuns, err := getPendingWorkflowRunByRecentRepositories(ctx, ds) if err != nil { return nil, fmt.Errorf("failed to get pending workflow runs: %w", err) } queuedJob, err := ds.ListJobs(ctx) if err != nil { return nil, fmt.Errorf("failed to get list of jobs: %w", err) } var result []PendingWorkflowRunWithTarget // We ignore the pending run if the job is already queued. for _, pendingRun := range pendingRuns { found := false for _, job := range queuedJob { webhookEvent, err := github.ParseWebHook("workflow_job", []byte(job.CheckEventJSON)) if err != nil { logger.Logf(false, "failed to parse webhook payload (job id: %s): %+v", job.UUID, err) continue } workflowJob, ok := webhookEvent.(*github.WorkflowJobEvent) if !ok { logger.Logf(false, "failed to cast to WorkflowJobEvent (job id: %s)", job.UUID) continue } if pendingRun.WorkflowRun.GetID() == workflowJob.GetWorkflowJob().GetRunID() { logger.Logf(true, "found job in datastore, So will ignore: (repo: %s, gh_run_id: %d, gh_job_id: %d)", pendingRun.WorkflowRun.GetRepository().GetFullName(), pendingRun.WorkflowRun.GetID(), workflowJob.GetWorkflowJob().GetID()) found = true break } } if !found { result = append(result, pendingRun) } } return result, nil } func getPendingWorkflowRunByRecentRepositories(ctx context.Context, ds Datastore) ([]PendingWorkflowRunWithTarget, error) { recentActiveRepositories, err := getRecentRepositories(ctx, ds) if err != nil { return nil, fmt.Errorf("failed to get recent repositories: %w", err) } var pendingRuns []PendingWorkflowRunWithTarget var wg sync.WaitGroup var mu sync.Mutex for _, repoRawURL := range recentActiveRepositories { wg.Add(1) go func(repoRawURL string) { defer wg.Done() u, err := url.Parse(repoRawURL) if err != nil { logger.Logf(false, "failed to get pending run by recent repositories: failed to parse repository url: %+v", err) return } fullName := strings.TrimPrefix(u.Path, "/") client, target, err := NewClientInstallationByRepo(ctx, ds, fullName) if err != nil { logger.Logf(false, "failed to get pending run by recent repositories: failed to create a client of GitHub by repo (full_name: %s) %+v", fullName, err) return } owner, repo := gh.DivideScope(fullName) pendingRunsByRepo, err := getPendingRunByRepo(ctx, client, owner, repo) if err != nil { logger.Logf(false, "failed to get pending run by recent repositories: failed to get pending run by repo (full_name: %s) %+v", fullName, err) return } mu.Lock() for _, run := range pendingRunsByRepo { pendingRuns = append(pendingRuns, PendingWorkflowRunWithTarget{ Target: target, WorkflowRun: run, }) } mu.Unlock() }(repoRawURL) } wg.Wait() return pendingRuns, nil } func getPendingRunByRepo(ctx context.Context, client *github.Client, owner, repo string) ([]*github.WorkflowRun, error) { runs, err := gh.ListWorkflowRunsNewest(ctx, client, owner, repo, 50) if err != nil { return nil, fmt.Errorf("failed to list runs: %w", err) } var pendingRuns []*github.WorkflowRun for _, r := range runs { if r.GetStatus() == "queued" || r.GetStatus() == "pending" { oldMinutes := 10 sinceMinutes := time.Since(r.CreatedAt.Time).Minutes() if sinceMinutes >= float64(oldMinutes) { logger.Logf(false, "workflow run %d is pending over %d minutes, So will enqueue (repo: %s/%s)", r.GetID(), oldMinutes, owner, repo) pendingRuns = append(pendingRuns, r) } else { logger.Logf(true, "workflow run %d is pending, but not over %d minutes. So ignore (since: %f minutes, repo: %s/%s)", r.GetID(), oldMinutes, sinceMinutes, owner, repo) } } } return pendingRuns, nil } func getRecentRepositories(ctx context.Context, ds Datastore) ([]string, error) { recent := time.Now().Add(-1 * time.Hour) recentRunners, err := ds.ListRunnersLogBySince(ctx, recent) if err != nil { return nil, fmt.Errorf("failed to get targets from datastore: %w", err) } // sort by created_at sort.SliceStable(recentRunners, func(i, j int) bool { return recentRunners[i].CreatedAt.After(recentRunners[j].CreatedAt) }) // unique repositories recentActiveRepositories := make(map[string]struct{}) for _, r := range recentRunners { u := r.RepositoryURL if _, ok := recentActiveRepositories[u]; !ok { recentActiveRepositories[u] = struct{}{} } } var result []string for repository := range recentActiveRepositories { result = append(result, repository) } return result, nil } ================================================ FILE: pkg/datastore/interface.go ================================================ package datastore import ( "context" "database/sql" "errors" "fmt" "net/url" "strings" "time" uuid "github.com/satori/go.uuid" "github.com/whywaita/myshoes/pkg/gh" "github.com/whywaita/myshoes/pkg/logger" ) // Error values var ( ErrNotFound = errors.New("not found") ) // Lock values var ( IsLocked = "is locked" IsNotLocked = "is not locked" ) // Datastore is persistent storage type Datastore interface { CreateTarget(ctx context.Context, target Target) error GetTarget(ctx context.Context, id uuid.UUID) (*Target, error) GetTargetByScope(ctx context.Context, scope string) (*Target, error) ListTargets(ctx context.Context) ([]Target, error) DeleteTarget(ctx context.Context, id uuid.UUID) error // Deprecated: Use datastore.UpdateTargetStatus. UpdateTargetStatus(ctx context.Context, targetID uuid.UUID, newStatus TargetStatus, description string) error UpdateToken(ctx context.Context, targetID uuid.UUID, newToken string, newExpiredAt time.Time) error UpdateTargetParam(ctx context.Context, targetID uuid.UUID, newResourceType ResourceType, newProviderURL sql.NullString) error EnqueueJob(ctx context.Context, job Job) error ListJobs(ctx context.Context) ([]Job, error) DeleteJob(ctx context.Context, id uuid.UUID) error CreateRunner(ctx context.Context, runner Runner) error ListRunners(ctx context.Context) ([]Runner, error) ListRunnersByTargetID(ctx context.Context, targetID uuid.UUID) ([]Runner, error) ListRunnersLogBySince(ctx context.Context, since time.Time) ([]Runner, error) GetRunner(ctx context.Context, id uuid.UUID) (*Runner, error) DeleteRunner(ctx context.Context, id uuid.UUID, deletedAt time.Time, reason RunnerStatus) error // Lock GetLock(ctx context.Context) error IsLocked(ctx context.Context) (string, error) } // Target is a target repository that will add auto-scaling runner. type Target struct { UUID uuid.UUID `db:"uuid" json:"id"` Scope string `db:"scope" json:"scope"` // repo (:owner/:repo) or org (:organization) // deprecated GitHubToken string `db:"github_token" json:"github_token"` TokenExpiredAt time.Time `db:"token_expired_at" json:"token_expired_at"` GHEDomain sql.NullString `db:"ghe_domain" json:"ghe_domain"` ResourceType ResourceType `db:"resource_type" json:"resource_type"` ProviderURL sql.NullString `db:"provider_url" json:"provider_url"` Status TargetStatus `db:"status" json:"status"` StatusDescription sql.NullString `db:"status_description" json:"status_description"` CreatedAt time.Time `db:"created_at" json:"created_at"` UpdatedAt time.Time `db:"updated_at" json:"updated_at"` } // OwnerRepo return :owner and :repo func (t *Target) OwnerRepo() (string, string) { return gh.DivideScope(t.Scope) } // CanReceiveJob check status in target func (t *Target) CanReceiveJob() bool { switch t.Status { case TargetStatusSuspend, TargetStatusDeleted: return false } return true } // ListTargets get list of target that can receive job func ListTargets(ctx context.Context, ds Datastore) ([]Target, error) { targets, err := ds.ListTargets(ctx) if err != nil { return nil, fmt.Errorf("failed to get targets from datastore: %w", err) } var result []Target for _, t := range targets { if t.CanReceiveJob() { result = append(result, t) } } return result, nil } // UpdateTargetStatus update datastore func UpdateTargetStatus(ctx context.Context, ds Datastore, targetID uuid.UUID, newStatus TargetStatus, description string) error { target, err := ds.GetTarget(ctx, targetID) if err != nil { return fmt.Errorf("failed to get target: %w", err) } if !target.CanReceiveJob() { // not change status return nil } if err := ds.UpdateTargetStatus(ctx, targetID, newStatus, description); err != nil { logger.Logf(false, "failed to update target status: %+v", err) return err } return nil } // SearchRepo search datastore.Target from datastore // format of repo is "orgs/repos" func SearchRepo(ctx context.Context, ds Datastore, repo string) (*Target, error) { sep := strings.Split(repo, "/") if len(sep) != 2 { return nil, fmt.Errorf("incorrect repo format ex: orgs/repo (input: %s)", repo) } // use repo scope if set repo repoTarget, err := ds.GetTargetByScope(ctx, repo) if err == nil && repoTarget.CanReceiveJob() { return repoTarget, nil } else if err != nil && !errors.Is(err, ErrNotFound) { return nil, fmt.Errorf("failed to get target from repo: %w", err) } // repo is not found, so search org target org := sep[0] orgTarget, err := ds.GetTargetByScope(ctx, org) if err != nil { return nil, fmt.Errorf("failed to get target from organization: %w", err) } if !orgTarget.CanReceiveJob() { return nil, fmt.Errorf("target is not active") } return orgTarget, nil } // TargetStatus is status for target type TargetStatus string // TargetStatus variables const ( TargetStatusActive TargetStatus = "active" //lint:ignore SA9004 this is status TargetStatusRunning = "running" TargetStatusSuspend = "suspend" TargetStatusDeleted = "deleted" TargetStatusErr = "error" ) // Job is a runner job type Job struct { UUID uuid.UUID `db:"uuid"` GHEDomain sql.NullString `db:"ghe_domain"` Repository string `db:"repository"` // repo (:owner/:repo) CheckEventJSON string `db:"check_event"` TargetID uuid.UUID `db:"target_id"` CreatedAt time.Time `db:"created_at" json:"created_at"` UpdatedAt time.Time `db:"updated_at" json:"updated_at"` } // RepoURL return repository URL that send webhook. func (j *Job) RepoURL() string { serverURL := "https://github.com" if j.GHEDomain.Valid { serverURL = j.GHEDomain.String } s := strings.Split(serverURL, "://") var u url.URL u.Scheme = s[0] u.Host = s[1] u.Path = j.Repository return u.String() } // Runner is a runner type Runner struct { UUID uuid.UUID `db:"runner_id"` ShoesType string `db:"shoes_type"` IPAddress string `db:"ip_address"` TargetID uuid.UUID `db:"target_id"` CloudID string `db:"cloud_id"` Deleted bool `db:"deleted"` Status RunnerStatus `db:"status"` ResourceType ResourceType `db:"resource_type"` RunnerUser sql.NullString `db:"runner_user" json:"runner_user"` ProviderURL sql.NullString `db:"provider_url" json:"provider_url"` RepositoryURL string `db:"repository_url"` RequestWebhook string `db:"request_webhook"` CreatedAt time.Time `db:"created_at"` UpdatedAt time.Time `db:"updated_at"` DeletedAt sql.NullTime `db:"deleted_at"` } // RunnerStatus is status for runner type RunnerStatus string // RunnerStatus variables const ( RunnerStatusCreated RunnerStatus = "created" //lint:ignore SA9004 this is status RunnerStatusCompleted = "completed" RunnerStatusReachHardLimit = "reach_hard_limit" ) ================================================ FILE: pkg/datastore/memory/memory.go ================================================ package memory import ( "context" "database/sql" "fmt" "sync" "time" uuid "github.com/satori/go.uuid" "github.com/whywaita/myshoes/pkg/datastore" ) // Memory is implement datastore on-memory type Memory struct { mu *sync.RWMutex targets map[uuid.UUID]datastore.Target jobs map[uuid.UUID]datastore.Job runners map[uuid.UUID]datastore.Runner } // New create map func New() (*Memory, error) { m := &sync.RWMutex{} t := map[uuid.UUID]datastore.Target{} j := map[uuid.UUID]datastore.Job{} r := map[uuid.UUID]datastore.Runner{} return &Memory{ mu: m, targets: t, jobs: j, runners: r, }, nil } // CreateTarget create a target func (m *Memory) CreateTarget(ctx context.Context, target datastore.Target) error { m.mu.Lock() defer m.mu.Unlock() m.targets[target.UUID] = target return nil } // GetTarget get a target func (m *Memory) GetTarget(ctx context.Context, id uuid.UUID) (*datastore.Target, error) { m.mu.RLock() defer m.mu.RUnlock() t, ok := m.targets[id] if !ok { return nil, datastore.ErrNotFound } return &t, nil } // GetTargetByScope get a target from scope func (m *Memory) GetTargetByScope(ctx context.Context, scope string) (*datastore.Target, error) { m.mu.RLock() defer m.mu.RUnlock() for _, t := range m.targets { if t.Scope == scope { // found return &t, nil } } return nil, datastore.ErrNotFound } // ListTargets get a all targets func (m *Memory) ListTargets(ctx context.Context) ([]datastore.Target, error) { m.mu.RLock() defer m.mu.RUnlock() var targets []datastore.Target for _, t := range m.targets { targets = append(targets, t) } return targets, nil } // DeleteTarget delete a target func (m *Memory) DeleteTarget(ctx context.Context, id uuid.UUID) error { m.mu.Lock() defer m.mu.Unlock() delete(m.targets, id) return nil } // UpdateTargetStatus update status in target func (m *Memory) UpdateTargetStatus(ctx context.Context, targetID uuid.UUID, newStatus datastore.TargetStatus, description string) error { m.mu.Lock() defer m.mu.Unlock() t, ok := m.targets[targetID] if !ok { return fmt.Errorf("not found") } t.Status = newStatus if description != "" { t.StatusDescription.Valid = true } else { t.StatusDescription.Valid = false } t.StatusDescription.String = description m.targets[targetID] = t return nil } // UpdateToken update token in target func (m *Memory) UpdateToken(ctx context.Context, targetID uuid.UUID, newToken string, newExpiredAt time.Time) error { m.mu.Lock() defer m.mu.Unlock() t, ok := m.targets[targetID] if !ok { return fmt.Errorf("not found") } t.GitHubToken = newToken t.TokenExpiredAt = newExpiredAt m.targets[targetID] = t return nil } // UpdateTargetParam update parameter of target func (m *Memory) UpdateTargetParam(ctx context.Context, targetID uuid.UUID, newResourceType datastore.ResourceType, newProviderURL string) error { m.mu.Lock() defer m.mu.Unlock() t, ok := m.targets[targetID] if !ok { return fmt.Errorf("not found") } t.ResourceType = newResourceType t.ProviderURL = sql.NullString{ String: newProviderURL, Valid: true, } m.targets[targetID] = t return nil } // EnqueueJob add a job func (m *Memory) EnqueueJob(ctx context.Context, job datastore.Job) error { m.mu.Lock() defer m.mu.Unlock() m.jobs[job.UUID] = job return nil } // ListJobs get all jobs func (m *Memory) ListJobs(ctx context.Context) ([]datastore.Job, error) { m.mu.RLock() defer m.mu.RUnlock() var jobs []datastore.Job for _, j := range m.jobs { jobs = append(jobs, j) } return jobs, nil } // DeleteJob delete a job func (m *Memory) DeleteJob(ctx context.Context, id uuid.UUID) error { m.mu.Lock() defer m.mu.Unlock() delete(m.jobs, id) return nil } // CreateRunner add a runner func (m *Memory) CreateRunner(ctx context.Context, runner datastore.Runner) error { m.mu.Lock() defer m.mu.Unlock() m.runners[runner.UUID] = runner return nil } // ListRunners get a all runners func (m *Memory) ListRunners(ctx context.Context) ([]datastore.Runner, error) { m.mu.Lock() defer m.mu.Unlock() var runners []datastore.Runner for _, r := range m.runners { runners = append(runners, r) } return runners, nil } // ListRunnersByTargetID get a not deleted runners that has target_id func (m *Memory) ListRunnersByTargetID(ctx context.Context, targetID uuid.UUID) ([]datastore.Runner, error) { m.mu.Lock() defer m.mu.Unlock() var runners []datastore.Runner for _, r := range m.runners { if uuid.Equal(r.TargetID, targetID) { runners = append(runners, r) } } return runners, nil } // ListRunnersLogBySince ListRunnerLog get a runners since time func (m *Memory) ListRunnersLogBySince(ctx context.Context, since time.Time) ([]datastore.Runner, error) { m.mu.Lock() defer m.mu.Unlock() var runners []datastore.Runner for _, r := range m.runners { if r.CreatedAt.After(since) { runners = append(runners, r) } } return runners, nil } // GetRunner get a runner func (m *Memory) GetRunner(ctx context.Context, id uuid.UUID) (*datastore.Runner, error) { m.mu.Lock() defer m.mu.Unlock() r, ok := m.runners[id] if !ok { return nil, datastore.ErrNotFound } return &r, nil } // DeleteRunner delete a runner func (m *Memory) DeleteRunner(ctx context.Context, id uuid.UUID, deletedAt time.Time, reason datastore.RunnerStatus) error { m.mu.Lock() defer m.mu.Unlock() delete(m.runners, id) return nil } // GetLock get lock func (m *Memory) GetLock(ctx context.Context) error { return nil } // IsLocked return status of lock func (m *Memory) IsLocked(ctx context.Context) (string, error) { return datastore.IsNotLocked, nil } ================================================ FILE: pkg/datastore/mysql/job.go ================================================ package mysql import ( "context" "database/sql" "errors" "fmt" uuid "github.com/satori/go.uuid" "github.com/whywaita/myshoes/pkg/datastore" ) // EnqueueJob add a job func (m *MySQL) EnqueueJob(ctx context.Context, job datastore.Job) error { query := `INSERT INTO jobs(uuid, ghe_domain, repository, check_event, target_id) VALUES (?, ?, ?, ?, ?)` if _, err := m.Conn.ExecContext(ctx, query, job.UUID, job.GHEDomain, job.Repository, job.CheckEventJSON, job.TargetID.String()); err != nil { return fmt.Errorf("failed to execute INSERT query: %w", err) } select { case m.notifyEnqueueCh <- struct{}{}: // notified to starter default: // no capacity on channel, do not block } return nil } // ListJobs get all jobs func (m *MySQL) ListJobs(ctx context.Context) ([]datastore.Job, error) { var jobs []datastore.Job query := `SELECT uuid, ghe_domain, repository, check_event, target_id, created_at, updated_at FROM jobs` if err := m.Conn.SelectContext(ctx, &jobs, query); err != nil { if errors.Is(err, sql.ErrNoRows) { return nil, datastore.ErrNotFound } return nil, fmt.Errorf("failed to execute SELECT query: %w", err) } return jobs, nil } // DeleteJob delete a job func (m *MySQL) DeleteJob(ctx context.Context, id uuid.UUID) error { query := `DELETE FROM jobs WHERE uuid = ?` if _, err := m.Conn.ExecContext(ctx, query, id.String()); err != nil { return fmt.Errorf("failed to execute DELETE query: %w", err) } return nil } ================================================ FILE: pkg/datastore/mysql/job_test.go ================================================ package mysql_test import ( "context" "database/sql" "errors" "fmt" "testing" "time" "github.com/google/go-cmp/cmp" "github.com/jmoiron/sqlx" uuid "github.com/satori/go.uuid" "github.com/whywaita/myshoes/internal/testutils" "github.com/whywaita/myshoes/pkg/datastore" ) var testJobID = uuid.FromStringOrNil("1b4e5b7a-e3c1-4829-9cfd-eac4183f2c95") func TestMySQL_EnqueueJob(t *testing.T) { testDatastore, teardown := testutils.GetTestDatastore() defer teardown() testDB, _ := testutils.GetTestDB() if err := testDatastore.CreateTarget(context.Background(), datastore.Target{ UUID: testTargetID, Scope: testScopeRepo, GHEDomain: sql.NullString{ Valid: false, }, GitHubToken: testGitHubToken, TokenExpiredAt: testTime, ResourceType: datastore.ResourceTypeNano, }); err != nil { t.Fatalf("failed to create target: %+v", err) } tests := []struct { input datastore.Job want *datastore.Job err bool }{ { input: datastore.Job{ UUID: testJobID, Repository: testScopeRepo, CheckEventJSON: `{"example": "json"}`, TargetID: testTargetID, }, want: &datastore.Job{ UUID: testJobID, Repository: testScopeRepo, CheckEventJSON: `{"example": "json"}`, TargetID: testTargetID, }, err: false, }, } for _, test := range tests { err := testDatastore.EnqueueJob(context.Background(), test.input) if !test.err && err != nil { t.Fatalf("failed to enqueue job: %+v", err) } got, err := getJobFromSQL(testDB, test.input.UUID) if err != nil { t.Fatalf("failed to get job from SQL: %+v", err) } if got != nil { got.CreatedAt = time.Time{} got.UpdatedAt = time.Time{} } if diff := cmp.Diff(test.want, got); diff != "" { t.Errorf("mismatch (-want +got):\n%s", diff) } } } func TestMySQL_ListJobs(t *testing.T) { testDatastore, teardown := testutils.GetTestDatastore() defer teardown() if err := testDatastore.CreateTarget(context.Background(), datastore.Target{ UUID: testTargetID, Scope: testScopeRepo, GHEDomain: sql.NullString{ Valid: false, }, GitHubToken: testGitHubToken, TokenExpiredAt: testTime, ResourceType: datastore.ResourceTypeNano, }); err != nil { t.Fatalf("failed to create target: %+v", err) } tests := []struct { input []datastore.Job want []datastore.Job err bool }{ { input: []datastore.Job{ { UUID: testJobID, Repository: testScopeRepo, CheckEventJSON: `{"example": "json"}`, TargetID: testTargetID, }, }, want: []datastore.Job{ { UUID: testJobID, Repository: testScopeRepo, CheckEventJSON: `{"example": "json"}`, TargetID: testTargetID, }, }, err: false, }, } for _, test := range tests { for _, input := range test.input { err := testDatastore.EnqueueJob(context.Background(), input) if !test.err && err != nil { t.Fatalf("failed to enqueue job: %+v", err) } } got, err := testDatastore.ListJobs(context.Background()) if err != nil { t.Fatalf("failed to get jobs: %+v", err) } if len(test.want) != len(got) { t.Fatalf("incorrect length jobs, want: %d but got: %d", len(test.want), len(got)) } for i := range got { got[i].CreatedAt = time.Time{} got[i].UpdatedAt = time.Time{} } if diff := cmp.Diff(test.want, got); diff != "" { t.Errorf("mismatch (-want +got):\n%s", diff) } } } func TestMySQL_DeleteJob(t *testing.T) { testDatastore, teardown := testutils.GetTestDatastore() defer teardown() testDB, _ := testutils.GetTestDB() if err := testDatastore.CreateTarget(context.Background(), datastore.Target{ UUID: testTargetID, Scope: testScopeRepo, GHEDomain: sql.NullString{ Valid: false, }, GitHubToken: testGitHubToken, TokenExpiredAt: testTime, ResourceType: datastore.ResourceTypeNano, }); err != nil { t.Fatalf("failed to create target: %+v", err) } if err := testDatastore.EnqueueJob(context.Background(), datastore.Job{ UUID: testJobID, Repository: testScopeRepo, CheckEventJSON: `{"example": "json"}`, TargetID: testTargetID, }); err != nil { t.Fatalf("failed to enqueue job: %+v", err) } tests := []struct { input uuid.UUID want *datastore.Job err bool }{ { input: testJobID, want: nil, err: false, }, } for _, test := range tests { err := testDatastore.DeleteJob(context.Background(), test.input) if !test.err && err != nil { t.Fatalf("failed to delete job: %+v", err) } got, err := getJobFromSQL(testDB, test.input) if err != nil && !errors.Is(err, sql.ErrNoRows) { t.Fatalf("failed to get job from SQL: %+v", err) } if got != nil { got.CreatedAt = time.Time{} got.UpdatedAt = time.Time{} } if diff := cmp.Diff(test.want, got); diff != "" { t.Errorf("mismatch (-want +got):\n%s", diff) } } } func getJobFromSQL(testDB *sqlx.DB, id uuid.UUID) (*datastore.Job, error) { var j datastore.Job query := `SELECT uuid, ghe_domain, repository, check_event, target_id FROM jobs WHERE uuid = ?` stmt, err := testDB.Preparex(query) if err != nil { return nil, fmt.Errorf("failed to prepare: %w", err) } err = stmt.Get(&j, id) if err != nil { return nil, fmt.Errorf("failed to get job: %w", err) } return &j, nil } ================================================ FILE: pkg/datastore/mysql/lock.go ================================================ package mysql import ( "context" "fmt" "github.com/go-sql-driver/mysql" "github.com/whywaita/myshoes/pkg/config" "github.com/whywaita/myshoes/pkg/datastore" ) // GetLock get lock func (m *MySQL) GetLock(ctx context.Context) error { var res int cfg, err := mysql.ParseDSN(config.Config.MySQLDSN) if err != nil { return fmt.Errorf("failed to parse DSN: %w", err) } lockKey := cfg.DBName query := fmt.Sprintf(`SELECT GET_LOCK('%s', 10)`, lockKey) if err := m.Conn.GetContext(ctx, &res, query); err != nil { return fmt.Errorf("failed to GET_LOCK: %w", err) } return nil } // IsLocked return status of lock func (m *MySQL) IsLocked(ctx context.Context) (string, error) { var res int cfg, err := mysql.ParseDSN(config.Config.MySQLDSN) if err != nil { return "", fmt.Errorf("failed to parse DSN: %w", err) } lockKey := cfg.DBName query := fmt.Sprintf(`SELECT IS_FREE_LOCK('%s')`, lockKey) if err := m.Conn.GetContext(ctx, &res, query); err != nil { return "", fmt.Errorf("failed to IS_FREE_LOCK: %w", err) } switch res { case 1: return datastore.IsNotLocked, nil case 0: return datastore.IsLocked, nil } return "", fmt.Errorf("IS_FREE_LOCK return NULL") } ================================================ FILE: pkg/datastore/mysql/mysql.go ================================================ package mysql import ( "fmt" "time" "github.com/go-sql-driver/mysql" "github.com/jmoiron/sqlx" ) // MySQL is implement datastore in MySQL type MySQL struct { Conn *sqlx.DB notifyEnqueueCh chan<- struct{} } // New create mysql connection func New(dsn string, notifyEnqueueCh chan<- struct{}) (*MySQL, error) { u, err := getMySQLURL(dsn) if err != nil { return nil, fmt.Errorf("failed to get MySQL URL: %w", err) } conn, err := sqlx.Open("mysql", u) if err != nil { return nil, fmt.Errorf("failed to create mysql connection: %w", err) } return &MySQL{ Conn: conn, notifyEnqueueCh: notifyEnqueueCh, }, nil } func getMySQLURL(dsn string) (string, error) { c, err := mysql.ParseDSN(dsn) if err != nil { return "", fmt.Errorf("failed to parse DSN: %w", err) } c.Loc = time.UTC c.ParseTime = true c.Collation = "utf8mb4_general_ci" if c.Params == nil { c.Params = map[string]string{} } c.Params["sql_mode"] = "'TRADITIONAL,NO_AUTO_VALUE_ON_ZERO,ONLY_FULL_GROUP_BY'" c.InterpolateParams = true return c.FormatDSN(), nil } ================================================ FILE: pkg/datastore/mysql/mysql_test.go ================================================ package mysql_test import ( "os" "testing" "github.com/whywaita/myshoes/internal/testutils" ) func TestMain(m *testing.M) { os.Exit(testutils.IntegrationTestRunner(m)) } ================================================ FILE: pkg/datastore/mysql/runner.go ================================================ package mysql import ( "context" "database/sql" "errors" "fmt" "time" uuid "github.com/satori/go.uuid" "github.com/whywaita/myshoes/pkg/datastore" ) // CreateRunner add a runner func (m *MySQL) CreateRunner(ctx context.Context, runner datastore.Runner) error { tx := m.Conn.MustBegin() queryRunner := `INSERT INTO runners(uuid) VALUES (?)` if _, err := tx.ExecContext(ctx, queryRunner, runner.UUID.String()); err != nil { tx.Rollback() return fmt.Errorf("failed to execute INSERT query runners: %w", err) } queryDetail := `INSERT INTO runner_detail(runner_id, shoes_type, ip_address, target_id, cloud_id, resource_type, runner_user, repository_url, request_webhook, provider_url) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` if _, err := tx.ExecContext(ctx, queryDetail, runner.UUID.String(), runner.ShoesType, runner.IPAddress, runner.TargetID.String(), runner.CloudID, runner.ResourceType, runner.RunnerUser, runner.RepositoryURL, runner.RequestWebhook, runner.ProviderURL); err != nil { tx.Rollback() return fmt.Errorf("failed to execute INSERT query runner_detail: %w", err) } queryRunning := `INSERT INTO runners_running(runner_id) VALUES (?)` if _, err := tx.ExecContext(ctx, queryRunning, runner.UUID.String()); err != nil { tx.Rollback() return fmt.Errorf("failed to execute INSERT query runners_running: %w", err) } if err := tx.Commit(); err != nil { tx.Rollback() return fmt.Errorf("failed to execute COMMIT: %w", err) } return nil } // ListRunners get a not deleted runners func (m *MySQL) ListRunners(ctx context.Context) ([]datastore.Runner, error) { var runners []datastore.Runner query := `SELECT runner.runner_id, detail.shoes_type, detail.ip_address, detail.target_id, detail.cloud_id, detail.created_at, detail.updated_at, detail.resource_type, detail.repository_url, detail.request_webhook, detail.runner_user, detail.provider_url FROM runners_running AS runner JOIN runner_detail AS detail ON runner.runner_id = detail.runner_id` err := m.Conn.SelectContext(ctx, &runners, query) if err != nil { if errors.Is(err, sql.ErrNoRows) { return nil, datastore.ErrNotFound } return nil, fmt.Errorf("failed to execute SELECT query: %w", err) } return runners, nil } // ListRunnersByTargetID get a not deleted runners that has target_id func (m *MySQL) ListRunnersByTargetID(ctx context.Context, targetID uuid.UUID) ([]datastore.Runner, error) { var runners []datastore.Runner query := `SELECT runner.runner_id, detail.shoes_type, detail.ip_address, detail.target_id, detail.cloud_id, detail.created_at, detail.updated_at, detail.resource_type, detail.repository_url, detail.request_webhook, detail.runner_user, detail.provider_url FROM runners_running AS runner JOIN runner_detail AS detail ON runner.runner_id = detail.runner_id WHERE detail.target_id = ?` err := m.Conn.SelectContext(ctx, &runners, query, targetID) if err != nil { if errors.Is(err, sql.ErrNoRows) { return nil, datastore.ErrNotFound } return nil, fmt.Errorf("failed to execute SELECT query: %w", err) } return runners, nil } // ListRunnersLogBySince ListRunnerLog get a runners since time func (m *MySQL) ListRunnersLogBySince(ctx context.Context, since time.Time) ([]datastore.Runner, error) { var runners []datastore.Runner query := `SELECT runner_id, shoes_type, ip_address, target_id, cloud_id, created_at, updated_at, resource_type, repository_url, request_webhook, runner_user, provider_url FROM runner_detail WHERE created_at > ?` err := m.Conn.SelectContext(ctx, &runners, query, since) if err != nil { if errors.Is(err, sql.ErrNoRows) { return nil, datastore.ErrNotFound } return nil, fmt.Errorf("failed to execute SELECT query: %w", err) } return runners, nil } // GetRunner get a runner func (m *MySQL) GetRunner(ctx context.Context, id uuid.UUID) (*datastore.Runner, error) { var r datastore.Runner query := `SELECT runner_id, shoes_type, ip_address, target_id, cloud_id, created_at, updated_at, resource_type, repository_url, request_webhook, runner_user, provider_url FROM runner_detail WHERE runner_id = ?` if err := m.Conn.GetContext(ctx, &r, query, id.String()); err != nil { if errors.Is(err, sql.ErrNoRows) { return nil, datastore.ErrNotFound } return nil, fmt.Errorf("failed to execute SELECT query: %w", err) } return &r, nil } // DeleteRunner delete a runner func (m *MySQL) DeleteRunner(ctx context.Context, id uuid.UUID, deletedAt time.Time, reason datastore.RunnerStatus) error { tx := m.Conn.MustBegin() queryDelete := `DELETE FROM runners_running WHERE runner_id = ?` if _, err := tx.ExecContext(ctx, queryDelete, id.String()); err != nil { tx.Rollback() return fmt.Errorf("failed to execute DELETE query: %w", err) } queryInsert := `INSERT INTO runners_deleted(runner_id, reason) VALUES (?, ?)` if _, err := tx.ExecContext(ctx, queryInsert, id.String(), reason); err != nil { tx.Rollback() return fmt.Errorf("failed to execute INSERT query: %w", err) } if err := tx.Commit(); err != nil { tx.Rollback() return fmt.Errorf("failed to execute COMMIT: %w", err) } return nil } ================================================ FILE: pkg/datastore/mysql/runner_test.go ================================================ package mysql_test import ( "context" "database/sql" "errors" "fmt" "testing" "time" "github.com/google/go-cmp/cmp" "github.com/jmoiron/sqlx" uuid "github.com/satori/go.uuid" "github.com/whywaita/myshoes/internal/testutils" "github.com/whywaita/myshoes/pkg/datastore" ) var testRunnerID = uuid.FromStringOrNil("7943e412-c0ae-4068-ab24-3e71a13fbe53") func TestMySQL_CreateRunner(t *testing.T) { testDatastore, teardown := testutils.GetTestDatastore() defer teardown() testDB, _ := testutils.GetTestDB() tests := []struct { input datastore.Runner want *datastore.Runner err bool }{ { input: datastore.Runner{ UUID: testRunnerID, ShoesType: "shoes-test", TargetID: testTargetID, CloudID: "mycloud-uuid", ResourceType: datastore.ResourceTypeNano, RepositoryURL: "https://github.com/octocat/Hello-World", RequestWebhook: "{}", }, want: &datastore.Runner{ UUID: testRunnerID, ShoesType: "shoes-test", TargetID: testTargetID, CloudID: "mycloud-uuid", ResourceType: datastore.ResourceTypeNano, RepositoryURL: "https://github.com/octocat/Hello-World", RequestWebhook: "{}", }, err: false, }, { input: datastore.Runner{ UUID: testRunnerID, ShoesType: "shoes-test", TargetID: testTargetID, CloudID: "mycloud-uuid", ResourceType: datastore.ResourceTypeNano, RunnerUser: sql.NullString{ String: "runner", Valid: true, }, ProviderURL: sql.NullString{ String: "./shoes-test", Valid: true, }, RepositoryURL: "https://github.com/octocat/Hello-World", RequestWebhook: "{}", }, want: &datastore.Runner{ UUID: testRunnerID, ShoesType: "shoes-test", TargetID: testTargetID, CloudID: "mycloud-uuid", ResourceType: datastore.ResourceTypeNano, RunnerUser: sql.NullString{ String: "runner", Valid: true, }, ProviderURL: sql.NullString{ String: "./shoes-test", Valid: true, }, RepositoryURL: "https://github.com/octocat/Hello-World", RequestWebhook: "{}", }, err: false, }, } for _, test := range tests { if err := testDatastore.CreateTarget(context.Background(), datastore.Target{ UUID: testTargetID, Scope: testScopeRepo, GitHubToken: testGitHubToken, TokenExpiredAt: testTime, ResourceType: datastore.ResourceTypeNano, }); err != nil { t.Fatalf("failed to create target: %+v", err) } err := testDatastore.CreateRunner(context.Background(), test.input) if !test.err && err != nil { t.Fatalf("failed to create runner: %+v", err) } got, err := getRunnerFromSQL(testDB, test.input.UUID) if err != nil { t.Fatalf("failed to get runner from SQL: %+v", err) } if got != nil { got.CreatedAt = time.Time{} got.UpdatedAt = time.Time{} } if diff := cmp.Diff(test.want, got); diff != "" { t.Errorf("mismatch (-want +got):\n%s", diff) } teardown() } } func TestMySQL_ListRunners(t *testing.T) { testDatastore, teardown := testutils.GetTestDatastore() defer teardown() if err := testDatastore.CreateTarget(context.Background(), datastore.Target{ UUID: testTargetID, Scope: testScopeRepo, GitHubToken: testGitHubToken, TokenExpiredAt: testTime, ResourceType: datastore.ResourceTypeNano, }); err != nil { t.Fatalf("failed to create target: %+v", err) } tests := []struct { input []datastore.Runner want []datastore.Runner err bool }{ { input: []datastore.Runner{ { UUID: testRunnerID, ShoesType: "shoes-test", TargetID: testTargetID, CloudID: "mycloud-uuid", ResourceType: datastore.ResourceTypeNano, RepositoryURL: "https://github.com/octocat/Hello-World", RequestWebhook: "{}", }, }, want: []datastore.Runner{ { UUID: testRunnerID, ShoesType: "shoes-test", TargetID: testTargetID, CloudID: "mycloud-uuid", ResourceType: datastore.ResourceTypeNano, RepositoryURL: "https://github.com/octocat/Hello-World", RequestWebhook: "{}", }, }, err: false, }, } for _, test := range tests { for _, input := range test.input { err := testDatastore.CreateRunner(context.Background(), input) if !test.err && err != nil { t.Fatalf("failed to create runner: %+v", err) } } got, err := testDatastore.ListRunners(context.Background()) if err != nil { t.Fatalf("failed to get runners: %+v", err) } if len(test.want) != len(got) { t.Fatalf("incorrect length runners, want: %d but got: %d", len(test.want), len(got)) } for i := range got { got[i].CreatedAt = time.Time{} got[i].UpdatedAt = time.Time{} } if diff := cmp.Diff(test.want, got); diff != "" { t.Errorf("mismatch (-want +got):\n%s", diff) } } } func TestMySQL_ListRunnersNotReturnDeleted(t *testing.T) { testDatastore, teardown := testutils.GetTestDatastore() defer teardown() if err := testDatastore.CreateTarget(context.Background(), datastore.Target{ UUID: testTargetID, Scope: testScopeRepo, GitHubToken: testGitHubToken, TokenExpiredAt: testTime, ResourceType: datastore.ResourceTypeNano, }); err != nil { t.Fatalf("failed to create target: %+v", err) } u := "00000000-0000-0000-0000-00000000000%d" for i := 0; i < 3; i++ { input := datastore.Runner{ UUID: testRunnerID, ShoesType: "shoes-test", TargetID: testTargetID, CloudID: "mycloud-uuid", ResourceType: datastore.ResourceTypeNano, RepositoryURL: "https://github.com/octocat/Hello-World", RequestWebhook: "{}", } input.UUID = uuid.FromStringOrNil(fmt.Sprintf(u, i)) err := testDatastore.CreateRunner(context.Background(), input) if err != nil { t.Fatalf("failed to create runner: %+v", err) } } err := testDatastore.DeleteRunner(context.Background(), uuid.FromStringOrNil(fmt.Sprintf(u, 0)), time.Now(), "deleted") if err != nil { t.Fatalf("failed to delete runner: %+v", err) } got, err := testDatastore.ListRunners(context.Background()) if err != nil { t.Fatalf("failed to get runners: %+v", err) } for i := range got { got[i].CreatedAt = time.Time{} got[i].UpdatedAt = time.Time{} } var want []datastore.Runner for i := 1; i < 3; i++ { r := datastore.Runner{ UUID: testRunnerID, ShoesType: "shoes-test", TargetID: testTargetID, CloudID: "mycloud-uuid", ResourceType: datastore.ResourceTypeNano, RepositoryURL: "https://github.com/octocat/Hello-World", RequestWebhook: "{}", } r.UUID = uuid.FromStringOrNil(fmt.Sprintf(u, i)) want = append(want, r) } if diff := cmp.Diff(want, got); diff != "" { t.Errorf("mismatch (-want +got):\n%s", diff) } } func TestMySQL_ListRunnersLogBySince(t *testing.T) { testDatastore, teardown := testutils.GetTestDatastore() defer teardown() if err := testDatastore.CreateTarget(context.Background(), datastore.Target{ UUID: testTargetID, Scope: testScopeRepo, GitHubToken: testGitHubToken, TokenExpiredAt: testTime, ResourceType: datastore.ResourceTypeNano, }); err != nil { t.Fatalf("failed to create target: %+v", err) } u := "00000000-0000-0000-0000-00000000000%d" for i := 1; i < 3; i++ { input := datastore.Runner{ UUID: testRunnerID, ShoesType: "shoes-test", TargetID: testTargetID, CloudID: "mycloud-uuid", ResourceType: datastore.ResourceTypeNano, RepositoryURL: "https://github.com/octocat/Hello-World", RequestWebhook: "{}", } input.UUID = uuid.FromStringOrNil(fmt.Sprintf(u, i)) err := testDatastore.CreateRunner(context.Background(), input) if err != nil { t.Fatalf("failed to create runner: %+v", err) } time.Sleep(500 * time.Millisecond) } recent := time.Now().Add(-10 * time.Second) got, err := testDatastore.ListRunnersLogBySince(context.Background(), recent) if err != nil { t.Fatalf("failed to get runners: %+v", err) } for i := range got { got[i].CreatedAt = time.Time{} got[i].UpdatedAt = time.Time{} } var want []datastore.Runner for i := 1; i < 3; i++ { r := datastore.Runner{ UUID: testRunnerID, ShoesType: "shoes-test", TargetID: testTargetID, CloudID: "mycloud-uuid", ResourceType: datastore.ResourceTypeNano, RepositoryURL: "https://github.com/octocat/Hello-World", RequestWebhook: "{}", } r.UUID = uuid.FromStringOrNil(fmt.Sprintf(u, i)) want = append(want, r) } if diff := cmp.Diff(want, got); diff != "" { t.Errorf("mismatch (-want +got):\n%s", diff) } } func TestMySQL_GetRunner(t *testing.T) { testDatastore, teardown := testutils.GetTestDatastore() defer teardown() if err := testDatastore.CreateTarget(context.Background(), datastore.Target{ UUID: testTargetID, Scope: testScopeRepo, GitHubToken: testGitHubToken, TokenExpiredAt: testTime, ResourceType: datastore.ResourceTypeNano, }); err != nil { t.Fatalf("failed to create target: %+v", err) } if err := testDatastore.CreateRunner(context.Background(), datastore.Runner{ UUID: testRunnerID, ShoesType: "shoes-test", TargetID: testTargetID, CloudID: "mycloud-uuid", ResourceType: datastore.ResourceTypeNano, RepositoryURL: "https://github.com/octocat/Hello-World", RequestWebhook: "{}", }); err != nil { t.Fatalf("failed to create runner: %+v", err) } tests := []struct { input uuid.UUID want *datastore.Runner err bool }{ { input: testRunnerID, want: &datastore.Runner{ UUID: testRunnerID, ShoesType: "shoes-test", TargetID: testTargetID, CloudID: "mycloud-uuid", ResourceType: datastore.ResourceTypeNano, RepositoryURL: "https://github.com/octocat/Hello-World", RequestWebhook: "{}", }, err: false, }, } for _, test := range tests { got, err := testDatastore.GetRunner(context.Background(), test.input) if err != nil { t.Fatalf("failed to get runner: %+v", err) } if got != nil { got.CreatedAt = time.Time{} got.UpdatedAt = time.Time{} } if diff := cmp.Diff(test.want, got); diff != "" { t.Errorf("mismatch (-want +got):\n%s", diff) } } } func TestMySQL_DeleteRunner(t *testing.T) { testDatastore, teardown := testutils.GetTestDatastore() defer teardown() testDB, _ := testutils.GetTestDB() if err := testDatastore.CreateTarget(context.Background(), datastore.Target{ UUID: testTargetID, Scope: testScopeRepo, GitHubToken: testGitHubToken, TokenExpiredAt: testTime, ResourceType: datastore.ResourceTypeNano, }); err != nil { t.Fatalf("failed to create target: %+v", err) } if err := testDatastore.CreateRunner(context.Background(), datastore.Runner{ UUID: testRunnerID, ShoesType: "shoes-test", TargetID: testTargetID, CloudID: "mycloud-uuid", ResourceType: datastore.ResourceTypeNano, RepositoryURL: "https://github.com/octocat/Hello-World", RequestWebhook: "{}", }); err != nil { t.Fatalf("failed to create runner: %+v", err) } deleted := datastore.Runner{ UUID: testRunnerID, ShoesType: "shoes-test", TargetID: testTargetID, CloudID: "mycloud-uuid", ResourceType: datastore.ResourceTypeNano, RepositoryURL: "https://github.com/octocat/Hello-World", RequestWebhook: "{}", } tests := []struct { input uuid.UUID want *datastore.Runner err bool }{ { input: testRunnerID, want: &deleted, err: false, }, } for _, test := range tests { err := testDatastore.DeleteRunner(context.Background(), test.input, time.Now().UTC(), datastore.RunnerStatusCompleted) if !test.err && err != nil { t.Fatalf("failed to create target: %+v", err) } got, err := getRunnerFromSQL(testDB, test.input) if err != nil { t.Fatalf("failed to get target from SQL: %+v", err) } if got != nil { got.CreatedAt = time.Time{} got.UpdatedAt = time.Time{} got.DeletedAt = sql.NullTime{} } if diff := cmp.Diff(test.want, got); diff != "" { t.Errorf("mismatch (-want +got):\n%s", diff) } if _, err := getRunningRunnerFromSQL(testDB, test.input); err == nil || errors.Is(err, sql.ErrNoRows) { t.Errorf("%s is deleted, but exist in runner_running: %+v", test.input, err) } if _, err := getDeletedRunnerFromSQL(testDB, test.input); err != nil { t.Fatalf("%s is not exist in runners_deleted: %+v", test.input, err) } } } func getRunnerFromSQL(testDB *sqlx.DB, id uuid.UUID) (*datastore.Runner, error) { var r datastore.Runner query := `SELECT runner_id, shoes_type, ip_address, target_id, cloud_id, created_at, updated_at, resource_type, repository_url, request_webhook, runner_user, provider_url FROM runner_detail WHERE runner_id = ?` stmt, err := testDB.Preparex(query) if err != nil { return nil, fmt.Errorf("failed to prepare: %w", err) } err = stmt.Get(&r, id) if err != nil { return nil, fmt.Errorf("failed to get runner: %w", err) } return &r, nil } func getRunningRunnerFromSQL(testDB *sqlx.DB, id uuid.UUID) (*datastore.Runner, error) { var r datastore.Runner query := `SELECT detail.runner_id, shoes_type, ip_address, target_id, cloud_id, detail.created_at, updated_at, detail.resource_type, detail.repository_url, detail.request_webhook FROM runner_detail AS detail JOIN runnesr_running AS running ON detail.runner_id = running.runner_id WHERE detail.runner_id = ?` stmt, err := testDB.Preparex(query) if err != nil { return nil, fmt.Errorf("failed to prepare: %w", err) } err = stmt.Get(&r, id) if err != nil { return nil, fmt.Errorf("failed to get runner: %w", err) } return &r, nil } func getDeletedRunnerFromSQL(testDB *sqlx.DB, id uuid.UUID) (*datastore.Runner, error) { var r datastore.Runner query := `SELECT detail.runner_id, shoes_type, ip_address, target_id, cloud_id, detail.created_at, updated_at, detail.resource_type, detail.repository_url, detail.request_webhook FROM runner_detail AS detail JOIN runners_deleted AS deleted ON detail.runner_id = deleted.runner_id WHERE detail.runner_id = ?` stmt, err := testDB.Preparex(query) if err != nil { return nil, fmt.Errorf("failed to prepare: %w", err) } err = stmt.Get(&r, id) if err != nil { return nil, fmt.Errorf("failed to get runner: %w", err) } return &r, nil } ================================================ FILE: pkg/datastore/mysql/schema.sql ================================================ CREATE TABLE `targets` ( `uuid` VARCHAR(36) NOT NULL PRIMARY KEY, `scope` VARCHAR(255) NOT NULL, `ghe_domain` VARCHAR(255), `github_token` VARCHAR(255) NOT NULL, `token_expired_at` TIMESTAMP NOT NULL, `resource_type` ENUM('nano', 'micro', 'small', 'medium', 'large', 'xlarge', '2xlarge', '3xlarge', '4xlarge') NOT NULL, `provider_url` VARCHAR(255), `status` VARCHAR(255) NOT NULL DEFAULT 'active', `status_description` VARCHAR(255), `created_at` TIMESTAMP NOT NULL DEFAULT current_timestamp, `updated_at` TIMESTAMP NOT NULL DEFAULT current_timestamp ON UPDATE current_timestamp, UNIQUE KEY `ghe_domain_scope` (`ghe_domain`, `scope`) ); CREATE TABLE `runners` ( `uuid` VARCHAR(36) NOT NULL PRIMARY KEY, `created_at` TIMESTAMP NOT NULL DEFAULT current_timestamp ); CREATE TABLE `runner_detail` ( `runner_id` VARCHAR(36) NOT NULL, `shoes_type` VARCHAR(255) NOT NULL, `ip_address` VARCHAR(255) NOT NULL, `target_id` VARCHAR(36) NOT NULL, `cloud_id` TEXT NOT NULL, `resource_type` ENUM('nano', 'micro', 'small', 'medium', 'large', 'xlarge', '2xlarge', '3xlarge', '4xlarge') NOT NULL, `runner_user` VARCHAR(255), `provider_url` VARCHAR(255), `repository_url` VARCHAR(255) NOT NULL, `request_webhook` TEXT NOT NULL, `created_at` TIMESTAMP NOT NULL DEFAULT current_timestamp, `updated_at` TIMESTAMP NOT NULL DEFAULT current_timestamp ON UPDATE current_timestamp, KEY `fk_runner_target_id` (`target_id`), CONSTRAINT `runners_ibfk_1` FOREIGN KEY fk_runner_target_id(`target_id`) REFERENCES targets(`uuid`) ON DELETE RESTRICT, KEY `fk_runner_detail_id` (`runner_id`), CONSTRAINT `runners_ibfk_2` FOREIGN KEY fk_runner_detail_id(`runner_id`) REFERENCES runners(`uuid`) ON DELETE RESTRICT ); CREATE TABLE `runners_running` ( `runner_id` VARCHAR(36) NOT NULL, `created_at` TIMESTAMP NOT NULL DEFAULT current_timestamp, KEY `fk_runner_deleted_id` (`runner_id`), CONSTRAINT `runners_running_ibfk_1` FOREIGN KEY fk_runner_deleted_id(`runner_id`) REFERENCES runners(`uuid`) ON DELETE CASCADE ); CREATE TABLE `runners_deleted` ( `runner_id` VARCHAR(36) NOT NULL, `created_at` TIMESTAMP NOT NULL DEFAULT current_timestamp, `reason` VARCHAR(255) NOT NULL, KEY `fk_runner_deleted_id` (`runner_id`), CONSTRAINT `runners_deleted_ibfk_1` FOREIGN KEY fk_runner_deleted_id(`runner_id`) REFERENCES runners(`uuid`) ON DELETE CASCADE ); CREATE TABLE `jobs` ( `uuid` VARCHAR(36) NOT NULL PRIMARY KEY, `ghe_domain` VARCHAR(255), `repository` VARCHAR(255) NOT NULL, `check_event` TEXT NOT NULL, `target_id` VARCHAR(36) NOT NULL, `created_at` TIMESTAMP NOT NULL DEFAULT current_timestamp, `updated_at` TIMESTAMP NOT NULL DEFAULT current_timestamp ON UPDATE current_timestamp, KEY `fk_job_target_id` (`target_id`), CONSTRAINT `jobs_ibfk_1` FOREIGN KEY fk_job_target_id(`target_id`) REFERENCES targets(`uuid`) ON DELETE RESTRICT ); ================================================ FILE: pkg/datastore/mysql/target.go ================================================ package mysql import ( "context" "database/sql" "errors" "fmt" "time" uuid "github.com/satori/go.uuid" "github.com/whywaita/myshoes/pkg/datastore" ) // CreateTarget create a target func (m *MySQL) CreateTarget(ctx context.Context, target datastore.Target) error { expiredAtRFC3339 := target.TokenExpiredAt.Format("2006-01-02 15:04:05") query := `INSERT INTO targets(uuid, scope, ghe_domain, github_token, token_expired_at, resource_type, provider_url) VALUES (?, ?, ?, ?, ?, ?, ?)` if _, err := m.Conn.ExecContext( ctx, query, target.UUID, target.Scope, target.GHEDomain, target.GitHubToken, expiredAtRFC3339, target.ResourceType, target.ProviderURL, ); err != nil { return fmt.Errorf("failed to execute INSERT query: %w", err) } return nil } // GetTarget get a target func (m *MySQL) GetTarget(ctx context.Context, id uuid.UUID) (*datastore.Target, error) { var t datastore.Target query := `SELECT uuid, scope, github_token, token_expired_at, resource_type, provider_url, status, status_description, created_at, updated_at FROM targets WHERE uuid = ?` if err := m.Conn.GetContext(ctx, &t, query, id.String()); err != nil { if errors.Is(err, sql.ErrNoRows) { return nil, datastore.ErrNotFound } return nil, fmt.Errorf("failed to execute SELECT query: %w", err) } return &t, nil } // GetTargetByScope get a target from scope func (m *MySQL) GetTargetByScope(ctx context.Context, scope string) (*datastore.Target, error) { var t datastore.Target query := fmt.Sprintf(`SELECT uuid, scope, github_token, token_expired_at, resource_type, provider_url, status, status_description, created_at, updated_at FROM targets WHERE scope = "%s"`, scope) if err := m.Conn.GetContext(ctx, &t, query); err != nil { if errors.Is(err, sql.ErrNoRows) { return nil, datastore.ErrNotFound } return nil, fmt.Errorf("failed to execute SELECT query: %w", err) } return &t, nil } // ListTargets get a all target func (m *MySQL) ListTargets(ctx context.Context) ([]datastore.Target, error) { var ts []datastore.Target query := `SELECT uuid, scope, github_token, token_expired_at, resource_type, provider_url, status, status_description, created_at, updated_at FROM targets` if err := m.Conn.SelectContext(ctx, &ts, query); err != nil { return nil, fmt.Errorf("failed to SELECT query: %w", err) } return ts, nil } // DeleteTarget delete a target func (m *MySQL) DeleteTarget(ctx context.Context, id uuid.UUID) error { query := `UPDATE targets SET status = "deleted" WHERE uuid = ?` if _, err := m.Conn.ExecContext(ctx, query, id.String()); err != nil { return fmt.Errorf("failed to execute DELETE query: %w", err) } return nil } // UpdateTargetStatus update status in target func (m *MySQL) UpdateTargetStatus(ctx context.Context, targetID uuid.UUID, newStatus datastore.TargetStatus, description string) error { query := `UPDATE targets SET status = ?, status_description = ? WHERE uuid = ?` if _, err := m.Conn.ExecContext(ctx, query, newStatus, description, targetID.String()); err != nil { return fmt.Errorf("failed to execute UPDATE query: %w", err) } return nil } // UpdateToken update token in target func (m *MySQL) UpdateToken(ctx context.Context, targetID uuid.UUID, newToken string, newExpiredAt time.Time) error { query := `UPDATE targets SET github_token = ?, token_expired_at = ? WHERE uuid = ?` if _, err := m.Conn.ExecContext(ctx, query, newToken, newExpiredAt, targetID.String()); err != nil { return fmt.Errorf("failed to execute UPDATE query: %w", err) } return nil } // UpdateTargetParam update parameter of target func (m *MySQL) UpdateTargetParam(ctx context.Context, targetID uuid.UUID, newResourceType datastore.ResourceType, newProviderURL sql.NullString) error { query := `UPDATE targets SET resource_type = ?, provider_url = ? WHERE uuid = ?` if _, err := m.Conn.ExecContext(ctx, query, newResourceType, newProviderURL, targetID.String()); err != nil { return fmt.Errorf("failed to execute UPDATE query: %w", err) } return nil } ================================================ FILE: pkg/datastore/mysql/target_test.go ================================================ package mysql_test import ( "context" "database/sql" "errors" "fmt" "testing" "time" "github.com/google/go-cmp/cmp" "github.com/jmoiron/sqlx" uuid "github.com/satori/go.uuid" "github.com/whywaita/myshoes/internal/testutils" "github.com/whywaita/myshoes/pkg/datastore" ) var testTargetID = uuid.FromStringOrNil("8a72d42c-372c-4e0d-9c6a-4304d44af137") var testTargetID2 = uuid.FromStringOrNil("d14ccfea-b123-4ada-974e-bbff0937e9c7") var testScopeOrg = "octocat" var testScopeRepo = "octocat/hello-world" var testScopeRepo2 = "octocat/hello-world2" var testGitHubToken = "this-code-is-github-token" var testRunnerUser = "testing-super-user" var testProviderURL = "/shoes-mock" var testTime = time.Date(2037, 9, 3, 0, 0, 0, 0, time.UTC) func TestMySQL_CreateTarget(t *testing.T) { testDatastore, teardown := testutils.GetTestDatastore() defer teardown() testDB, _ := testutils.GetTestDB() tests := []struct { input datastore.Target want *datastore.Target err bool }{ { input: datastore.Target{ UUID: testTargetID, Scope: testScopeRepo, GitHubToken: testGitHubToken, TokenExpiredAt: testTime, ResourceType: datastore.ResourceTypeNano, ProviderURL: sql.NullString{ String: testProviderURL, Valid: true, }, }, want: &datastore.Target{ UUID: testTargetID, Scope: testScopeRepo, GitHubToken: testGitHubToken, TokenExpiredAt: testTime, Status: datastore.TargetStatusActive, ResourceType: datastore.ResourceTypeNano, ProviderURL: sql.NullString{ String: testProviderURL, Valid: true, }, }, err: false, }, { input: datastore.Target{ UUID: testTargetID2, Scope: testScopeRepo2, GitHubToken: testGitHubToken, TokenExpiredAt: testTime, GHEDomain: sql.NullString{ String: "https://example.com", Valid: true, }, ResourceType: datastore.ResourceTypeNano, ProviderURL: sql.NullString{ String: testProviderURL, Valid: true, }, }, want: &datastore.Target{ UUID: testTargetID2, Scope: testScopeRepo2, GitHubToken: testGitHubToken, TokenExpiredAt: testTime, GHEDomain: sql.NullString{ String: "https://example.com", Valid: true, }, Status: datastore.TargetStatusActive, ResourceType: datastore.ResourceTypeNano, ProviderURL: sql.NullString{ String: testProviderURL, Valid: true, }, }, err: false, }, } for _, test := range tests { err := testDatastore.CreateTarget(context.Background(), test.input) if !test.err && err != nil { t.Fatalf("failed to create target: %+v", err) } got, err := getTargetFromSQL(testDB, test.input.UUID) if err != nil { t.Fatalf("failed to get target from SQL: %+v", err) } if got != nil { got.CreatedAt = time.Time{} got.UpdatedAt = time.Time{} } if diff := cmp.Diff(test.want, got); diff != "" { t.Errorf("mismatch (-want +got):\n%s", diff) } } } func TestMySQL_GetTarget(t *testing.T) { testDatastore, teardown := testutils.GetTestDatastore() defer teardown() err := testDatastore.CreateTarget(context.Background(), datastore.Target{ UUID: testTargetID, Scope: testScopeRepo, GitHubToken: testGitHubToken, TokenExpiredAt: testTime, ResourceType: datastore.ResourceTypeNano, ProviderURL: sql.NullString{ String: testProviderURL, Valid: true, }, }) if err != nil { t.Fatalf("failed to create target: %+v", err) } tests := []struct { input uuid.UUID want *datastore.Target err bool }{ { input: testTargetID, want: &datastore.Target{ UUID: testTargetID, Scope: testScopeRepo, GitHubToken: testGitHubToken, TokenExpiredAt: testTime, Status: datastore.TargetStatusActive, ResourceType: datastore.ResourceTypeNano, ProviderURL: sql.NullString{ String: testProviderURL, Valid: true, }, }, err: false, }, } for _, test := range tests { got, err := testDatastore.GetTarget(context.Background(), test.input) if err != nil { t.Fatalf("failed to get target: %+v", err) } if got != nil { got.CreatedAt = time.Time{} got.UpdatedAt = time.Time{} } if diff := cmp.Diff(test.want, got); diff != "" { t.Errorf("mismatch (-want +got):\n%s", diff) } } } func TestMySQL_GetTargetByScope(t *testing.T) { testDatastore, teardown := testutils.GetTestDatastore() defer teardown() tests := []struct { input string want *datastore.Target prepare func() error err bool }{ { // create single instance input: testScopeRepo, want: &datastore.Target{ UUID: testTargetID, Scope: testScopeRepo, GitHubToken: testGitHubToken, TokenExpiredAt: testTime, Status: datastore.TargetStatusActive, ResourceType: datastore.ResourceTypeNano, ProviderURL: sql.NullString{ String: testProviderURL, Valid: true, }, }, prepare: func() error { return testDatastore.CreateTarget(context.Background(), datastore.Target{ UUID: testTargetID, Scope: testScopeRepo, GitHubToken: testGitHubToken, TokenExpiredAt: testTime, ResourceType: datastore.ResourceTypeNano, ProviderURL: sql.NullString{ String: testProviderURL, Valid: true, }, }) }, err: false, }, { // repository is active and organization is deleted, correct return repository input: testScopeRepo, want: &datastore.Target{ UUID: testTargetID, Scope: testScopeRepo, GitHubToken: testGitHubToken, TokenExpiredAt: testTime, Status: datastore.TargetStatusActive, ResourceType: datastore.ResourceTypeNano, ProviderURL: sql.NullString{ String: testProviderURL, Valid: true, }, }, prepare: func() error { if err := testDatastore.CreateTarget(context.Background(), datastore.Target{ UUID: testTargetID, Scope: testScopeRepo, GitHubToken: testGitHubToken, TokenExpiredAt: testTime, ResourceType: datastore.ResourceTypeNano, ProviderURL: sql.NullString{ String: testProviderURL, Valid: true, }, }); err != nil { return fmt.Errorf("failed to create repository: %w", err) } if err := testDatastore.CreateTarget(context.Background(), datastore.Target{ UUID: testTargetID2, Scope: testScopeOrg, GitHubToken: testGitHubToken, TokenExpiredAt: testTime, ResourceType: datastore.ResourceTypeNano, ProviderURL: sql.NullString{ String: testProviderURL, Valid: true, }, }); err != nil { return fmt.Errorf("failed to create organization (will delete): %w", err) } if err := testDatastore.DeleteTarget(context.Background(), testTargetID2); err != nil { return fmt.Errorf("failed to delete organization: %w", err) } return nil }, err: false, }, { // repository is deleted and organization is active, correct return organization input: testScopeOrg, want: &datastore.Target{ UUID: testTargetID2, Scope: testScopeOrg, GitHubToken: testGitHubToken, TokenExpiredAt: testTime, Status: datastore.TargetStatusActive, ResourceType: datastore.ResourceTypeNano, ProviderURL: sql.NullString{ String: testProviderURL, Valid: true, }, }, prepare: func() error { if err := testDatastore.CreateTarget(context.Background(), datastore.Target{ UUID: testTargetID, Scope: testScopeRepo, GitHubToken: testGitHubToken, TokenExpiredAt: testTime, ResourceType: datastore.ResourceTypeNano, ProviderURL: sql.NullString{ String: testProviderURL, Valid: true, }, }); err != nil { return fmt.Errorf("failed to create repository (will delete): %w", err) } if err := testDatastore.DeleteTarget(context.Background(), testTargetID); err != nil { return fmt.Errorf("failed to delete repository: %w", err) } if err := testDatastore.CreateTarget(context.Background(), datastore.Target{ UUID: testTargetID2, Scope: testScopeOrg, GitHubToken: testGitHubToken, TokenExpiredAt: testTime, ResourceType: datastore.ResourceTypeNano, ProviderURL: sql.NullString{ String: testProviderURL, Valid: true, }, }); err != nil { return fmt.Errorf("failed to create deleted organization: %w", err) } return nil }, err: false, }, } for _, test := range tests { if err := test.prepare(); err != nil { t.Fatalf("failed to prepare function: %+v", err) } got, err := testDatastore.GetTargetByScope(context.Background(), test.input) if err != nil { t.Fatalf("failed to get target: %+v", err) } if got != nil { got.CreatedAt = time.Time{} got.UpdatedAt = time.Time{} } if diff := cmp.Diff(test.want, got); diff != "" { t.Errorf("mismatch (-want +got):\n%s", diff) } teardown() } } func TestMySQL_ListTargets(t *testing.T) { testDatastore, teardown := testutils.GetTestDatastore() defer teardown() if err := testDatastore.CreateTarget(context.Background(), datastore.Target{ UUID: testTargetID, Scope: testScopeRepo, GitHubToken: testGitHubToken, TokenExpiredAt: testTime, ResourceType: datastore.ResourceTypeNano, ProviderURL: sql.NullString{ String: testProviderURL, Valid: true, }, }); err != nil { t.Fatalf("failed to create target: %+v", err) } tests := []struct { input interface{} want []datastore.Target err bool }{ { input: nil, want: []datastore.Target{ { UUID: testTargetID, Scope: testScopeRepo, GitHubToken: testGitHubToken, TokenExpiredAt: testTime, Status: datastore.TargetStatusActive, ResourceType: datastore.ResourceTypeNano, ProviderURL: sql.NullString{ String: testProviderURL, Valid: true, }, }, }, err: false, }, } for _, test := range tests { got, err := testDatastore.ListTargets(context.Background()) if !test.err && err != nil { t.Fatalf("failed to list targets: %+v", err) } for i := range got { got[i].CreatedAt = time.Time{} got[i].UpdatedAt = time.Time{} } if diff := cmp.Diff(test.want, got); diff != "" { t.Errorf("mismatch (-want +got):\n%s", diff) } } } func TestMySQL_DeleteTarget(t *testing.T) { testDatastore, teardown := testutils.GetTestDatastore() defer teardown() testDB, _ := testutils.GetTestDB() if err := testDatastore.CreateTarget(context.Background(), datastore.Target{ UUID: testTargetID, Scope: testScopeRepo, GitHubToken: testGitHubToken, TokenExpiredAt: testTime, ResourceType: datastore.ResourceTypeNano, ProviderURL: sql.NullString{ String: testProviderURL, Valid: true, }, }); err != nil { t.Fatalf("failed to create target: %+v", err) } tests := []struct { input uuid.UUID want *datastore.Target err bool }{ { input: testTargetID, want: &datastore.Target{ UUID: testTargetID, Scope: testScopeRepo, GitHubToken: testGitHubToken, TokenExpiredAt: testTime, ResourceType: datastore.ResourceTypeNano, ProviderURL: sql.NullString{ String: testProviderURL, Valid: true, }, Status: datastore.TargetStatusDeleted, }, err: false, }, } for _, test := range tests { err := testDatastore.DeleteTarget(context.Background(), test.input) if !test.err && err != nil { t.Fatalf("failed to delete target: %+v", err) } got, err := getTargetFromSQL(testDB, test.input) if err != nil && !errors.Is(err, sql.ErrNoRows) { t.Fatalf("failed to get target from SQL: %+v", err) } if got != nil { got.CreatedAt = time.Time{} got.UpdatedAt = time.Time{} } if diff := cmp.Diff(test.want, got); diff != "" { t.Errorf("mismatch (-want +got):\n%s", diff) } } } func TestMySQL_UpdateStatus(t *testing.T) { testDatastore, teardown := testutils.GetTestDatastore() defer teardown() testDB, _ := testutils.GetTestDB() type Input struct { status datastore.TargetStatus description string } tests := []struct { input Input want *datastore.Target err bool }{ { input: Input{ status: datastore.TargetStatusActive, description: "", }, want: &datastore.Target{ Scope: testScopeRepo, GitHubToken: testGitHubToken, TokenExpiredAt: testTime, ResourceType: datastore.ResourceTypeNano, ProviderURL: sql.NullString{ String: testProviderURL, Valid: true, }, Status: datastore.TargetStatusActive, StatusDescription: sql.NullString{ String: "", Valid: true, }, }, err: false, }, { input: Input{ status: datastore.TargetStatusRunning, description: "job-id", }, want: &datastore.Target{ Scope: testScopeRepo, GitHubToken: testGitHubToken, TokenExpiredAt: testTime, ResourceType: datastore.ResourceTypeNano, ProviderURL: sql.NullString{ String: testProviderURL, Valid: true, }, Status: datastore.TargetStatusRunning, StatusDescription: sql.NullString{ String: "job-id", Valid: true, }, }, err: false, }, } for _, test := range tests { tID := uuid.NewV4() if err := testDatastore.CreateTarget(context.Background(), datastore.Target{ UUID: tID, Scope: testScopeRepo, GitHubToken: testGitHubToken, TokenExpiredAt: testTime, ResourceType: datastore.ResourceTypeNano, ProviderURL: sql.NullString{ String: testProviderURL, Valid: true, }, }); err != nil { t.Fatalf("failed to create target: %+v", err) } //lint:ignore SA1019 only execute in test err := testDatastore.UpdateTargetStatus(context.Background(), tID, test.input.status, test.input.description) if !test.err && err != nil { t.Fatalf("failed to update status: %+v", err) } got, err := getTargetFromSQL(testDB, tID) if err != nil && !errors.Is(err, sql.ErrNoRows) { t.Fatalf("failed to get target from SQL: %+v", err) } if got != nil { got.UUID = uuid.UUID{} got.CreatedAt = time.Time{} got.UpdatedAt = time.Time{} } if diff := cmp.Diff(test.want, got); diff != "" { t.Errorf("mismatch (-want +got):\n%s", diff) } if err := testDatastore.DeleteTarget(context.Background(), tID); err != nil { t.Fatalf("failed to delete target: %+v", err) } } } func TestMySQL_UpdateToken(t *testing.T) { testDatastore, teardown := testutils.GetTestDatastore() defer teardown() testDB, _ := testutils.GetTestDB() type Input struct { token string expired time.Time } tests := []struct { input Input want *datastore.Target err bool }{ { input: Input{ token: "new-token", expired: testTime.Add(1 * time.Hour), }, want: &datastore.Target{ Scope: testScopeRepo, GitHubToken: "new-token", TokenExpiredAt: testTime.Add(1 * time.Hour), ResourceType: datastore.ResourceTypeNano, ProviderURL: sql.NullString{ String: testProviderURL, Valid: true, }, Status: datastore.TargetStatusActive, StatusDescription: sql.NullString{ String: "", Valid: false, }, }, err: false, }, } for _, test := range tests { tID := uuid.NewV4() if err := testDatastore.CreateTarget(context.Background(), datastore.Target{ UUID: tID, Scope: testScopeRepo, GitHubToken: testGitHubToken, TokenExpiredAt: testTime, ResourceType: datastore.ResourceTypeNano, ProviderURL: sql.NullString{ String: testProviderURL, Valid: true, }, }); err != nil { t.Fatalf("failed to create target: %+v", err) } err := testDatastore.UpdateToken(context.Background(), tID, test.input.token, test.input.expired) if !test.err && err != nil { t.Fatalf("failed to update status: %+v", err) } got, err := getTargetFromSQL(testDB, tID) if err != nil && !errors.Is(err, sql.ErrNoRows) { t.Fatalf("failed to get target from SQL: %+v", err) } if got != nil { got.UUID = uuid.UUID{} got.CreatedAt = time.Time{} got.UpdatedAt = time.Time{} } if diff := cmp.Diff(test.want, got); diff != "" { t.Errorf("mismatch (-want +got):\n%s", diff) } if err := testDatastore.DeleteTarget(context.Background(), tID); err != nil { t.Fatalf("failed to delete target: %+v", err) } } } func TestMySQL_UpdateTargetParam(t *testing.T) { testDatastore, teardown := testutils.GetTestDatastore() defer teardown() testDB, _ := testutils.GetTestDB() type input struct { resourceType datastore.ResourceType runnerUser sql.NullString providerURL sql.NullString } tests := []struct { input input want *datastore.Target err bool }{ { input: input{ resourceType: datastore.ResourceTypeLarge, runnerUser: sql.NullString{ String: "", Valid: false, }, providerURL: sql.NullString{ String: "", Valid: false, }, }, want: &datastore.Target{ Scope: testScopeRepo, GitHubToken: testGitHubToken, ResourceType: datastore.ResourceTypeLarge, ProviderURL: sql.NullString{ String: "", Valid: false, }, Status: datastore.TargetStatusActive, StatusDescription: sql.NullString{ String: "", Valid: false, }, }, err: false, }, { input: input{ resourceType: datastore.ResourceTypeLarge, runnerUser: sql.NullString{ String: testRunnerUser, Valid: true, }, providerURL: sql.NullString{ String: testProviderURL, Valid: true, }, }, want: &datastore.Target{ Scope: testScopeRepo, GitHubToken: testGitHubToken, ResourceType: datastore.ResourceTypeLarge, ProviderURL: sql.NullString{ String: testProviderURL, Valid: true, }, Status: datastore.TargetStatusActive, StatusDescription: sql.NullString{ String: "", Valid: false, }, }, err: false, }, { input: input{ resourceType: datastore.ResourceTypeLarge, runnerUser: sql.NullString{ String: testRunnerUser, Valid: true, }, providerURL: sql.NullString{ String: "", Valid: false, }, }, want: &datastore.Target{ Scope: testScopeRepo, GitHubToken: testGitHubToken, ResourceType: datastore.ResourceTypeLarge, ProviderURL: sql.NullString{ String: "", Valid: false, }, Status: datastore.TargetStatusActive, StatusDescription: sql.NullString{ String: "", Valid: false, }, }, err: false, }, } for _, test := range tests { tID := uuid.NewV4() if err := testDatastore.CreateTarget(context.Background(), datastore.Target{ UUID: tID, Scope: testScopeRepo, GitHubToken: testGitHubToken, TokenExpiredAt: testTime, ResourceType: datastore.ResourceTypeNano, ProviderURL: sql.NullString{ String: "test-default-string", Valid: true, }, }); err != nil { t.Fatalf("failed to create target: %+v", err) } if err := testDatastore.UpdateTargetParam(context.Background(), tID, test.input.resourceType, test.input.providerURL); err != nil { t.Fatalf("failed to UpdateResourceTyoe: %+v", err) } got, err := getTargetFromSQL(testDB, tID) if err != nil && !errors.Is(err, sql.ErrNoRows) { t.Fatalf("failed to get target from SQL: %+v", err) } if got != nil { got.UUID = uuid.UUID{} got.CreatedAt = time.Time{} got.UpdatedAt = time.Time{} got.TokenExpiredAt = time.Time{} } if diff := cmp.Diff(test.want, got); diff != "" { t.Errorf("mismatch (-want +got):\n%s", diff) } if err := testDatastore.DeleteTarget(context.Background(), tID); err != nil { t.Fatalf("failed to delete target: %+v", err) } } } func getTargetFromSQL(testDB *sqlx.DB, uuid uuid.UUID) (*datastore.Target, error) { var t datastore.Target query := `SELECT uuid, scope, ghe_domain, github_token, token_expired_at, resource_type, provider_url, status, status_description, created_at, updated_at FROM targets WHERE uuid = ?` stmt, err := testDB.Preparex(query) if err != nil { return nil, fmt.Errorf("failed to prepare: %w", err) } err = stmt.Get(&t, uuid) if err != nil { return nil, fmt.Errorf("failed to get target: %w", err) } return &t, nil } ================================================ FILE: pkg/datastore/resource_type.go ================================================ package datastore import ( "database/sql/driver" "encoding/json" "fmt" pb "github.com/whywaita/myshoes/api/proto.go" ) // ResourceType is runner machine spec type ResourceType int // ResourceTypes variables const ( ResourceTypeUnknown ResourceType = iota ResourceTypeNano ResourceTypeMicro ResourceTypeSmall ResourceTypeMedium ResourceTypeLarge ResourceTypeXLarge ResourceType2XLarge ResourceType3XLarge ResourceType4XLarge ) // String implement interface for fmt.Stringer func (r ResourceType) String() string { switch r { case ResourceTypeNano: return "nano" case ResourceTypeMicro: return "micro" case ResourceTypeSmall: return "small" case ResourceTypeMedium: return "medium" case ResourceTypeLarge: return "large" case ResourceTypeXLarge: return "xlarge" case ResourceType2XLarge: return "2xlarge" case ResourceType3XLarge: return "3xlarge" case ResourceType4XLarge: return "4xlarge" } return "unknown" } // Value implements the database/sql/driver Valuer interface func (r ResourceType) Value() (driver.Value, error) { return driver.Value(r.String()), nil } // Scan implements the database/sql Scanner interface func (r *ResourceType) Scan(src interface{}) error { var rt *ResourceType switch src := src.(type) { case string: unmarshaled := UnmarshalResourceType(src) rt = &unmarshaled case []uint8: str := string(src) unmarshaled := UnmarshalResourceType(str) rt = &unmarshaled default: return fmt.Errorf("incompatible type for ResourceType: %T", src) } *r = *rt return nil } // UnmarshalResourceType cast type to ResourceType func UnmarshalResourceType(src interface{}) ResourceType { switch src := src.(type) { case string: return UnmarshalResourceTypeString(src) case pb.ResourceType: return UnmarshalResourceTypePb(src) } return ResourceTypeUnknown } // UnmarshalResourceTypeString cast type from string to ResourceType func UnmarshalResourceTypeString(in string) ResourceType { switch in { case "nano": return ResourceTypeNano case "micro": return ResourceTypeMicro case "small": return ResourceTypeSmall case "medium": return ResourceTypeMedium case "large": return ResourceTypeLarge case "xlarge": return ResourceTypeXLarge case "2xlarge": return ResourceType2XLarge case "3xlarge": return ResourceType3XLarge case "4xlarge": return ResourceType4XLarge } return ResourceTypeUnknown } // UnmarshalResourceTypePb cast type from pb.ResourceType to ResourceType func UnmarshalResourceTypePb(in pb.ResourceType) ResourceType { switch in { case pb.ResourceType_Nano: return ResourceTypeNano case pb.ResourceType_Micro: return ResourceTypeMicro case pb.ResourceType_Small: return ResourceTypeSmall case pb.ResourceType_Medium: return ResourceTypeMedium case pb.ResourceType_Large: return ResourceTypeLarge case pb.ResourceType_XLarge: return ResourceTypeXLarge case pb.ResourceType_XLarge2: return ResourceType2XLarge case pb.ResourceType_XLarge3: return ResourceType3XLarge case pb.ResourceType_XLarge4: return ResourceType4XLarge } return ResourceTypeUnknown } // ToPb convert type of protobuf func (r ResourceType) ToPb() pb.ResourceType { switch r { case ResourceTypeNano: return pb.ResourceType_Nano case ResourceTypeMicro: return pb.ResourceType_Micro case ResourceTypeSmall: return pb.ResourceType_Small case ResourceTypeMedium: return pb.ResourceType_Medium case ResourceTypeLarge: return pb.ResourceType_Large case ResourceTypeXLarge: return pb.ResourceType_XLarge case ResourceType2XLarge: return pb.ResourceType_XLarge2 case ResourceType3XLarge: return pb.ResourceType_XLarge3 case ResourceType4XLarge: return pb.ResourceType_XLarge4 } return pb.ResourceType_Unknown } // MarshalJSON implements the encoding/json Marshaler interface func (r ResourceType) MarshalJSON() ([]byte, error) { return json.Marshal(r.String()) } // UnmarshalJSON implements the encoding/json Unmarshaler interface func (r *ResourceType) UnmarshalJSON(data []byte) error { var s string if err := json.Unmarshal(data, &s); err != nil { return fmt.Errorf("data should be a string, but got %s", data) } rt := UnmarshalResourceTypeString(s) *r = rt return nil } ================================================ FILE: pkg/docker/ratelimit.go ================================================ package docker import ( "encoding/json" "fmt" "io" "net/http" "strconv" "strings" "time" "github.com/golang-jwt/jwt/v4" "github.com/whywaita/myshoes/pkg/config" ) // RateLimit is Docker Hub API rate limit type RateLimit struct { Limit int Remaining int } type tokenCache struct { expire time.Time token string } var cacheMap = make(map[int]tokenCache, 1) func getToken() (string, error) { url := "https://auth.docker.io/token?service=registry.docker.io&scope=repository:ratelimitpreview/test:pull" req, err := http.NewRequest("GET", url, nil) if err != nil { return "", fmt.Errorf("create request: %w", err) } if config.Config.DockerHubCredential.Password != "" && config.Config.DockerHubCredential.Username != "" { req.SetBasicAuth(config.Config.DockerHubCredential.Username, config.Config.DockerHubCredential.Password) } resp, err := http.DefaultClient.Do(req) if err != nil { return "", fmt.Errorf("request token: %w", err) } if cache, ok := cacheMap[0]; ok && cache.expire.After(time.Now()) { return cache.token, nil } defer resp.Body.Close() byteArray, err := io.ReadAll(resp.Body) if err != nil { return "", fmt.Errorf("read body: %w", err) } jsonMap := make(map[string]interface{}) if err := json.Unmarshal(byteArray, &jsonMap); err != nil { return "", fmt.Errorf("unmarshal json: %w", err) } tokenString, ok := jsonMap["token"].(string) if !ok { return "", fmt.Errorf("tokenString is not string") } token, _, err := new(jwt.Parser).ParseUnverified(tokenString, jwt.MapClaims{}) if err != nil { return "", fmt.Errorf("parse token: %w", err) } exp, ok := token.Claims.(jwt.MapClaims)["exp"].(float64) if !ok { return "", fmt.Errorf("exp is not float64") } cacheMap[0] = tokenCache{ expire: time.Unix(int64(exp), 0), token: tokenString, } return tokenString, nil } // GetRateLimit get Docker Hub API rate limit func GetRateLimit() (RateLimit, error) { token, err := getToken() if err != nil { return RateLimit{}, fmt.Errorf("get token: %w", err) } url := "https://registry-1.docker.io/v2/ratelimitpreview/test/manifests/latest" req, err := http.NewRequest("HEAD", url, nil) if err != nil { return RateLimit{}, fmt.Errorf("create request: %w", err) } req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token)) resp, err := http.DefaultClient.Do(req) if err != nil { return RateLimit{}, fmt.Errorf("get rate limit: %w", err) } defer resp.Body.Close() limitHeader := resp.Header.Get("ratelimit-limit") if limitHeader == "" { return RateLimit{}, fmt.Errorf("not found ratelimit-limit header") } limit, err := strconv.Atoi(strings.Split(limitHeader, ";")[0]) if err != nil { return RateLimit{}, fmt.Errorf("parse limit: %w", err) } remainingHeader := resp.Header.Get("ratelimit-remaining") if remainingHeader == "" { return RateLimit{}, fmt.Errorf("not found ratelimit-remaining header") } remaining, err := strconv.Atoi(strings.Split(remainingHeader, ";")[0]) if err != nil { return RateLimit{}, fmt.Errorf("parse remaining: %w", err) } return RateLimit{ Limit: limit, Remaining: remaining, }, nil } ================================================ FILE: pkg/gh/github.go ================================================ package gh import ( "fmt" "net/http" "net/url" "path" "sync" "time" "github.com/bradleyfalzon/ghinstallation/v2" "github.com/google/go-github/v80/github" "github.com/m4ns0ur/httpcache" "github.com/patrickmn/go-cache" "github.com/whywaita/myshoes/pkg/config" "golang.org/x/oauth2" ) var ( // ErrNotFound is error for not found ErrNotFound = fmt.Errorf("not found") // ResponseCache is cache variable responseCache = cache.New(5*time.Minute, 10*time.Minute) // rateLimitRemain is remaining of Rate limit, for metrics rateLimitRemain = sync.Map{} // rateLimitLimit is limit of Rate limit, for metrics rateLimitLimit = sync.Map{} // httpCache is shareable response cache httpCache = httpcache.NewMemoryCache() // appTransport is transport for GitHub Apps appTransport = ghinstallation.AppsTransport{} // installationTransports is map of ghinstallation.Transport for cache token of installation. // key: installationID, value: ghinstallation.Transport installationTransports = sync.Map{} ) // InitializeCache create a cache func InitializeCache(appID int64, appPEM []byte) error { tr := httpcache.NewTransport(httpCache) itr, err := ghinstallation.NewAppsTransport(tr, appID, appPEM) if err != nil { return fmt.Errorf("failed to create Apps transport: %w", err) } appTransport = *itr return nil } // NewClient create a client of GitHub func NewClient(token string) (*github.Client, error) { oauth2Transport := &oauth2.Transport{ Source: oauth2.StaticTokenSource( &oauth2.Token{AccessToken: token}, ), } transport := &httpcache.Transport{ Transport: oauth2Transport, Cache: httpCache, MarkCachedResponses: true, } clientTransport := newInstrumentedTransport(transport) if !config.Config.IsGHES() { return github.NewClient(&http.Client{Transport: clientTransport}), nil } return github.NewClient(&http.Client{Transport: clientTransport}).WithEnterpriseURLs(config.Config.GitHubURL, config.Config.GitHubURL) } // NewClientGitHubApps create a client of GitHub using Private Key from GitHub Apps // header is "Authorization: Bearer YOUR_JWT" // docs: https://docs.github.com/en/developers/apps/building-github-apps/authenticating-with-github-apps#authenticating-as-a-github-app func NewClientGitHubApps() (*github.Client, error) { if !config.Config.IsGHES() { return github.NewClient(&http.Client{Transport: newInstrumentedTransport(&appTransport)}), nil } apiEndpoint, err := getAPIEndpoint() if err != nil { return nil, fmt.Errorf("failed to get GitHub API Endpoint: %w", err) } itr := appTransport itr.BaseURL = apiEndpoint.String() return github.NewClient(&http.Client{Transport: newInstrumentedTransport(&appTransport)}).WithEnterpriseURLs(config.Config.GitHubURL, config.Config.GitHubURL) } // NewClientInstallation create a client of GitHub using installation ID from GitHub Apps // header is "Authorization: token YOUR_INSTALLATION_ACCESS_TOKEN" // docs: https://docs.github.com/en/developers/apps/building-github-apps/authenticating-with-github-apps#authenticating-as-an-installation func NewClientInstallation(installationID int64) (*github.Client, error) { itr := getInstallationTransport(installationID) if !config.Config.IsGHES() { return github.NewClient(&http.Client{Transport: newInstrumentedTransport(itr)}), nil } apiEndpoint, err := getAPIEndpoint() if err != nil { return nil, fmt.Errorf("failed to get GitHub API Endpoint: %w", err) } itr.BaseURL = apiEndpoint.String() return github.NewClient(&http.Client{Transport: newInstrumentedTransport(itr)}).WithEnterpriseURLs(config.Config.GitHubURL, config.Config.GitHubURL) } func setInstallationTransport(installationID int64, itr ghinstallation.Transport) { installationTransports.Store(installationID, itr) } func getInstallationTransport(installationID int64) *ghinstallation.Transport { got, found := installationTransports.Load(installationID) if !found { return generateInstallationTransport(installationID) } itr, ok := got.(ghinstallation.Transport) if !ok { return generateInstallationTransport(installationID) } return &itr } func generateInstallationTransport(installationID int64) *ghinstallation.Transport { itr := ghinstallation.NewFromAppsTransport(&appTransport, installationID) setInstallationTransport(installationID, *itr) return itr } // CheckSignature check trust installation id from event. func CheckSignature(installationID int64) error { if itr := ghinstallation.NewFromAppsTransport(&appTransport, installationID); itr == nil { return fmt.Errorf("failed to create GitHub installation") } return nil } // ExistRunnerReleases check exist of runner file func ExistRunnerReleases(runnerVersion string) error { releasesURL := fmt.Sprintf("https://github.com/actions/runner/releases/tag/%s", runnerVersion) resp, err := http.Get(releasesURL) if err != nil { return fmt.Errorf("failed to GET from %s: %w", releasesURL, ErrNotFound) } if resp.StatusCode == http.StatusOK { return nil } else if resp.StatusCode == http.StatusNotFound { return ErrNotFound } return fmt.Errorf("invalid response code (%d)", resp.StatusCode) } // ExistGitHubRepository check exist of GitHub repository func ExistGitHubRepository(scope string, accessToken string) error { repoURL, err := getRepositoryURL(scope) if err != nil { return fmt.Errorf("failed to get repository url: %w", err) } client := &http.Client{Transport: newInstrumentedTransport(http.DefaultTransport)} req, err := http.NewRequest(http.MethodGet, repoURL, nil) if err != nil { return fmt.Errorf("failed to create request: %w", err) } req.Header.Add("Authorization", fmt.Sprintf("token %s", accessToken)) resp, err := client.Do(req) if err != nil { return fmt.Errorf("failed to do request: %w", err) } if resp.StatusCode == http.StatusOK { return nil } else if resp.StatusCode == http.StatusNotFound { return ErrNotFound } return fmt.Errorf("invalid response code (%d)", resp.StatusCode) } func getRepositoryURL(scope string) (string, error) { // github.com // => https://api.github.com/repos/:owner/:repo // => https://api.github.com/orgs/:owner // GitHub Enterprise Server // => https://{your_ghe_server_url}/api/repos/:owner/:repo // => https://{your_ghe_server_url}/api/orgs/:owner s := DetectScope(scope) if s == Unknown { return "", fmt.Errorf("failed to detect valid scope") } apiEndpoint, err := getAPIEndpoint() if err != nil { return "", fmt.Errorf("failed to get API Endpoint: %w", err) } p := path.Join(apiEndpoint.Path, s.String(), scope) apiEndpoint.Path = p return apiEndpoint.String(), nil } func getAPIEndpoint() (*url.URL, error) { var apiEndpoint *url.URL if config.Config.IsGHES() { u, err := url.Parse(config.Config.GitHubURL) if err != nil { return nil, fmt.Errorf("failed to parse GHE url: %w", err) } p := u.Path p = path.Join(p, "api", "v3") u.Path = p apiEndpoint = u } else { u, err := url.Parse("https://api.github.com") if err != nil { return nil, fmt.Errorf("failed to parse github.com: %w", err) } apiEndpoint = u } return apiEndpoint, nil } ================================================ FILE: pkg/gh/github_test.go ================================================ package gh import ( "os" "testing" "github.com/whywaita/myshoes/pkg/config" ) func TestDetectScope(t *testing.T) { tests := []struct { input string want Scope }{ { input: "org/repo", want: Repository, }, { input: "org", want: Organization, }, { input: "org/repo/whats", want: Unknown, }, { input: "https://github.com/octocat/Spoon-Knife", want: Unknown, }, } for _, test := range tests { got := DetectScope(test.input) if got != test.want { t.Fatalf("want %+v, but got %+v", test.want, got) } } } type TestGetRepositoryURLInput struct { scope string gheDomain string } func TestGetRepositoryURL(t *testing.T) { tests := []struct { input TestGetRepositoryURLInput want string err error }{ { input: TestGetRepositoryURLInput{ scope: "org/repo", gheDomain: "", }, want: "https://api.github.com/repos/org/repo", err: nil, }, { input: TestGetRepositoryURLInput{ scope: "org", gheDomain: "", }, want: "https://api.github.com/orgs/org", err: nil, }, { input: TestGetRepositoryURLInput{ scope: "org/repo", gheDomain: "https://github-enterprise.example.com", }, want: "https://github-enterprise.example.com/api/v3/repos/org/repo", err: nil, }, { input: TestGetRepositoryURLInput{ scope: "org", gheDomain: "https://github-enterprise.example.com", }, want: "https://github-enterprise.example.com/api/v3/orgs/org", err: nil, }, { input: TestGetRepositoryURLInput{ scope: "org/repo", gheDomain: "https://github-enterprise.example.com/", }, want: "https://github-enterprise.example.com/api/v3/repos/org/repo", err: nil, }, { input: TestGetRepositoryURLInput{ scope: "org/repo", gheDomain: "https://github-enterprise.example.com/github", }, want: "https://github-enterprise.example.com/github/api/v3/repos/org/repo", err: nil, }, } for _, test := range tests { f := func() { if test.input.gheDomain != "" { t.Setenv("GITHUB_URL", test.input.gheDomain) defer os.Unsetenv("GITHUB_URL") } config.LoadWithDefault() got, err := getRepositoryURL(test.input.scope) if err != test.err { t.Fatalf("getRepositoryURL want err %+v, but return err %+v", test.err, err) } if got != test.want { t.Fatalf("want %s, but got %s", test.want, got) } } f() } } ================================================ FILE: pkg/gh/installation.go ================================================ package gh import ( "context" "fmt" "time" "github.com/google/go-github/v80/github" "github.com/whywaita/myshoes/pkg/logger" ) func listInstallations(ctx context.Context) ([]*github.Installation, error) { if cachedRs, found := responseCache.Get(getCacheInstallationsKey()); found { return cachedRs.([]*github.Installation), nil } inst, err := _listInstallations(ctx) if err != nil { return nil, fmt.Errorf("failed to list installations: %w", err) } responseCache.Set(getCacheInstallationsKey(), inst, 1*time.Hour) return _listInstallations(ctx) } func getCacheInstallationsKey() string { return "installations" } func _listInstallations(ctx context.Context) ([]*github.Installation, error) { clientApps, err := NewClientGitHubApps() if err != nil { return nil, fmt.Errorf("failed to create a client Apps: %w", err) } var opts = &github.ListOptions{ Page: 0, PerPage: 100, } var installations []*github.Installation for { logger.Logf(true, "get installations from GitHub, page: %d, now all installations: %d", opts.Page, len(installations)) is, resp, err := clientApps.Apps.ListInstallations(ctx, opts) if err != nil { return nil, fmt.Errorf("failed to list installations: %w", err) } installations = append(installations, is...) if resp.NextPage == 0 { break } opts.Page = resp.NextPage } return installations, nil } func listAppsInstalledRepo(ctx context.Context, installationID int64) ([]*github.Repository, error) { if cachedRs, found := responseCache.Get(getCacheInstalledRepoKey(installationID)); found { return cachedRs.([]*github.Repository), nil } inst, err := _listAppsInstalledRepo(ctx, installationID) if err != nil { return nil, fmt.Errorf("failed to list installations: %w", err) } responseCache.Set(getCacheInstalledRepoKey(installationID), inst, 1*time.Hour) return _listAppsInstalledRepo(ctx, installationID) } func getCacheInstalledRepoKey(installationID int64) string { return fmt.Sprintf("installed-repo-%d", installationID) } func _listAppsInstalledRepo(ctx context.Context, installationID int64) ([]*github.Repository, error) { clientInstallation, err := NewClientInstallation(installationID) if err != nil { return nil, fmt.Errorf("failed to create a client installation: %w", err) } var opts = &github.ListOptions{ Page: 0, PerPage: 100, } var repositories []*github.Repository for { logger.Logf(true, "get list of repository from installation, page: %d, now all repositories: %d", opts.Page, len(repositories)) lr, resp, err := clientInstallation.Apps.ListRepos(ctx, opts) if err != nil { return nil, fmt.Errorf("failed to get installed repositories: %w", err) } repositories = append(repositories, lr.Repositories...) if resp.NextPage == 0 { break } opts.Page = resp.NextPage } return repositories, nil } // GetInstallationByID returns installation from cache by ID func GetInstallationByID(ctx context.Context, installationID int64) (*github.Installation, error) { installations, err := listInstallations(ctx) if err != nil { return nil, fmt.Errorf("failed to get installations: %w", err) } for _, installation := range installations { if installation.GetID() == installationID { return installation, nil } } return nil, fmt.Errorf("installation not found: %d", installationID) } // PurgeInstallationCache purges the cache of installations func PurgeInstallationCache(ctx context.Context) error { installations, err := listInstallations(ctx) if err != nil { return fmt.Errorf("failed to get installations: %w", err) } for _, installation := range installations { responseCache.Delete(getCacheInstalledRepoKey(installation.GetID())) } responseCache.Delete(getCacheInstallationsKey()) return nil } ================================================ FILE: pkg/gh/jwt.go ================================================ package gh import ( "context" "fmt" "strings" "time" "github.com/whywaita/myshoes/pkg/config" "github.com/google/go-github/v80/github" ) // function pointers (for testing) var ( GHlistInstallations = listInstallations GHlistAppsInstalledRepo = listAppsInstalledRepo ) // GenerateGitHubAppsToken generate token of GitHub Apps using private key // clientApps needs to response of `NewClientGitHubApps()` func GenerateGitHubAppsToken(ctx context.Context, clientApps *github.Client, installationID int64, scope string) (string, *time.Time, error) { token, resp, err := clientApps.Apps.CreateInstallationToken(ctx, installationID, nil) if err != nil { return "", nil, fmt.Errorf("failed to generate token from API: %w", err) } storeRateLimit(scope, resp.Rate) expiresAt := token.ExpiresAt.GetTime() return *token.Token, expiresAt, nil } // IsInstalledGitHubApp check installed GitHub Apps in gheDomain + inputScope // clientApps needs to response of `NewClientGitHubApps()` func IsInstalledGitHubApp(ctx context.Context, inputScope string) (int64, error) { installations, err := GHlistInstallations(ctx) if err != nil { return -1, fmt.Errorf("failed to get list of installations: %w", err) } for _, i := range installations { if i.SuspendedAt != nil { continue } if strings.HasPrefix(inputScope, *i.Account.Login) { // i.Account.Login is username or Organization name. // e.g.) `https://github.com/example/sample` -> `example/sample` // strings.HasPrefix search scope include i.Account.Login. switch { case strings.EqualFold(*i.RepositorySelection, "all"): // "all" can use GitHub Apps in all repositories that joined i.Account.Login. return *i.ID, nil case strings.EqualFold(*i.RepositorySelection, "selected"): // "selected" can use GitHub Apps in only some repositories that permitted. // So, need to check more using other endpoint. err := isInstalledGitHubAppSelected(ctx, inputScope, *i.ID) if err == nil { // found return *i.ID, nil } } } } return -1, &ErrIsNotInstalledGitHubApps{ githubURL: config.Config.GitHubURL, inputScope: inputScope, } } type ErrIsNotInstalledGitHubApps struct { githubURL string inputScope string } func (e *ErrIsNotInstalledGitHubApps) Error() string { return fmt.Sprintf("%s/%s is not installed configured GitHub Apps", e.githubURL, e.inputScope) } func (e *ErrIsNotInstalledGitHubApps) Unwrap() error { return fmt.Errorf("%s", e.Error()) } func isInstalledGitHubAppSelected(ctx context.Context, inputScope string, installationID int64) error { installedRepository, err := GHlistAppsInstalledRepo(ctx, installationID) if err != nil { return fmt.Errorf("failed to get list of installed repositories: %w", err) } if len(installedRepository) <= 0 { return fmt.Errorf("installed repository is not found") } switch DetectScope(inputScope) { case Organization: // Scope is Organization and installed repository is existed // So GitHub Apps installed return nil case Repository: for _, repo := range installedRepository { if strings.EqualFold(*repo.FullName, inputScope) { return nil } } return fmt.Errorf("not found") default: return fmt.Errorf("%s can't detect scope", inputScope) } } ================================================ FILE: pkg/gh/jwt_test.go ================================================ package gh import ( "context" "testing" "time" "github.com/google/go-github/v80/github" ) func setStubFunctions() { GHlistInstallations = func(ctx context.Context) ([]*github.Installation, error) { i10 := int64(10) i11 := int64(11) i12 := int64(12) all := "all" selected := "selected" exampleAll := "example-all" exampleSelected := "example-selected" exampleSuspented := "example-suspended" return []*github.Installation{ { ID: &i10, Account: &github.User{ Login: &exampleAll, }, RepositorySelection: &all, SuspendedBy: nil, }, { ID: &i11, Account: &github.User{ Login: &exampleSelected, }, RepositorySelection: &selected, SuspendedBy: nil, }, { ID: &i12, Account: &github.User{ Login: &exampleSuspented, }, RepositorySelection: &selected, SuspendedAt: &github.Timestamp{ Time: time.Now(), }, }, }, nil } GHlistAppsInstalledRepo = func(ctx context.Context, installationID int64) ([]*github.Repository, error) { fullName1 := "example-selected/sample-registered" return []*github.Repository{ { FullName: &fullName1, }, }, nil } } func Test_IsInstalledGitHubApp(t *testing.T) { setStubFunctions() tests := []struct { input struct { gheDomain string scope string } want int64 err bool }{ { input: struct { gheDomain string scope string }{gheDomain: "", scope: "example-all"}, want: 10, err: false, }, { input: struct { gheDomain string scope string }{gheDomain: "", scope: "example-all/sample"}, want: 10, err: false, }, { input: struct { gheDomain string scope string }{gheDomain: "", scope: "example-selected"}, want: 11, err: false, }, { input: struct { gheDomain string scope string }{gheDomain: "", scope: "example-selected/sample-registered"}, want: 11, err: false, }, { input: struct { gheDomain string scope string }{gheDomain: "", scope: "example-selected/sample-not-registered"}, want: -1, err: true, }, { input: struct { gheDomain string scope string }{gheDomain: "", scope: "example-suspended"}, want: -1, err: true, }, } for _, test := range tests { got, err := IsInstalledGitHubApp(context.Background(), test.input.scope) if !test.err && err != nil { t.Fatalf("failed to check GitHub Apps: %+v", err) } if got != test.want { t.Fatalf("want %d, but got %d", test.want, got) } } } ================================================ FILE: pkg/gh/label.go ================================================ package gh import "strings" // IsRequestedMyshoesLabel checks if the job has appropriate labels for myshoes func IsRequestedMyshoesLabel(labels []string) bool { // Accept dependabot runner in GHES if len(labels) == 1 && strings.EqualFold(labels[0], "dependabot") { return true } for _, label := range labels { if strings.EqualFold(label, "myshoes") || strings.EqualFold(label, "self-hosted") { return true } } return false } ================================================ FILE: pkg/gh/metrics.go ================================================ package gh import ( "context" "errors" "fmt" "net" "net/http" "time" "github.com/m4ns0ur/httpcache" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" ) const githubAPINamespace = "myshoes" var ( githubAPIRequestsTotal = promauto.NewCounterVec( prometheus.CounterOpts{ Namespace: githubAPINamespace, Subsystem: "github_api", Name: "requests_total", Help: "Total number of GitHub API requests.", }, []string{"path", "method", "status_class"}, ) githubAPIRequestDuration = promauto.NewHistogramVec( prometheus.HistogramOpts{ Namespace: githubAPINamespace, Subsystem: "github_api", Name: "request_duration_seconds", Help: "Duration of GitHub API requests in seconds.", Buckets: prometheus.DefBuckets, }, []string{"path", "method", "status_class"}, ) githubAPIErrorsTotal = promauto.NewCounterVec( prometheus.CounterOpts{ Namespace: githubAPINamespace, Subsystem: "github_api", Name: "errors_total", Help: "Total number of GitHub API request errors.", }, []string{"path", "method", "error_type"}, ) githubAPIInflight = promauto.NewGaugeVec( prometheus.GaugeOpts{ Namespace: githubAPINamespace, Subsystem: "github_api", Name: "inflight", Help: "Number of in-flight GitHub API requests.", }, []string{"path", "method"}, ) githubAPICacheTotal = promauto.NewCounterVec( prometheus.CounterOpts{ Namespace: githubAPINamespace, Subsystem: "github_api", Name: "cache_total", Help: "Total number of GitHub API cache hits/misses.", }, []string{"path", "method", "result"}, ) ) type instrumentedTransport struct { next http.RoundTripper } func newInstrumentedTransport(next http.RoundTripper) http.RoundTripper { if next == nil { next = http.DefaultTransport } if _, ok := next.(*instrumentedTransport); ok { return next } return &instrumentedTransport{next: next} } func (t *instrumentedTransport) RoundTrip(req *http.Request) (*http.Response, error) { start := time.Now() path := "unknown" method := "UNKNOWN" if req != nil { method = req.Method if req.URL != nil && req.URL.Path != "" { path = req.URL.Path } } githubAPIInflight.WithLabelValues(path, method).Inc() defer githubAPIInflight.WithLabelValues(path, method).Dec() resp, err := t.next.RoundTrip(req) statusClass := "error" if err == nil && resp != nil { statusClass = fmt.Sprintf("%dxx", resp.StatusCode/100) } githubAPIRequestsTotal.WithLabelValues(path, method, statusClass).Inc() githubAPIRequestDuration.WithLabelValues(path, method, statusClass).Observe(time.Since(start).Seconds()) if err != nil { githubAPIErrorsTotal.WithLabelValues(path, method, classifyGitHubAPIError(err)).Inc() return resp, err } if resp != nil { cacheResult := "miss" if resp.Header.Get(httpcache.XFromCache) == "1" { cacheResult = "hit" } githubAPICacheTotal.WithLabelValues(path, method, cacheResult).Inc() } return resp, err } func classifyGitHubAPIError(err error) string { if errors.Is(err, context.Canceled) { return "canceled" } if errors.Is(err, context.DeadlineExceeded) { return "deadline_exceeded" } var netErr net.Error if errors.As(err, &netErr) && netErr.Timeout() { return "timeout" } return "transport" } ================================================ FILE: pkg/gh/metrics_test.go ================================================ package gh import ( "bytes" "io" "net/http" "testing" "github.com/m4ns0ur/httpcache" "github.com/prometheus/client_golang/prometheus/testutil" ) type stubTransport struct { resp *http.Response err error } func (s *stubTransport) RoundTrip(req *http.Request) (*http.Response, error) { if s.resp != nil && s.resp.Request == nil { s.resp.Request = req } return s.resp, s.err } type timeoutErr struct{} func (timeoutErr) Error() string { return "timeout" } func (timeoutErr) Timeout() bool { return true } func (timeoutErr) Temporary() bool { return true } func TestInstrumentedTransportMetrics(t *testing.T) { req, err := http.NewRequest(http.MethodGet, "https://api.github.com/repos/org/repo", nil) if err != nil { t.Fatalf("failed to create request: %v", err) } path := req.URL.Path method := req.Method resp := &http.Response{ StatusCode: http.StatusOK, Header: make(http.Header), Body: io.NopCloser(bytes.NewBufferString("ok")), } transport := newInstrumentedTransport(&stubTransport{resp: resp}) baseReq := testutil.ToFloat64(githubAPIRequestsTotal.WithLabelValues(path, method, "2xx")) baseCache := testutil.ToFloat64(githubAPICacheTotal.WithLabelValues(path, method, "miss")) baseInflight := testutil.ToFloat64(githubAPIInflight.WithLabelValues(path, method)) if _, err := transport.RoundTrip(req); err != nil { t.Fatalf("RoundTrip error: %v", err) } if got := testutil.ToFloat64(githubAPIRequestsTotal.WithLabelValues(path, method, "2xx")); got != baseReq+1 { t.Fatalf("requests_total mismatch: got=%v want=%v", got, baseReq+1) } if got := testutil.ToFloat64(githubAPICacheTotal.WithLabelValues(path, method, "miss")); got != baseCache+1 { t.Fatalf("cache_total miss mismatch: got=%v want=%v", got, baseCache+1) } if got := testutil.ToFloat64(githubAPIInflight.WithLabelValues(path, method)); got != baseInflight { t.Fatalf("inflight mismatch: got=%v want=%v", got, baseInflight) } } func TestInstrumentedTransportCacheHit(t *testing.T) { req, err := http.NewRequest(http.MethodGet, "https://api.github.com/repos/org/repo", nil) if err != nil { t.Fatalf("failed to create request: %v", err) } path := req.URL.Path method := req.Method resp := &http.Response{ StatusCode: http.StatusOK, Header: make(http.Header), Body: io.NopCloser(bytes.NewBufferString("cached")), } resp.Header.Set(httpcache.XFromCache, "1") transport := newInstrumentedTransport(&stubTransport{resp: resp}) baseCache := testutil.ToFloat64(githubAPICacheTotal.WithLabelValues(path, method, "hit")) if _, err := transport.RoundTrip(req); err != nil { t.Fatalf("RoundTrip error: %v", err) } if got := testutil.ToFloat64(githubAPICacheTotal.WithLabelValues(path, method, "hit")); got != baseCache+1 { t.Fatalf("cache_total hit mismatch: got=%v want=%v", got, baseCache+1) } } func TestInstrumentedTransportErrorMetrics(t *testing.T) { req, err := http.NewRequest(http.MethodGet, "https://api.github.com/repos/org/repo", nil) if err != nil { t.Fatalf("failed to create request: %v", err) } path := req.URL.Path method := req.Method transport := newInstrumentedTransport(&stubTransport{err: timeoutErr{}}) baseReq := testutil.ToFloat64(githubAPIRequestsTotal.WithLabelValues(path, method, "error")) baseErr := testutil.ToFloat64(githubAPIErrorsTotal.WithLabelValues(path, method, "timeout")) if _, err := transport.RoundTrip(req); err == nil { t.Fatal("expected error, got nil") } if got := testutil.ToFloat64(githubAPIRequestsTotal.WithLabelValues(path, method, "error")); got != baseReq+1 { t.Fatalf("requests_total error mismatch: got=%v want=%v", got, baseReq+1) } if got := testutil.ToFloat64(githubAPIErrorsTotal.WithLabelValues(path, method, "timeout")); got != baseErr+1 { t.Fatalf("errors_total mismatch: got=%v want=%v", got, baseErr+1) } } ================================================ FILE: pkg/gh/ratelimit.go ================================================ package gh import ( "fmt" "sync" "github.com/google/go-github/v80/github" ) func storeRateLimit(scope string, rateLimit github.Rate) { if rateLimit.Reset.IsZero() { // Not configure rate limit, don't need to store (e.g. GHES) return } rateLimitLimit.Store(scope, rateLimit.Limit) rateLimitRemain.Store(scope, rateLimit.Remaining) } func getRateLimitKey(org, repo string) string { if repo == "" { return org } return fmt.Sprintf("%s/%s", org, repo) } // GetRateLimitRemain get a list of rate limit remaining // key: scope, value: remain func GetRateLimitRemain() map[string]int { m := map[string]int{} mu := sync.Mutex{} rateLimitRemain.Range(func(key, value interface{}) bool { k, ok := key.(string) if !ok { return false } v, ok := value.(int) if !ok { return false } mu.Lock() m[k] = v mu.Unlock() return true }) return m } // GetRateLimitLimit get a list of rate limit // key: scope, value: remain func GetRateLimitLimit() map[string]int { m := map[string]int{} mu := sync.Mutex{} rateLimitLimit.Range(func(key, value interface{}) bool { k, ok := key.(string) if !ok { return false } v, ok := value.(int) if !ok { return false } mu.Lock() m[k] = v mu.Unlock() return true }) return m } ================================================ FILE: pkg/gh/runner.go ================================================ package gh import ( "context" "fmt" "strings" "time" "github.com/google/go-github/v80/github" "github.com/whywaita/myshoes/pkg/logger" ) // ExistGitHubRunner check exist registered of GitHub runner func ExistGitHubRunner(ctx context.Context, client *github.Client, owner, repo, runnerName string) (*github.Runner, error) { runners, err := ListRunners(ctx, client, owner, repo) if err != nil { return nil, fmt.Errorf("failed to get list of runners: %w", err) } return ExistGitHubRunnerWithRunner(runners, runnerName) } // ExistGitHubRunnerWithRunner check exist registered of GitHub runner from a list of runner func ExistGitHubRunnerWithRunner(runners []*github.Runner, runnerName string) (*github.Runner, error) { for _, r := range runners { if strings.EqualFold(r.GetName(), runnerName) { return r, nil } } return nil, ErrNotFound } // ListRunners get runners that registered repository or org func ListRunners(ctx context.Context, client *github.Client, owner, repo string) ([]*github.Runner, error) { if cachedRs, found := responseCache.Get(getCacheKey(owner, repo)); found { return cachedRs.([]*github.Runner), nil } var opts = &github.ListRunnersOptions{ ListOptions: github.ListOptions{ Page: 0, PerPage: 100, }, } var rs []*github.Runner for { logger.Logf(true, "get runners from GitHub, page: %d, now all runners: %d", opts.Page, len(rs)) runners, resp, err := listRunners(ctx, client, owner, repo, opts) if err != nil { return nil, fmt.Errorf("failed to list runners: %w", err) } storeRateLimit(getRateLimitKey(owner, repo), resp.Rate) rs = append(rs, runners.Runners...) if resp.NextPage == 0 { break } opts.ListOptions.Page = resp.NextPage } responseCache.Set(getCacheKey(owner, repo), rs, 1*time.Second) logger.Logf(true, "found %d runners in GitHub", len(rs)) return rs, nil } func getCacheKey(owner, repo string) string { return fmt.Sprintf("owner-%s-repo-%s", owner, repo) } func listRunners(ctx context.Context, client *github.Client, owner, repo string, opts *github.ListRunnersOptions) (*github.Runners, *github.Response, error) { if repo == "" { runners, resp, err := client.Actions.ListOrganizationRunners(ctx, owner, opts) if err != nil { return nil, nil, fmt.Errorf("failed to list organization runners: %w", err) } return runners, resp, nil } runners, resp, err := client.Actions.ListRunners(ctx, owner, repo, opts) if err != nil { return nil, nil, fmt.Errorf("failed to list repository runners: %w", err) } return runners, resp, nil } // GetLatestRunnerVersion get a latest version of actions/runner func GetLatestRunnerVersion(ctx context.Context, scope string) (string, error) { clientApps, err := NewClientGitHubApps() if err != nil { return "", fmt.Errorf("failed to create a client from Apps: %+v", err) } installationID, err := IsInstalledGitHubApp(ctx, scope) if err != nil { return "", fmt.Errorf("failed to get installlation id: %w", err) } token, _, err := GenerateGitHubAppsToken(ctx, clientApps, installationID, scope) if err != nil { return "", fmt.Errorf("failed to get registration token: %w", err) } client, err := NewClient(token) if err != nil { return "", fmt.Errorf("failed to create GitHub client: %w", err) } switch DetectScope(scope) { case Repository: owner, repo := DivideScope(scope) applications, resp, err := client.Actions.ListRunnerApplicationDownloads(ctx, owner, repo) if err != nil { return "", fmt.Errorf("failed to get latest runner version: %w", err) } storeRateLimit(getRateLimitKey(owner, repo), resp.Rate) return getRunnerVersion(applications) case Organization: applications, resp, err := client.Actions.ListOrganizationRunnerApplicationDownloads(ctx, scope) if err != nil { return "", fmt.Errorf("failed to get latest runner version: %w", err) } storeRateLimit(getRateLimitKey(scope, ""), resp.Rate) return getRunnerVersion(applications) } return "", fmt.Errorf("invalid scope: %s", scope) } func getRunnerVersion(applications []*github.RunnerApplicationDownload) (string, error) { // filename": "actions-runner-linux-x64-2.164.0.tar.gz" for _, app := range applications { if *app.OS == "linux" && *app.Architecture == "x64" { v := strings.ReplaceAll(*app.Filename, "actions-runner-linux-x64-", "") v = strings.ReplaceAll(v, ".tar.gz", "") return fmt.Sprintf("v%s", v), nil } } return "", fmt.Errorf("not found runner version") } // ConcatLabels concat labels from check event JSON func ConcatLabels(checkEventJSON string) (string, error) { runsOnLabels, err := ExtractRunsOnLabels([]byte(checkEventJSON)) if err != nil { return "", fmt.Errorf("failed to extract runs-on labels: %w", err) } runsOnConcat := "none" if len(runsOnLabels) != 0 { runsOnConcat = strings.Join(runsOnLabels, ",") // e.g. "self-hosted,linux" } return runsOnConcat, nil } ================================================ FILE: pkg/gh/scope.go ================================================ package gh import "strings" // Scope is scope for auto-scaling target type Scope int // Scope values const ( Unknown Scope = iota Repository Organization ) // String is fmt.Stringer interface func (s Scope) String() string { switch s { case Repository: return "repos" case Organization: return "orgs" default: return "unknown" } } // DetectScope detect a scope (repo or org) func DetectScope(scope string) Scope { sep := strings.Split(scope, "/") switch len(sep) { case 1: return Organization case 2: return Repository default: return Unknown } } // DivideScope divide scope to owner and repo func DivideScope(scope string) (string, string) { var owner, repo string switch DetectScope(scope) { case Organization: owner = scope repo = "" case Repository: s := strings.Split(scope, "/") owner = s[0] repo = s[1] } return owner, repo } ================================================ FILE: pkg/gh/token_registration.go ================================================ package gh import ( "context" "fmt" "time" "github.com/patrickmn/go-cache" ) var ( cacheRegistrationToken = cache.New(1*time.Hour, 1*time.Hour) ) // GetRunnerRegistrationToken get token for register runner // clientInstallation needs to response of `NewClientInstallation()` func GetRunnerRegistrationToken(ctx context.Context, installationID int64, scope string) (string, error) { cachedToken := getRunnerRegisterTokenFromCache(installationID, scope) if cachedToken != "" { return cachedToken, nil } rrToken, expiresAt, err := generateRunnerRegisterToken(ctx, installationID, scope) if err != nil { return "", fmt.Errorf("failed to generate runner register token: %w", err) } setRunnerRegisterTokenCache(installationID, scope, rrToken, *expiresAt) return rrToken, nil } // generateRunnerRegistrationToken generate token for register runner // clientInstallation needs to response of `NewClientInstallation()` func generateRunnerRegisterToken(ctx context.Context, installationID int64, scope string) (string, *time.Time, error) { clientInstallation, err := NewClientInstallation(installationID) if err != nil { return "", nil, fmt.Errorf("failed to create a client installation: %w", err) } switch DetectScope(scope) { case Organization: token, _, err := clientInstallation.Actions.CreateOrganizationRegistrationToken(ctx, scope) if err != nil { return "", nil, fmt.Errorf("failed to generate registration token for organization (scope: %s): %w", scope, err) } return *token.Token, &token.ExpiresAt.Time, nil case Repository: owner, repo := DivideScope(scope) token, _, err := clientInstallation.Actions.CreateRegistrationToken(ctx, owner, repo) if err != nil { return "", nil, fmt.Errorf("failed to generate registration token for repository (scope: %s): %w", scope, err) } return *token.Token, &token.ExpiresAt.Time, nil default: return "", nil, fmt.Errorf("failed to detect scope (scope: %s)", scope) } } func setRunnerRegisterTokenCache(installationID int64, scope, token string, expiresAt time.Time) { expiresDuration := time.Until(expiresAt.Add(-6 * time.Minute)) cacheRegistrationToken.Set(getCacheKeyRegistrationToken(installationID, scope), token, expiresDuration) } func getRunnerRegisterTokenFromCache(installationID int64, scope string) string { got, found := cacheRegistrationToken.Get(getCacheKeyRegistrationToken(installationID, scope)) if !found { return "" } token, ok := got.(string) if !ok { return "" } return token } func getCacheKeyRegistrationToken(installationID int64, scope string) string { return fmt.Sprintf("%s-%d", scope, installationID) } ================================================ FILE: pkg/gh/webhook.go ================================================ package gh import ( "encoding/json" "fmt" "github.com/google/go-github/v80/github" ) // parseEventJSON parse a json of webhook from GitHub. // github.ParseWebHook need *http.Request because it checks headers in request. func parseEventJSON(in []byte) (interface{}, error) { var checkRun *github.CheckRunEvent err := json.Unmarshal(in, &checkRun) if err == nil && checkRun.GetCheckRun() != nil { return checkRun, nil } var workflowJobEvent *github.WorkflowJobEvent err = json.Unmarshal(in, &workflowJobEvent) if err == nil && workflowJobEvent.GetWorkflowJob() != nil { return workflowJobEvent, nil } var workflowJob *github.WorkflowJob err = json.Unmarshal(in, &workflowJob) if err == nil && workflowJob != nil { return workflowJob, nil } return nil, fmt.Errorf("input json is unsupported type") } // ExtractRunsOnLabels extract labels from github.WorkflowJobEvent func ExtractRunsOnLabels(in []byte) ([]string, error) { event, err := parseEventJSON(in) if err != nil { return nil, fmt.Errorf("failed to parse event json: %w", err) } switch t := event.(type) { case *github.WorkflowJobEvent: // workflow_job has labels, can extract labels return t.GetWorkflowJob().Labels, nil case *github.WorkflowJob: return t.Labels, nil } return []string{}, nil } ================================================ FILE: pkg/gh/workflow_job.go ================================================ package gh import ( "context" "fmt" "time" "github.com/google/go-github/v80/github" ) func listWorkflowJob(ctx context.Context, client *github.Client, owner, repo string, runID int64, opts *github.ListWorkflowJobsOptions) ([]*github.WorkflowJob, *github.Response, error) { jobs, resp, err := client.Actions.ListWorkflowJobs(ctx, owner, repo, runID, opts) if err != nil { return nil, nil, fmt.Errorf("failed to list workflow runs: %w", err) } return jobs.Jobs, resp, nil } // ListWorkflowJobByRunID get workflow job by run ID func ListWorkflowJobByRunID(ctx context.Context, client *github.Client, owner, repo string, runID int64) ([]*github.WorkflowJob, error) { if cachedWorkflowJobs, found := responseCache.Get(getWorkflowJobCacheKey(owner, repo, runID)); found { return cachedWorkflowJobs.([]*github.WorkflowJob), nil } opts := &github.ListWorkflowJobsOptions{ Filter: "latest", } jobs, _, err := listWorkflowJob(ctx, client, owner, repo, runID, opts) if err != nil { return nil, fmt.Errorf("failed to list workflow jobs: %w", err) } responseCache.Set(getWorkflowJobCacheKey(owner, repo, runID), jobs, 1*time.Minute) return jobs, nil } func getWorkflowJobCacheKey(owner, repo string, runID int64) string { return fmt.Sprintf("runs-owner-%s-repo-%s-runid-%d", owner, repo, runID) } ================================================ FILE: pkg/gh/workflow_run.go ================================================ package gh import ( "context" "fmt" "time" "github.com/google/go-github/v80/github" "github.com/whywaita/myshoes/pkg/logger" ) func listWorkflowRuns(ctx context.Context, client *github.Client, owner, repo string, opts *github.ListWorkflowRunsOptions) (*github.WorkflowRuns, *github.Response, error) { runs, resp, err := client.Actions.ListRepositoryWorkflowRuns(ctx, owner, repo, opts) if err != nil { return nil, nil, fmt.Errorf("failed to list workflow runs: %w", err) } return runs, resp, nil } // ListWorkflowRunsNewest get workflow runs that registered in the last (%d: limit) runs func ListWorkflowRunsNewest(ctx context.Context, client *github.Client, owner, repo string, limit int) ([]*github.WorkflowRun, error) { if cachedWorkflowRuns, found := responseCache.Get(getRunsCacheKey(owner, repo)); found { return cachedWorkflowRuns.([]*github.WorkflowRun), nil } var opts = &github.ListWorkflowRunsOptions{ ListOptions: github.ListOptions{ Page: 0, PerPage: 100, }, } var workflowRuns []*github.WorkflowRun for { logger.Logf(true, "get workflow runs from GitHub, page: %d, now all runners: %d (repo: %s/%s)", opts.Page, len(workflowRuns), owner, repo) runs, resp, err := listWorkflowRuns(ctx, client, owner, repo, opts) if err != nil { return nil, fmt.Errorf("failed to list workflow runs: %w", err) } storeRateLimit(getRateLimitKey(owner, repo), resp.Rate) workflowRuns = append(workflowRuns, runs.WorkflowRuns...) if resp.NextPage == 0 { break } if len(workflowRuns) >= limit { break } opts.Page = resp.NextPage } responseCache.Set(getRunsCacheKey(owner, repo), workflowRuns, 3*time.Minute) return workflowRuns, nil } func getRunsCacheKey(owner, repo string) string { return fmt.Sprintf("runs-owner-%s-repo-%s", owner, repo) } ================================================ FILE: pkg/logger/logger.go ================================================ package logger import ( "log" "os" "sync" "github.com/whywaita/myshoes/pkg/config" ) var ( logger = log.New(os.Stderr, "", log.LstdFlags) logMu sync.Mutex ) // SetLogger set logger in outside of library func SetLogger(l *log.Logger) { if l == nil { l = log.New(os.Stderr, "", log.LstdFlags) } logMu.Lock() logger = l logMu.Unlock() } // Logf is interface for logger func Logf(isDebug bool, format string, v ...interface{}) { logMu.Lock() defer logMu.Unlock() switch { case !isDebug: // normal logging logger.Printf(format, v...) case isDebug && config.Config.Debug: // debug logging format = "[DEBUG] " + format logger.Printf(format, v...) } } ================================================ FILE: pkg/metric/collector.go ================================================ package metric import ( "context" "fmt" "sync" "time" "github.com/prometheus/client_golang/prometheus" "github.com/whywaita/myshoes/pkg/datastore" "github.com/whywaita/myshoes/pkg/logger" ) const ( namespace = "myshoes" ) var ( scrapeDurationDesc = prometheus.NewDesc( prometheus.BuildFQName(namespace, "", "collector_duration_seconds"), "Collector time duration.", []string{"collector"}, nil, ) ) // Collector is a collector for prometheus type Collector struct { ctx context.Context metrics Metrics ds datastore.Datastore scrapers []Scraper } // NewCollector create a collector func NewCollector(ctx context.Context, ds datastore.Datastore) *Collector { return &Collector{ ctx: ctx, metrics: NewMetrics(), ds: ds, scrapers: NewScrapers(), } } // Describe describe metrics func (c *Collector) Describe(ch chan<- *prometheus.Desc) { ch <- c.metrics.TotalScrapes.Desc() ch <- c.metrics.Error.Desc() c.metrics.ScrapeErrors.Describe(ch) } // Collect collect metrics func (c *Collector) Collect(ch chan<- prometheus.Metric) { c.scrape(c.ctx, ch) ch <- c.metrics.TotalScrapes ch <- c.metrics.Error c.metrics.ScrapeErrors.Collect(ch) } func (c *Collector) scrape(ctx context.Context, ch chan<- prometheus.Metric) { c.metrics.TotalScrapes.Inc() c.metrics.Error.Set(0) var wg sync.WaitGroup for _, scraper := range c.scrapers { wg.Add(1) go func(scraper Scraper) { defer wg.Done() label := fmt.Sprintf("collect.%s", scraper.Name()) scrapeStartTime := time.Now() if err := scraper.Scrape(ctx, c.ds, ch); err != nil { logger.Logf(false, "failed to scrape metrics (name: %s): %+v", scraper.Name(), err) c.metrics.ScrapeErrors.WithLabelValues(label).Inc() c.metrics.Error.Set(1) } ch <- prometheus.MustNewConstMetric(scrapeDurationDesc, prometheus.GaugeValue, time.Since(scrapeStartTime).Seconds(), label) }(scraper) } wg.Wait() } // Scraper is interface for scraping type Scraper interface { Name() string Help() string Scrape(ctx context.Context, ds datastore.Datastore, ch chan<- prometheus.Metric) error } // NewScrapers return list of scraper func NewScrapers() []Scraper { return []Scraper{ ScraperDatastore{}, ScraperMemory{}, ScraperGitHub{}, } } // Metrics is data in scraper type Metrics struct { TotalScrapes prometheus.Counter ScrapeErrors *prometheus.CounterVec Error prometheus.Gauge } // NewMetrics create a metrics func NewMetrics() Metrics { return Metrics{ TotalScrapes: prometheus.NewCounter(prometheus.CounterOpts{ Namespace: namespace, Subsystem: "", Name: "scrapes_total", Help: "Total number of times myshoes was scraped for metrics.", }), ScrapeErrors: prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: namespace, Subsystem: "", Name: "scrape_errors_total", Help: "Total number of times an error occurred scraping a myshoes.", }, []string{"collector"}), Error: prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: namespace, Subsystem: "", Name: "last_scrape_error", Help: "Whether the last scrape of metrics from myshoes resulted in an error (1 for error, 0 for success).", }), } } ================================================ FILE: pkg/metric/scrape_datastore.go ================================================ package metric import ( "context" "fmt" "sort" "strings" "time" "github.com/prometheus/client_golang/prometheus" "github.com/whywaita/myshoes/pkg/datastore" "github.com/whywaita/myshoes/pkg/gh" "github.com/whywaita/myshoes/pkg/logger" "github.com/whywaita/myshoes/pkg/starter" ) const datastoreName = "datastore" var ( datastoreJobsDesc = prometheus.NewDesc( prometheus.BuildFQName(namespace, datastoreName, "jobs"), "Number of jobs", []string{"target_id", "runs_on"}, nil, ) datastoreTargetsDesc = prometheus.NewDesc( prometheus.BuildFQName(namespace, datastoreName, "targets"), "Number of targets", []string{"resource_type"}, nil, ) datastoreTargetDesc = prometheus.NewDesc( prometheus.BuildFQName(namespace, datastoreName, "target_describe"), "Target", []string{ "target_id", "scope", "resource_type", }, nil, ) datastoreTargetTokenExpiresDesc = prometheus.NewDesc( prometheus.BuildFQName(namespace, datastoreName, "target_token_expires_seconds"), "Token expires time", []string{"target_id"}, nil, ) datastoreJobDurationOldest = prometheus.NewDesc( prometheus.BuildFQName(namespace, datastoreName, "job_duration_oldest_seconds"), "Duration time of oldest job", []string{"job_id", "runs_on"}, nil, ) datastoreDeletedJobsDesc = prometheus.NewDesc( prometheus.BuildFQName(namespace, datastoreName, "deleted_jobs"), "Number of deleted jobs", []string{"runs_on"}, nil, ) datastoreRunnersRunningDesc = prometheus.NewDesc( prometheus.BuildFQName(namespace, datastoreName, "runners_running"), "Number of runners running", []string{"target_id"}, nil, ) ) // ScraperDatastore is scraper implement for datastore.Datastore type ScraperDatastore struct{} // Name return name func (ScraperDatastore) Name() string { return datastoreName } // Help return help func (ScraperDatastore) Help() string { return "Collect from datastore" } // Scrape scrape metrics func (ScraperDatastore) Scrape(ctx context.Context, ds datastore.Datastore, ch chan<- prometheus.Metric) error { if err := scrapeJobs(ctx, ds, ch); err != nil { return fmt.Errorf("failed to scrape jobs: %w", err) } if err := scrapeTargets(ctx, ds, ch); err != nil { return fmt.Errorf("failed to scrape targets: %w", err) } if err := scrapeRunners(ctx, ds, ch); err != nil { return fmt.Errorf("failed to scrape runners: %w", err) } return nil } func scrapeJobs(ctx context.Context, ds datastore.Datastore, ch chan<- prometheus.Metric) error { if err := scrapeJobCounter(ctx, ds, ch); err != nil { return fmt.Errorf("failed to scrape job counter: %w", err) } jobs, err := ds.ListJobs(ctx) if err != nil { return fmt.Errorf("failed to list jobs: %w", err) } if len(jobs) == 0 { ch <- prometheus.MustNewConstMetric( datastoreJobsDesc, prometheus.GaugeValue, 0, "none", "none", ) return nil } sort.SliceStable(jobs, func(i, j int) bool { // oldest job is first return jobs[i].CreatedAt.Before(jobs[j].CreatedAt) }) type storedValue struct { OldestJob datastore.Job Count float64 } stored := map[string]storedValue{} // job separate target_id and runs-on labels for _, j := range jobs { runsOnConcat, err := gh.ConcatLabels(j.CheckEventJSON) if err != nil { logger.Logf(false, "failed to concat labels: %+v", err) continue } key := fmt.Sprintf("%s-_-%s", j.TargetID.String(), runsOnConcat) v, ok := stored[key] if !ok { stored[key] = storedValue{ OldestJob: j, Count: 1, } } else { if j.CreatedAt.Before(v.OldestJob.CreatedAt) { stored[key] = storedValue{ OldestJob: j, Count: v.Count + 1, } } else { stored[key] = storedValue{ OldestJob: v.OldestJob, Count: v.Count + 1, } } } } for key, value := range stored { // key: target_id-_-runs-on // value: storedValue split := strings.Split(key, "-_-") ch <- prometheus.MustNewConstMetric( datastoreJobsDesc, prometheus.GaugeValue, value.Count, split[0], // target_id split[1], // runs-on ) ch <- prometheus.MustNewConstMetric( datastoreJobDurationOldest, prometheus.GaugeValue, time.Since(value.OldestJob.CreatedAt).Seconds(), value.OldestJob.UUID.String(), split[1], ) } return nil } func scrapeJobCounter(ctx context.Context, ds datastore.Datastore, ch chan<- prometheus.Metric) error { starter.DeletedJobMap.Range(func(key, value interface{}) bool { runsOn := key.(string) number := value.(int) ch <- prometheus.MustNewConstMetric( datastoreDeletedJobsDesc, prometheus.CounterValue, float64(number), runsOn, ) return true }) return nil } func scrapeTargets(ctx context.Context, ds datastore.Datastore, ch chan<- prometheus.Metric) error { targets, err := datastore.ListTargets(ctx, ds) if err != nil { return fmt.Errorf("failed to list targets: %w", err) } now := time.Now() result := map[string]float64{} // key: resource_type, value: number for _, t := range targets { ch <- prometheus.MustNewConstMetric( datastoreTargetDesc, prometheus.GaugeValue, 1, t.UUID.String(), t.Scope, t.ResourceType.String(), ) result[t.ResourceType.String()]++ ch <- prometheus.MustNewConstMetric( datastoreTargetTokenExpiresDesc, prometheus.GaugeValue, t.TokenExpiredAt.Sub(now).Seconds(), t.UUID.String(), ) } for rt, number := range result { ch <- prometheus.MustNewConstMetric( datastoreTargetsDesc, prometheus.GaugeValue, number, rt, ) } return nil } func scrapeRunners(ctx context.Context, ds datastore.Datastore, ch chan<- prometheus.Metric) error { runners, err := ds.ListRunners(ctx) if err != nil { return fmt.Errorf("failed to list runners: %w", err) } result := map[string]float64{} // key: target_id, value: number for _, r := range runners { result[r.TargetID.String()]++ } for targetID, number := range result { ch <- prometheus.MustNewConstMetric( datastoreRunnersRunningDesc, prometheus.GaugeValue, number, targetID, ) } return nil } var _ Scraper = ScraperDatastore{} ================================================ FILE: pkg/metric/scrape_github.go ================================================ package metric import ( "context" "fmt" "strconv" "time" "github.com/prometheus/client_golang/prometheus" "github.com/whywaita/myshoes/pkg/datastore" "github.com/whywaita/myshoes/pkg/gh" "github.com/whywaita/myshoes/pkg/logger" ) const githubName = "github" var ( githubPendingRunsDesc = prometheus.NewDesc( prometheus.BuildFQName(namespace, githubName, "pending_runs"), "Number of pending runs", []string{"target_id", "scope"}, nil, ) githubPendingWorkflowRunSecondsDesc = prometheus.NewDesc( prometheus.BuildFQName(namespace, githubName, "pending_workflow_run_seconds"), "Second of Pending time in workflow run", []string{"target_id", "workflow_id", "workflow_run_id", "html_url"}, nil, ) githubInstallationDesc = prometheus.NewDesc( prometheus.BuildFQName(namespace, githubName, "installation"), "installations", []string{ "installation_id", "account_login", "account_type", "target_type", "repository_selection", "html_url", }, nil, ) ) // ScraperGitHub is scraper implement for GitHub type ScraperGitHub struct{} // Name return name func (ScraperGitHub) Name() string { return githubName } // Help return help func (ScraperGitHub) Help() string { return "Collect from GitHub" } // Scrape scrape metrics func (s ScraperGitHub) Scrape(ctx context.Context, ds datastore.Datastore, ch chan<- prometheus.Metric) error { if err := scrapePendingRuns(ctx, ds, ch); err != nil { return fmt.Errorf("failed to scrape pending runs: %w", err) } if err := scrapeInstallation(ctx, ch); err != nil { return fmt.Errorf("failed to scrape installations: %w", err) } return nil } func scrapePendingRuns(ctx context.Context, ds datastore.Datastore, ch chan<- prometheus.Metric) error { pendingRuns, err := datastore.GetPendingWorkflowRunByRecentRepositories(ctx, ds) if err != nil { return fmt.Errorf("failed to get pending workflow runs: %w", err) } for _, pendingRun := range pendingRuns { sinceSeconds := time.Since(pendingRun.WorkflowRun.CreatedAt.Time).Seconds() ch <- prometheus.MustNewConstMetric( githubPendingWorkflowRunSecondsDesc, prometheus.GaugeValue, sinceSeconds, pendingRun.Target.UUID.String(), strconv.FormatInt(pendingRun.WorkflowRun.GetWorkflowID(), 10), strconv.FormatInt(pendingRun.WorkflowRun.GetID(), 10), pendingRun.WorkflowRun.GetHTMLURL(), ) } // count pending runs by target countPendingMap := make(map[string]int) targetCache := make(map[string]*datastore.Target) for _, pendingRun := range pendingRuns { countPendingMap[pendingRun.Target.UUID.String()]++ targetCache[pendingRun.Target.UUID.String()] = pendingRun.Target } for targetID, countPending := range countPendingMap { target, ok := targetCache[targetID] if !ok { logger.Logf(false, "failed to get target by targetID from targetCache: %s", targetID) continue } ch <- prometheus.MustNewConstMetric(githubPendingRunsDesc, prometheus.GaugeValue, float64(countPending), target.UUID.String(), target.Scope) } return nil } func scrapeInstallation(ctx context.Context, ch chan<- prometheus.Metric) error { installations, err := gh.GHlistInstallations(ctx) if err != nil { return fmt.Errorf("failed to list installations: %w", err) } for _, installation := range installations { ch <- prometheus.MustNewConstMetric( githubInstallationDesc, prometheus.GaugeValue, 1, fmt.Sprint(installation.GetID()), installation.GetAccount().GetLogin(), installation.GetAccount().GetType(), installation.GetTargetType(), installation.GetRepositorySelection(), installation.GetAccount().GetHTMLURL(), ) } return nil } ================================================ FILE: pkg/metric/scrape_memory.go ================================================ package metric import ( "context" "fmt" "sync/atomic" uuid "github.com/satori/go.uuid" "github.com/prometheus/client_golang/prometheus" "github.com/whywaita/myshoes/pkg/config" "github.com/whywaita/myshoes/pkg/datastore" "github.com/whywaita/myshoes/pkg/docker" "github.com/whywaita/myshoes/pkg/gh" "github.com/whywaita/myshoes/pkg/runner" "github.com/whywaita/myshoes/pkg/starter" ) const memoryName = "memory" var ( memoryStarterMaxRunning = prometheus.NewDesc( prometheus.BuildFQName(namespace, memoryName, "starter_max_running"), "The number of max running in starter (Config)", []string{"starter"}, nil, ) memoryStarterQueueRunning = prometheus.NewDesc( prometheus.BuildFQName(namespace, memoryName, "starter_queue_running"), "running queue in starter", []string{"starter"}, nil, ) memoryStarterQueueWaiting = prometheus.NewDesc( prometheus.BuildFQName(namespace, memoryName, "starter_queue_waiting"), "waiting queue in starter", []string{"starter"}, nil, ) memoryStarterRescuedRuns = prometheus.NewDesc( prometheus.BuildFQName(namespace, memoryName, "starter_rescued_runs"), "rescued runs in starter", []string{"starter", "target"}, nil, ) memoryGitHubRateLimitRemaining = prometheus.NewDesc( prometheus.BuildFQName(namespace, memoryName, "github_rate_limit_remaining"), "The number of rate limit remaining in GitHub", []string{"scope"}, nil, ) memoryGitHubRateLimitLimiting = prometheus.NewDesc( prometheus.BuildFQName(namespace, memoryName, "github_rate_limit_limiting"), "The number of rate limit max in GitHub", []string{"scope"}, nil, ) memoryDockerHubRateLimitRemaining = prometheus.NewDesc( prometheus.BuildFQName(namespace, memoryName, "dockerhub_rate_limit_remaining"), "The number of rate limit remaining in DockerHub", []string{}, nil, ) memoryDockerHubRateLimitLimiting = prometheus.NewDesc( prometheus.BuildFQName(namespace, memoryName, "dockerhub_rate_limit_limiting"), "The number of rate limit max in DockerHub", []string{}, nil, ) memoryRunnerMaxConcurrencyDeleting = prometheus.NewDesc( prometheus.BuildFQName(namespace, memoryName, "runner_max_concurrency_deleting"), "The number of max concurrency deleting in runner (Config)", []string{"runner"}, nil, ) memoryRunnerQueueConcurrencyDeleting = prometheus.NewDesc( prometheus.BuildFQName(namespace, memoryName, "runner_queue_concurrency_deleting"), "deleting concurrency in runner", []string{"runner"}, nil, ) memoryRunnerDeleteRetryCount = prometheus.NewDesc( prometheus.BuildFQName(namespace, memoryName, "runner_delete_retry_count"), "retry count of deleting in runner", []string{"runner"}, nil, ) memoryRunnerCreateRetryCount = prometheus.NewDesc( prometheus.BuildFQName(namespace, memoryName, "runner_create_retry_count"), "retry count of creating in runner", []string{"runner"}, nil, ) ) // ScraperMemory is scraper implement for memory type ScraperMemory struct{} // Name return name func (ScraperMemory) Name() string { return memoryName } // Help return help func (ScraperMemory) Help() string { return "Collect from memory" } // Scrape scrape metrics func (ScraperMemory) Scrape(ctx context.Context, ds datastore.Datastore, ch chan<- prometheus.Metric) error { if err := scrapeStarterValues(ch); err != nil { return fmt.Errorf("failed to scrape starter values: %w", err) } if err := scrapeGitHubValues(ch); err != nil { return fmt.Errorf("failed to scrape GitHub values: %w", err) } if config.Config.ProvideDockerHubMetrics { if err := scrapeDockerValues(ch); err != nil { return fmt.Errorf("failed to scrape Docker values: %w", err) } } return nil } func scrapeStarterValues(ch chan<- prometheus.Metric) error { configMax := config.Config.MaxConnectionsToBackend const labelStarter = "starter" ch <- prometheus.MustNewConstMetric( memoryStarterMaxRunning, prometheus.GaugeValue, float64(configMax), labelStarter) countRunning := starter.CountRunning.Load() countWaiting := starter.CountWaiting.Load() ch <- prometheus.MustNewConstMetric( memoryStarterQueueRunning, prometheus.GaugeValue, float64(countRunning), labelStarter) ch <- prometheus.MustNewConstMetric( memoryStarterQueueWaiting, prometheus.GaugeValue, float64(countWaiting), labelStarter) starter.CountRescued.Range(func(key, value interface{}) bool { counter := value.(*atomic.Int64) ch <- prometheus.MustNewConstMetric( memoryStarterRescuedRuns, prometheus.GaugeValue, float64(counter.Load()), labelStarter, key.(string), ) return true }) const labelRunner = "runner" configRunnerDeletingMax := config.Config.MaxConcurrencyDeleting countRunnerDeletingNow := runner.ConcurrencyDeleting.Load() ch <- prometheus.MustNewConstMetric( memoryRunnerMaxConcurrencyDeleting, prometheus.GaugeValue, float64(configRunnerDeletingMax), labelRunner) ch <- prometheus.MustNewConstMetric( memoryRunnerQueueConcurrencyDeleting, prometheus.GaugeValue, float64(countRunnerDeletingNow), labelRunner) runner.DeleteRetryCount.Range(func(key, value any) bool { ch <- prometheus.MustNewConstMetric( memoryRunnerDeleteRetryCount, prometheus.GaugeValue, float64(value.(int)), key.(uuid.UUID).String()) return true }) starter.AddInstanceRetryCount.Range(func(key, value any) bool { ch <- prometheus.MustNewConstMetric( memoryRunnerCreateRetryCount, prometheus.GaugeValue, float64(value.(int)), key.(uuid.UUID).String()) return true }) return nil } func scrapeGitHubValues(ch chan<- prometheus.Metric) error { rateLimitRemain := gh.GetRateLimitRemain() for scope, remain := range rateLimitRemain { ch <- prometheus.MustNewConstMetric( memoryGitHubRateLimitRemaining, prometheus.GaugeValue, float64(remain), scope, ) } rateLimitLimit := gh.GetRateLimitLimit() for scope, limit := range rateLimitLimit { ch <- prometheus.MustNewConstMetric( memoryGitHubRateLimitLimiting, prometheus.GaugeValue, float64(limit), scope, ) } return nil } var _ Scraper = ScraperMemory{} func scrapeDockerValues(ch chan<- prometheus.Metric) error { rateLimit, err := docker.GetRateLimit() if err != nil { return fmt.Errorf("failed to get rate limit: %w", err) } ch <- prometheus.MustNewConstMetric( memoryDockerHubRateLimitRemaining, prometheus.GaugeValue, float64(rateLimit.Remaining), ) ch <- prometheus.MustNewConstMetric( memoryDockerHubRateLimitLimiting, prometheus.GaugeValue, float64(rateLimit.Limit), ) return nil } ================================================ FILE: pkg/metric/webhook.go ================================================ package metric import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" ) var ( // WebhookReceivedTotal is the total number of webhooks received WebhookReceivedTotal = promauto.NewCounterVec( prometheus.CounterOpts{ Namespace: namespace, Subsystem: "webhook", Name: "received_total", Help: "Total number of webhooks received from GitHub", }, []string{"event_type", "status", "runs_on"}, ) // WebhookProcessingDuration is the duration of webhook processing WebhookProcessingDuration = promauto.NewHistogramVec( prometheus.HistogramOpts{ Namespace: namespace, Subsystem: "webhook", Name: "processing_duration_seconds", Help: "Duration of webhook processing in seconds", Buckets: prometheus.DefBuckets, }, []string{"event_type", "runs_on"}, ) // WebhookJobsEnqueued is the total number of jobs enqueued from webhooks WebhookJobsEnqueued = promauto.NewCounterVec( prometheus.CounterOpts{ Namespace: namespace, Subsystem: "webhook", Name: "jobs_enqueued_total", Help: "Total number of jobs enqueued from webhooks", }, []string{"event_type", "repository", "runs_on"}, ) ) ================================================ FILE: pkg/runner/metrics.go ================================================ package runner import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" ) var ( // DeleteRunnerBackoffDuration is histogram of exponential backoff duration for deleting runner DeleteRunnerBackoffDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ Namespace: "myshoes", Subsystem: "runner", Name: "delete_runner_backoff_duration_seconds", Help: "Histogram of exponential backoff duration in seconds for deleting runner", Buckets: prometheus.ExponentialBuckets(1, 2, 10), // 1s, 2s, 4s, 8s, 16s, 32s, 64s, 128s, 256s, 512s }, []string{"runner_uuid"}) // DeleteRunnerRetryTotal is counter of total retries for deleting runner DeleteRunnerRetryTotal = promauto.NewCounterVec(prometheus.CounterOpts{ Namespace: "myshoes", Subsystem: "runner", Name: "delete_runner_retry_total", Help: "Total number of retries for deleting runner", }, []string{"runner_uuid"}) ) ================================================ FILE: pkg/runner/runner.go ================================================ package runner import ( "context" "fmt" "time" "github.com/hashicorp/go-version" "github.com/whywaita/myshoes/pkg/datastore" "github.com/whywaita/myshoes/pkg/logger" ) var ( // GoalCheckerInterval is interval time of check deleting runner GoalCheckerInterval = 1 * time.Minute // MustGoalTime is hard limit for idle runner. // So it is same as the limit of GitHub Actions MustGoalTime = 6 * time.Hour // MustRunningTime is set time of instance create + download binaries + etc MustRunningTime = 5 * time.Minute // TargetTokenInterval is interval time of checking target token TargetTokenInterval = 5 * time.Minute //NeedRefreshToken is time of token expired NeedRefreshToken = 10 * time.Minute ) // Manager is runner management type Manager struct { ds datastore.Datastore runnerVersion string } // New create a Manager func New(ds datastore.Datastore, runnerVersion string) *Manager { return &Manager{ ds: ds, runnerVersion: runnerVersion, } } // Loop check func (m *Manager) Loop(ctx context.Context) error { logger.Logf(false, "start runner loop") ticker := time.NewTicker(GoalCheckerInterval) defer ticker.Stop() if err := m.doTargetToken(ctx); err != nil { logger.Logf(false, "failed to refresh token in initialize: %+v", err) } go func(ctx context.Context) { tokenRefreshTicker := time.NewTicker(TargetTokenInterval) defer tokenRefreshTicker.Stop() for { select { case <-tokenRefreshTicker.C: if err := m.doTargetToken(ctx); err != nil { logger.Logf(false, "failed to refresh token: %+v", err) } case <-ctx.Done(): return } } }(ctx) for { select { case <-ticker.C: if err := m.do(ctx); err != nil { logger.Logf(false, "failed to starter: %+v", err) } case <-ctx.Done(): return nil } } } // TemporaryMode is mode of temporary runner type TemporaryMode int // RunnerEphemeralModes variable const ( TemporaryUnknown TemporaryMode = iota TemporaryOnce TemporaryEphemeral ) // StringFlag return flag func (rtm TemporaryMode) StringFlag() string { switch rtm { case TemporaryOnce: return "--once" case TemporaryEphemeral: return "--ephemeral" } return "unknown" } // GetRunnerTemporaryMode get runner version and RunnerTemporaryMode func GetRunnerTemporaryMode(runnerVersion string) (string, TemporaryMode, error) { ephemeralSupportVersion, err := version.NewVersion("v2.282.0") if err != nil { return "", TemporaryUnknown, fmt.Errorf("failed to parse ephemeral runner version: %w", err) } inputVersion, err := version.NewVersion(runnerVersion) if err != nil { return "", TemporaryUnknown, fmt.Errorf("failed to parse input runner version: %w", err) } if ephemeralSupportVersion.GreaterThan(inputVersion) { logger.Logf(false, "WARNING: runner version is lower than v2.282.0, use --once mode. It's deprecated. will be removed in future.") return runnerVersion, TemporaryOnce, nil } return runnerVersion, TemporaryEphemeral, nil } ================================================ FILE: pkg/runner/runner_delete.go ================================================ package runner import ( "context" "errors" "fmt" "strings" "sync" "sync/atomic" "time" "github.com/google/go-github/v80/github" "github.com/whywaita/myshoes/internal/util" "github.com/whywaita/myshoes/pkg/config" "github.com/whywaita/myshoes/pkg/datastore" "github.com/whywaita/myshoes/pkg/gh" "github.com/whywaita/myshoes/pkg/logger" "github.com/whywaita/myshoes/pkg/shoes" "golang.org/x/sync/errgroup" "golang.org/x/sync/semaphore" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) var ( // ConcurrencyDeleting is value of concurrency ConcurrencyDeleting atomic.Int64 // DeletingTimeout is timeout of deleting runner DeletingTimeout = 3 * time.Minute // DeleteRetryCount is retry count of deleting runner DeleteRetryCount = sync.Map{} // key: runner.UUID // MaxDeleteRetry is max retry count of delete runner MaxDeleteRetry = 10 ) func (m *Manager) do(ctx context.Context) error { logger.Logf(true, "start runner manager") targets, err := datastore.ListTargets(ctx, m.ds) if err != nil { return fmt.Errorf("failed to get targets: %w", err) } logger.Logf(true, "found %d targets in datastore", len(targets)) for _, target := range targets { logger.Logf(true, "start to search runner in %s", target.Scope) if err := m.removeRunners(ctx, target); err != nil { logger.Logf(false, "failed to delete runners (target: %s): %+v", target.Scope, err) } } return nil } func (m *Manager) removeRunners(ctx context.Context, t datastore.Target) error { runners, err := m.ds.ListRunnersByTargetID(ctx, t.UUID) if err != nil { return fmt.Errorf("failed to retrieve list of running runner: %w", err) } var mode TemporaryMode if strings.EqualFold(m.runnerVersion, "latest") { mode = TemporaryEphemeral } else { _, m, err := GetRunnerTemporaryMode(m.runnerVersion) if err != nil { return fmt.Errorf("failed to get runner mode: %w", err) } mode = m } ghRunners, err := isRegisteredRunnerZeroInGitHub(ctx, t) if err != nil { return fmt.Errorf("failed to check number of registerd runner: %w", err) } if len(ghRunners) == 0 && len(runners) == 0 { switch mode { case TemporaryOnce: logger.Logf(false, "runner for queueing is not found in %s", t.Scope) if err := datastore.UpdateTargetStatus(ctx, m.ds, t.UUID, datastore.TargetStatusErr, ErrDescriptionRunnerForQueueingIsNotFound); err != nil { logger.Logf(false, "failed to update target status (target ID: %s): %+v\n", t.UUID, err) } default: if t.Status == datastore.TargetStatusErr && t.StatusDescription.Valid && strings.EqualFold(t.StatusDescription.String, ErrDescriptionRunnerForQueueingIsNotFound) { if err := datastore.UpdateTargetStatus(ctx, m.ds, t.UUID, datastore.TargetStatusActive, ""); err != nil { logger.Logf(false, "failed to update target status (target ID: %s): %+v\n", t.UUID, err) } } } return nil } sem := semaphore.NewWeighted(config.Config.MaxConcurrencyDeleting) var eg errgroup.Group ConcurrencyDeleting.Store(0) for _, runner := range runners { runner := runner c, _ := DeleteRetryCount.LoadOrStore(runner.UUID, 0) count, _ := c.(int) if count > MaxDeleteRetry { logger.Logf(false, "runner %s is retry count over %d, so will ignore", runner.UUID, MaxDeleteRetry) continue } if err := sem.Acquire(ctx, 1); err != nil { return fmt.Errorf("failed to Acquire: %w", err) } ConcurrencyDeleting.Add(1) eg.Go(func() error { cctx, cancel := context.WithTimeout(ctx, DeletingTimeout) defer cancel() defer func() { sem.Release(1) ConcurrencyDeleting.Add(-1) }() sleep := util.CalcRetryTime(count) if count > 0 { DeleteRunnerRetryTotal.WithLabelValues(runner.UUID.String()).Inc() DeleteRunnerBackoffDuration.WithLabelValues(runner.UUID.String()).Observe(sleep.Seconds()) } time.Sleep(sleep) if err := m.removeRunner(cctx, t, runner, ghRunners); err != nil { DeleteRetryCount.Store(runner.UUID, count+1) logger.Logf(false, "failed to delete runner: %+v", err) } else { DeleteRetryCount.Delete(runner.UUID) } return nil }) } if err := eg.Wait(); err != nil { return fmt.Errorf("failed to wait errgroup.Wait(): %w", err) } if t.Status == datastore.TargetStatusRunning { if err := datastore.UpdateTargetStatus(ctx, m.ds, t.UUID, datastore.TargetStatusActive, ""); err != nil { logger.Logf(false, "failed to update target status (target ID: %s): %+v\n", t.UUID, err) } } return nil } func (m *Manager) removeRunner(ctx context.Context, t datastore.Target, runner datastore.Runner, ghRunners []*github.Runner) error { if err := sanitizeRunnerMustRunningTime(runner); errors.Is(err, ErrNotWillDeleteRunner) { logger.Logf(false, "%s is not running MustRunningTime", runner.UUID) return nil } var mode TemporaryMode if strings.EqualFold(m.runnerVersion, "latest") { mode = TemporaryEphemeral } else { _, m, err := GetRunnerTemporaryMode(m.runnerVersion) if err != nil { return fmt.Errorf("failed to get runner mode: %w", err) } mode = m } switch mode { case TemporaryOnce: if err := m.removeRunnerModeOnce(ctx, t, runner, ghRunners); err != nil { return fmt.Errorf("failed to remove runner (mode once): %w", err) } case TemporaryEphemeral: if err := m.removeRunnerModeEphemeral(ctx, t, runner, ghRunners); err != nil { return fmt.Errorf("failed to remove runner (mode ephemeral): %w", err) } } return nil } func isRegisteredRunnerZeroInGitHub(ctx context.Context, t datastore.Target) ([]*github.Runner, error) { owner, repo := t.OwnerRepo() client, err := gh.NewClient(t.GitHubToken) if err != nil { return nil, fmt.Errorf("failed to create github client: %w", err) } ghRunners, err := gh.ListRunners(ctx, client, owner, repo) if err != nil { return nil, fmt.Errorf("failed to get list of runner in GitHub: %w", err) } return ghRunners, nil } var ( // ErrNotWillDeleteRunner is error message for "not will delete runner" ErrNotWillDeleteRunner = fmt.Errorf("not will delete runner") ) const ( // ErrDescriptionRunnerForQueueingIsNotFound is error message for datastore.StatusDescription "runner for queueing is not found" ErrDescriptionRunnerForQueueingIsNotFound = "runner for queueing is not found" ) var ( // StatusWillDelete will delete target in GitHub runners StatusWillDelete = "offline" // StatusSleep is sleeping runners StatusSleep = "online" ) func sanitizeGitHubRunner(ghRunner github.Runner, dsRunner datastore.Runner) error { if ghRunner.GetBusy() { // runner is busy, so not will delete return ErrNotWillDeleteRunner } switch ghRunner.GetStatus() { case StatusWillDelete: if err := sanitizeRunner(dsRunner, MustRunningTime); err != nil { logger.Logf(false, "%s is offline and not running %s, so not will delete (created_at: %s, now: %s)", dsRunner.UUID, MustRunningTime, dsRunner.CreatedAt, time.Now().UTC()) return fmt.Errorf("failed to sanitize will delete runner: %w", err) } return nil case StatusSleep: if err := sanitizeRunner(dsRunner, MustGoalTime); err != nil { logger.Logf(false, "%s is idle and not running %s, so not will delete (created_at: %s, now: %s)", dsRunner.UUID, MustGoalTime, dsRunner.CreatedAt, time.Now().UTC()) return fmt.Errorf("failed to sanitize idle runner: %w", err) } return nil } return ErrNotWillDeleteRunner } func sanitizeRunnerMustRunningTime(runner datastore.Runner) error { return sanitizeRunner(runner, MustRunningTime) } func sanitizeRunner(runner datastore.Runner, needTime time.Duration) error { spent := runner.CreatedAt.Add(needTime) now := time.Now().UTC() if !spent.Before(now) { return ErrNotWillDeleteRunner } return nil } // deleteRunnerWithGitHub delete runner in github, shoes, datastore. // runnerUUID is uuid in datastore, runnerID is id from GitHub. func (m *Manager) deleteRunnerWithGitHub(ctx context.Context, githubClient *github.Client, runner datastore.Runner, runnerID int64, owner, repo, runnerStatus string) error { logger.Logf(false, "will delete runner with GitHub: %s", runner.UUID.String()) isOrg := false if repo == "" { isOrg = true } if isOrg { if _, err := githubClient.Actions.RemoveOrganizationRunner(ctx, owner, runnerID); err != nil { return fmt.Errorf("failed to remove organization runner (runner uuid: %s): %+v", runner.UUID.String(), err) } } else { if _, err := githubClient.Actions.RemoveRunner(ctx, owner, repo, runnerID); err != nil { return fmt.Errorf("failed to remove repository runner (runner uuid: %s): %+v", runner.UUID.String(), err) } } if err := m.deleteRunner(ctx, runner, runnerStatus); err != nil { return fmt.Errorf("failed to delete runner: %w", err) } return nil } // deleteRunner delete runner in shoes, datastore. func (m *Manager) deleteRunner(ctx context.Context, runner datastore.Runner, runnerStatus string) error { logger.Logf(false, "will delete runner: %s", runner.UUID.String()) client, teardown, err := shoes.GetClient() if err != nil { return fmt.Errorf("failed to get plugin client: %w", err) } defer teardown() labels, err := gh.ExtractRunsOnLabels([]byte(runner.RequestWebhook)) if err != nil { return fmt.Errorf("failed to extract labels: %w", err) } if err := client.DeleteInstance(ctx, runner.CloudID, labels); err != nil { if status.Code(errors.Unwrap(err)) == codes.NotFound { logger.Logf(true, "%s is not found, will ignore from shoes", runner.UUID) } else { return fmt.Errorf("failed to delete instance: %w", err) } } now := time.Now().UTC() if err := m.ds.DeleteRunner(ctx, runner.UUID, now, ToReason(runnerStatus)); err != nil { return fmt.Errorf("failed to remove runner from datastore (runner uuid: %s): %+v", runner.UUID.String(), err) } return nil } ================================================ FILE: pkg/runner/runner_delete_ephemeral.go ================================================ package runner import ( "context" "errors" "fmt" "github.com/google/go-github/v80/github" "github.com/whywaita/myshoes/pkg/datastore" "github.com/whywaita/myshoes/pkg/gh" "github.com/whywaita/myshoes/pkg/logger" ) // removeRunnerModeEphemeral remove runner that created by --ephemeral flag. // --ephemeral flag is delete self-hosted runner when end of job. So, The origin list of runner from datastore. func (m *Manager) removeRunnerModeEphemeral(ctx context.Context, t datastore.Target, runner datastore.Runner, ghRunners []*github.Runner) error { owner, repo := t.OwnerRepo() client, err := gh.NewClient(t.GitHubToken) if err != nil { return fmt.Errorf("failed to create github client: %w", err) } ghRunner, err := gh.ExistGitHubRunnerWithRunner(ghRunners, ToName(runner.UUID.String())) switch { case errors.Is(err, gh.ErrNotFound): // deleted in GitHub, It's completed if err := m.deleteRunner(ctx, runner, StatusWillDelete); err != nil { if err := datastore.UpdateTargetStatus(ctx, m.ds, t.UUID, datastore.TargetStatusErr, ""); err != nil { logger.Logf(false, "failed to update target status (target ID: %s): %+v\n", t.UUID, err) } return fmt.Errorf("failed to delete runner: %w", err) } return nil case err != nil: return fmt.Errorf("failed to check runner exist in GitHub (runner: %s): %w", runner.UUID, err) } if err := sanitizeGitHubRunner(*ghRunner, runner); err != nil { if errors.Is(err, ErrNotWillDeleteRunner) { return nil } return fmt.Errorf("failed to check runner of status: %w", err) } if err := m.deleteRunnerWithGitHub(ctx, client, runner, ghRunner.GetID(), owner, repo, ghRunner.GetStatus()); err != nil { if err := datastore.UpdateTargetStatus(ctx, m.ds, t.UUID, datastore.TargetStatusErr, ""); err != nil { logger.Logf(false, "failed to update target status (target ID: %s): %+v\n", t.UUID, err) } return fmt.Errorf("failed to delete runner with GitHub: %w", err) } return nil } ================================================ FILE: pkg/runner/runner_delete_once.go ================================================ package runner import ( "context" "errors" "fmt" "github.com/google/go-github/v80/github" "github.com/whywaita/myshoes/pkg/datastore" "github.com/whywaita/myshoes/pkg/gh" "github.com/whywaita/myshoes/pkg/logger" ) // removeRunnerModeOnce remove runner that created by --once flag. // --once flag is not delete self-hosted runner when end of job. So, The origin list of runner from GitHub. func (m *Manager) removeRunnerModeOnce(ctx context.Context, t datastore.Target, runner datastore.Runner, ghRunners []*github.Runner) error { owner, repo := t.OwnerRepo() client, err := gh.NewClient(t.GitHubToken) if err != nil { return fmt.Errorf("failed to create github client: %w", err) } ghRunner, err := gh.ExistGitHubRunnerWithRunner(ghRunners, ToName(runner.UUID.String())) switch { case errors.Is(err, gh.ErrNotFound): logger.Logf(false, "NotFound in GitHub, so will delete in datastore without GitHub (runner: %s)", runner.UUID.String()) if err := m.deleteRunner(ctx, runner, StatusWillDelete); err != nil { if err := datastore.UpdateTargetStatus(ctx, m.ds, t.UUID, datastore.TargetStatusErr, ""); err != nil { logger.Logf(false, "failed to update target status (target ID: %s): %+v\n", t.UUID, err) } return fmt.Errorf("failed to delete runner: %w", err) } return nil case err != nil: return fmt.Errorf("failed to check runner exist in GitHub (runner: %s): %w", runner.UUID, err) } if err := sanitizeGitHubRunner(*ghRunner, runner); err != nil { if errors.Is(err, ErrNotWillDeleteRunner) { return nil } return fmt.Errorf("failed to check runner of status: %w", err) } if err := m.deleteRunnerWithGitHub(ctx, client, runner, ghRunner.GetID(), owner, repo, ghRunner.GetStatus()); err != nil { if err := datastore.UpdateTargetStatus(ctx, m.ds, t.UUID, datastore.TargetStatusErr, ""); err != nil { logger.Logf(false, "failed to update target status (target ID: %s): %+v\n", t.UUID, err) } return fmt.Errorf("failed to delete runner with GitHub: %w", err) } return nil } ================================================ FILE: pkg/runner/token_update.go ================================================ package runner import ( "context" "fmt" "time" "github.com/whywaita/myshoes/pkg/datastore" "github.com/whywaita/myshoes/pkg/gh" "github.com/whywaita/myshoes/pkg/logger" ) func (m *Manager) doTargetToken(ctx context.Context) error { logger.Logf(true, "start refresh token") targets, err := datastore.ListTargets(ctx, m.ds) if err != nil { return fmt.Errorf("failed to get targets: %w", err) } for _, target := range targets { needRefreshTime := target.TokenExpiredAt.Add(-1 * NeedRefreshToken) if time.Now().Before(needRefreshTime) { // no need refresh continue } // do refresh logger.Logf(true, "%s need to update GitHub token, will be update", target.UUID) clientApps, err := gh.NewClientGitHubApps() if err != nil { logger.Logf(false, "failed to create a client from Apps: %+v", err) continue } installationID, err := gh.IsInstalledGitHubApp(ctx, target.Scope) if err != nil { logger.Logf(false, "failed to get installationID: %+v", err) continue } // TODO: replace to ghinstallation.AppTransport token, expiredAt, err := gh.GenerateGitHubAppsToken(ctx, clientApps, installationID, target.Scope) if err != nil { logger.Logf(false, "failed to get Apps Token: %+v", err) continue } if err := m.ds.UpdateToken(ctx, target.UUID, token, *expiredAt); err != nil { logger.Logf(false, "failed to update token (target: %s): %+v", target.UUID, err) if err := datastore.UpdateTargetStatus(ctx, m.ds, target.UUID, datastore.TargetStatusErr, "can not update token"); err != nil { logger.Logf(false, "failed to update target status (target ID: %s): %+v\n", target.UUID, err) } continue } } return nil } ================================================ FILE: pkg/runner/util.go ================================================ package runner import ( "fmt" "strings" uuid "github.com/satori/go.uuid" "github.com/whywaita/myshoes/pkg/datastore" ) // ToName convert uuid to runner name func ToName(u string) string { return fmt.Sprintf("myshoes-%s", u) } // ToUUID convert runner name to uuid func ToUUID(name string) (uuid.UUID, error) { u := strings.TrimPrefix(name, "myshoes-") return uuid.FromString(u) } // ToReason convert status from GitHub to datastore.RunnerStatus func ToReason(status string) datastore.RunnerStatus { switch status { case StatusWillDelete: // is offline return datastore.RunnerStatusCompleted case StatusSleep: // is idle, reach hard limit return datastore.RunnerStatusReachHardLimit } return "" } ================================================ FILE: pkg/shoes/shoes.go ================================================ package shoes import ( "context" "fmt" "os" "os/exec" "github.com/hashicorp/go-plugin" pb "github.com/whywaita/myshoes/api/proto.go" "github.com/whywaita/myshoes/pkg/config" "github.com/whywaita/myshoes/pkg/datastore" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) // GetClient retrieve ShoesClient use shoes-plugin func GetClient() (Client, func(), error) { Handshake := plugin.HandshakeConfig{ ProtocolVersion: 1, MagicCookieKey: "SHOES_PLUGIN_MAGIC_COOKIE", MagicCookieValue: "are_you_a_shoes?", } PluginMap := map[string]plugin.Plugin{ "shoes_grpc": &Plugin{}, } client := plugin.NewClient(&plugin.ClientConfig{ HandshakeConfig: Handshake, Plugins: PluginMap, Cmd: exec.Command(config.Config.ShoesPluginPath), Managed: true, Stderr: os.Stderr, SyncStdout: os.Stdout, SyncStderr: os.Stderr, AllowedProtocols: []plugin.Protocol{plugin.ProtocolGRPC}, }) rpcClient, err := client.Client() if err != nil { return nil, nil, fmt.Errorf("failed to get shoes client: %w", err) } raw, err := rpcClient.Dispense("shoes_grpc") if err != nil { return nil, nil, fmt.Errorf("failed to shoes client instance: %w", err) } return raw.(Client), client.Kill, nil } // Plugin is plugin implement type Plugin struct { plugin.Plugin Impl Client } // GRPCServer is server func (p *Plugin) GRPCServer(broker *plugin.GRPCBroker, s *grpc.Server) error { return nil } // GRPCClient is client func (p *Plugin) GRPCClient(ctx context.Context, broker *plugin.GRPCBroker, c *grpc.ClientConn) (interface{}, error) { return &GRPCClient{client: pb.NewShoesClient(c)}, nil } // Client is plugin client interface type Client interface { AddInstance(ctx context.Context, runnerID, setupScript string, resourceType datastore.ResourceType, labels []string) (string, string, string, datastore.ResourceType, error) DeleteInstance(ctx context.Context, cloudID string, labels []string) error } // GRPCClient is plugin client implement type GRPCClient struct { client pb.ShoesClient } // AddInstance create instance for runner func (c *GRPCClient) AddInstance(ctx context.Context, runnerName, setupScript string, resourceType datastore.ResourceType, labels []string) (string, string, string, datastore.ResourceType, error) { req := &pb.AddInstanceRequest{ RunnerName: runnerName, SetupScript: setupScript, ResourceType: resourceType.ToPb(), Labels: labels, } resp, err := c.client.AddInstance(ctx, req) if err != nil { // will delete a job if labels of a job are invalid if stat, _ := status.FromError(err); stat.Code() == codes.InvalidArgument { return "", "", "", datastore.ResourceTypeUnknown, err } return "", "", "", datastore.ResourceTypeUnknown, fmt.Errorf("failed to AddInstance: %w", err) } return resp.CloudId, resp.IpAddress, resp.ShoesType, datastore.UnmarshalResourceType(resp.ResourceType), nil } // DeleteInstance delete instance for runner func (c *GRPCClient) DeleteInstance(ctx context.Context, cloudID string, labels []string) error { req := &pb.DeleteInstanceRequest{ CloudId: cloudID, Labels: labels, } _, err := c.client.DeleteInstance(ctx, req) if err != nil { return fmt.Errorf("failed to DeleteInstance: %w", err) } return nil } ================================================ FILE: pkg/starter/README.md ================================================ ## starter starter is a dispatcher for running job ================================================ FILE: pkg/starter/error.go ================================================ package starter import "errors" type Error struct { kind internalError err error } func (e Error) Error() string { return e.kind.String() + ": " + e.err.Error() } func (e Error) Unwrap() error { return e.err } type internalError int const ( errorInvalidLabel internalError = iota ) func (i internalError) String() string { switch i { case errorInvalidLabel: return "invalid label" default: return "unknown error" } } var ( ErrInvalidLabel = Error{kind: errorInvalidLabel, err: nil} ) func NewInvalidLabel(err error) error { e := ErrInvalidLabel e.err = err return e } func (e Error) Is(target error) bool { var t Error ok := errors.As(target, &t) if !ok { return false } return e.kind == t.kind } ================================================ FILE: pkg/starter/metric.go ================================================ package starter import ( "fmt" "sync" "github.com/whywaita/myshoes/pkg/datastore" "github.com/whywaita/myshoes/pkg/gh" ) var ( // DeletedJobMap is map for deleted jobs. key: runs_on, value: number of deleted jobs DeletedJobMap = sync.Map{} ) func incrementDeleteJobMap(j datastore.Job) error { runsOnConcat, err := gh.ConcatLabels(j.CheckEventJSON) if err != nil { return fmt.Errorf("failed to concat labels: %+v", err) } v, ok := DeletedJobMap.Load(runsOnConcat) if !ok { DeletedJobMap.Store(runsOnConcat, 1) return nil } DeletedJobMap.Store(runsOnConcat, v.(int)+1) return nil } ================================================ FILE: pkg/starter/metrics.go ================================================ package starter import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" ) var ( // AddInstanceBackoffDuration is histogram of exponential backoff duration for adding instance AddInstanceBackoffDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ Namespace: "myshoes", Subsystem: "starter", Name: "add_instance_backoff_duration_seconds", Help: "Histogram of exponential backoff duration in seconds for adding instance", Buckets: prometheus.ExponentialBuckets(1, 2, 10), // 1s, 2s, 4s, 8s, 16s, 32s, 64s, 128s, 256s, 512s }, []string{"job_uuid"}) // AddInstanceRetryTotal is counter of total retries for adding instance AddInstanceRetryTotal = promauto.NewCounterVec(prometheus.CounterOpts{ Namespace: "myshoes", Subsystem: "starter", Name: "add_instance_retry_total", Help: "Total number of retries for adding instance", }, []string{"job_uuid"}) ) ================================================ FILE: pkg/starter/safety/README.md ================================================ # safety safety is interface of check to enable runner start. ================================================ FILE: pkg/starter/safety/safety.go ================================================ package safety import ( "github.com/whywaita/myshoes/pkg/datastore" ) // Safety is interface for safety type Safety interface { // Check check that can create a runner. if can create a runner, return true. Check(job *datastore.Job) (bool, error) } ================================================ FILE: pkg/starter/safety/unlimited/unlimited.go ================================================ package unlimited import "github.com/whywaita/myshoes/pkg/datastore" // Unlimited is implement of safety. // Unlimited has not safety, so create a runner quickly. type Unlimited struct{} // Check is not limited func (u Unlimited) Check(job *datastore.Job) (bool, error) { return true, nil } ================================================ FILE: pkg/starter/scripts/RunnerService.js ================================================ #!/usr/bin/env node // Copyright (c) GitHub. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. var childProcess = require("child_process"); var path = require("path") var supported = ['linux', 'darwin'] if (supported.indexOf(process.platform) == -1) { console.log('Unsupported platform: ' + process.platform); console.log('Supported platforms are: ' + supported.toString()); process.exit(1); } var stopping = false; var listener = null; var runService = function() { var listenerExePath = path.join(__dirname, '../bin/Runner.Listener'); var interactive = process.argv[2] === "interactive"; if(!stopping) { try { if (interactive) { console.log('Starting Runner listener interactively'); listener = childProcess.spawn(listenerExePath, ['run'].concat(process.argv.slice(3)), { env: process.env }); } else { console.log('Starting Runner listener with startup type: service'); listener = childProcess.spawn(listenerExePath, ['run', '--startuptype', 'service'].concat(process.argv.slice(2)), { env: process.env }); } console.log('Started listener process'); listener.stdout.on('data', (data) => { process.stdout.write(data.toString('utf8')); }); listener.stderr.on('data', (data) => { process.stdout.write(data.toString('utf8')); }); listener.on('close', (code) => { console.log(`Runner listener exited with error code ${code}`); if (code === 0) { console.log('Runner listener exit with 0 return code, stop the service, no retry needed.'); stopping = true; } else if (code === 1) { console.log('Runner listener exit with terminated error, stop the service, no retry needed.'); stopping = true; } else if (code === 2) { console.log('Runner listener exit with retryable error, re-launch runner in 5 seconds.'); } else if (code === 3) { console.log('Runner listener exit because of updating, re-launch runner in 5 seconds.'); } else { console.log('Runner listener exit with undefined return code, re-launch runner in 5 seconds.'); } if(!stopping) { setTimeout(runService, 5000); } }); } catch(ex) { console.log(ex); } } } runService(); console.log('Started running service'); var gracefulShutdown = function(code) { console.log('Shutting down runner listener'); stopping = true; if (listener) { console.log('Sending SIGINT to runner listener to stop'); listener.kill('SIGINT'); // TODO wait for 30 seconds and send a SIGKILL } } process.on('SIGINT', () => { gracefulShutdown(0); }); process.on('SIGTERM', () => { gracefulShutdown(0); }); ================================================ FILE: pkg/starter/scripts.go ================================================ package starter import ( "bytes" "compress/gzip" "context" _ "embed" // TODO: "encoding/base64" "fmt" "strings" "text/template" "github.com/whywaita/myshoes/pkg/config" "github.com/whywaita/myshoes/pkg/gh" "github.com/whywaita/myshoes/pkg/runner" ) //go:embed scripts/RunnerService.js var runnerService string func getPatchedFiles() (string, error) { return runnerService, nil } type templateCompressedScriptValue struct { CompressedScript string RunnerBaseDirectory string } func (s *Starter) GetSetupScript(ctx context.Context, targetScope, runnerName string) (string, error) { rawScript, err := s.getSetupRawScript(ctx, targetScope, runnerName) if err != nil { return "", fmt.Errorf("failed to get raw setup scripts: %w", err) } var compressedScript bytes.Buffer gz := gzip.NewWriter(&compressedScript) if _, err := gz.Write([]byte(rawScript)); err != nil { return "", fmt.Errorf("failed to compress gzip: %w", err) } if err := gz.Flush(); err != nil { return "", fmt.Errorf("failed to flush gzip: %w", err) } if err := gz.Close(); err != nil { return "", fmt.Errorf("failed to close gzip: %w", err) } encoded := base64.StdEncoding.EncodeToString(compressedScript.Bytes()) v := templateCompressedScriptValue{ CompressedScript: encoded, RunnerBaseDirectory: config.Config.RunnerBaseDirectory, } t, err := template.New("templateCompressedScript").Parse(templateCompressedScript) if err != nil { return "", fmt.Errorf("failed to create template: %w", err) } var buff bytes.Buffer if err := t.Execute(&buff, v); err != nil { return "", fmt.Errorf("failed to execute compressed script: %w", err) } return buff.String(), nil } func (s *Starter) getSetupRawScript(ctx context.Context, targetScope, runnerName string) (string, error) { runnerUser := config.Config.RunnerUser targetRunnerVersion := s.runnerVersion if strings.EqualFold(s.runnerVersion, "latest") { latestVersion, err := gh.GetLatestRunnerVersion(ctx, targetScope) if err != nil { return "", fmt.Errorf("failed to get latest version of actions/runner: %w", err) } targetRunnerVersion = latestVersion } runnerVersion, runnerTemporaryMode, err := runner.GetRunnerTemporaryMode(targetRunnerVersion) if err != nil { return "", fmt.Errorf("failed to get runner version: %w", err) } runnerServiceJs, err := getPatchedFiles() if err != nil { return "", fmt.Errorf("failed to get patched files: %w", err) } installationID, err := gh.IsInstalledGitHubApp(ctx, targetScope) if err != nil { return "", fmt.Errorf("failed to get installlation id: %w", err) } token, err := gh.GetRunnerRegistrationToken(ctx, installationID, targetScope) if err != nil { return "", fmt.Errorf("failed to generate runner register token: %w", err) } var labels []string // The "dependabot" label is always added to ensure compatibility with Dependabot-related workflows. labels = append(labels, "dependabot") v := templateCreateLatestRunnerOnceValue{ Scope: targetScope, GHEDomain: config.Config.GitHubURL, RunnerRegistrationToken: token, RunnerName: runnerName, RunnerUser: runnerUser, RunnerVersion: runnerVersion, RunnerServiceJS: runnerServiceJs, RunnerArg: runnerTemporaryMode.StringFlag(), AdditionalLabels: labelsToOneLine(labels), RunnerBaseDirectory: config.Config.RunnerBaseDirectory, } t, err := template.New("templateCreateLatestRunnerOnce").Parse(templateCreateLatestRunnerOnce) if err != nil { return "", fmt.Errorf("failed to create template") } var buff bytes.Buffer if err := t.Execute(&buff, v); err != nil { return "", fmt.Errorf("failed to execute scripts: %w", err) } return buff.String(), nil } func labelsToOneLine(labels []string) string { if len(labels) == 0 { return "" } return fmt.Sprintf(",%s", strings.Join(labels, ",")) } const templateCompressedScript = `#!/bin/bash set -e # main script compressed base64 and gzip export COMPRESSED_SCRIPT={{.CompressedScript}} export MAIN_SCRIPT_PATH={{.RunnerBaseDirectory}}/main.sh echo ${COMPRESSED_SCRIPT} | base64 -d | gzip -d > ${MAIN_SCRIPT_PATH} chmod +x ${MAIN_SCRIPT_PATH} bash -c ${MAIN_SCRIPT_PATH}` type templateCreateLatestRunnerOnceValue struct { Scope string GHEDomain string RunnerRegistrationToken string RunnerName string RunnerUser string RunnerVersion string RunnerServiceJS string RunnerArg string AdditionalLabels string RunnerBaseDirectory string } // templateCreateLatestRunnerOnce is script template of setup runner. // need to set runnerUser if execute using root permission. (for example, use cloud-init) // original script: https://github.com/actions/runner/blob/80bf68db812beb298b7534012b261e6f222e004a/scripts/create-latest-svc.sh const templateCreateLatestRunnerOnce = `#!/bin/bash set -e runner_scope={{.Scope}} ghe_hostname={{.GHEDomain}} runner_name={{.RunnerName}} RUNNER_TOKEN={{.RunnerRegistrationToken}} RUNNER_USER={{.RunnerUser}} RUNNER_VERSION={{.RunnerVersion}} RUNNER_BASE_DIRECTORY={{.RunnerBaseDirectory}} sudo_prefix="" if [ $(id -u) -eq 0 ]; then # if root sudo_prefix="sudo -E -u ${RUNNER_USER} " fi echo "Configuring runner @ ${runner_scope}" #--------------------------------------- # Validate Environment #--------------------------------------- runner_plat=linux [ ! -z "$(which sw_vers)" ] && runner_plat=osx; function fatal() { echo "error: $1" >&2 exit 1 } function configure_environment() { export HOME="/home/${RUNNER_USER}" if [ "${runner_plat}" = "osx" ]; then export HOME="/Users/${RUNNER_USER}" fi } function install_jq() { echo "jq is not installed, will be install jq." if [ -e /etc/debian_version ] || [ -e /etc/debian_release ]; then sudo apt-get update -y -qq sudo apt-get install -y jq elif [ -e /etc/redhat-release ]; then sudo yum install -y jq fi if [ "${runner_plat}" = "osx" ]; then brew install jq fi } function install_docker() { echo "docker is not installed, will be install docker." if [ -e /etc/debian_version ] || [ -e /etc/debian_release ]; then sudo apt-get update -y -qq sudo apt-get install -y docker.io fi if [ "${runner_plat}" = "osx" ]; then echo "No install in macOS, It is same that GitHub-hosted" fi } function get_runner_file_name() { runner_version=$1 runner_plat=$2 trimmed_runner_version=$(echo ${RUNNER_VERSION:1}) if [ "${runner_plat}" = "linux" ]; then echo "actions-runner-${runner_plat}-x64-${trimmed_runner_version}.tar.gz" fi if [ "${runner_plat}" = "osx" ]; then runner_arch=x64 [ "$(uname -m)" = "arm64" ] && runner_arch=arm64; echo "actions-runner-${runner_plat}-${runner_arch}-${trimmed_runner_version}.tar.gz" fi } function download_runner() { runner_version=$1 runner_file=$2 runner_url="https://github.com/actions/runner/releases/download/${runner_version}/${runner_file}" echo "Downloading ${runner_version} for ${runner_plat} ..." echo $runner_url curl -O -L ${runner_url} ls -la *.tar.gz } function extract_runner() { runner_file=$1 runner_user=$2 echo "Extracting ${runner_file} to ./runner" tar xzf "./${runner_file}" -C runner # export of pass if [ $(id -u) -eq 0 ]; then chown -R ${runner_user} ./runner fi } if [ -z "${runner_scope}" ]; then fatal "supply scope as argument 1"; fi which curl || fatal "curl required. Please install in PATH with apt-get, brew, etc" which jq || install_jq which jq || fatal "jq required. Please install in PATH with apt-get, brew, etc" which docker || install_docker configure_environment cd ${RUNNER_BASE_DIRECTORY} ${sudo_prefix}mkdir -p runner #--------------------------------------- # Download latest released and extract #--------------------------------------- echo echo "Downloading latest runner ..." runner_file=$(get_runner_file_name ${RUNNER_VERSION} ${runner_plat}) if [ -f "${RUNNER_BASE_DIRECTORY}/runner/config.sh" ]; then # already extracted echo "${RUNNER_BASE_DIRECTORY}/runner/config.sh exists. skipping download and extract." elif [ -f "/usr/local/etc/runner-${RUNNER_VERSION}/config.sh" ]; then echo "runner-${RUNNER_VERSION} cache is found. skipping download and extract." rm -r ./runner mv /usr/local/etc/runner-${RUNNER_VERSION} ./runner elif [ -f "${runner_file}" ]; then echo "${runner_file} exists. skipping download." extract_runner ${runner_file} ${RUNNER_USER} elif [ -f "/usr/local/etc/${runner_file}" ]; then echo "${runner_file} cache is found. skipping download." mv /usr/local/etc/${runner_file} ./ extract_runner ${runner_file} ${RUNNER_USER} else download_runner ${RUNNER_VERSION} ${runner_file} extract_runner ${runner_file} ${RUNNER_USER} fi cd ${RUNNER_BASE_DIRECTORY}/runner #--------------------------------------- # Unattend config #--------------------------------------- runner_url="https://github.com/${runner_scope}" if [ -n "${ghe_hostname}" ]; then runner_url="${ghe_hostname}/${runner_scope}" fi echo echo "Configuring ${runner_name} @ $runner_url" {{ if eq .RunnerArg "--once" -}} echo "./config.sh --unattended --url $runner_url --token *** --name $runner_name --labels myshoes" ${sudo_prefix}bash -c "source /etc/environment; ./config.sh --unattended --url $runner_url --token $RUNNER_TOKEN --name $runner_name --labels myshoes{{.AdditionalLabels}}" {{ else -}} echo "./config.sh --unattended --url $runner_url --token *** --name $runner_name --labels myshoes {{.RunnerArg}}" ${sudo_prefix}bash -c "source /etc/environment; ./config.sh --unattended --url $runner_url --token $RUNNER_TOKEN --name $runner_name --labels myshoes{{.AdditionalLabels}} {{.RunnerArg}}" {{ end }} #--------------------------------------- # patch once commands #--------------------------------------- echo "apply patch file" cat << EOF > ./bin/runsvc.sh #!/bin/bash # convert SIGTERM signal to SIGINT # for more info on how to propagate SIGTERM to a child process see: http://veithen.github.io/2014/11/16/sigterm-propagation.html trap 'kill -INT \$PID' TERM INT if [ -f ".path" ]; then # configure export PATH=\$(cat .path) echo ".path=\${PATH}" fi # insert anything to setup env when running as a service # run the host process which keep the listener alive NODE_PATH="./externals/node20/bin/node" if [ ! -e "\${NODE_PATH}" ]; then NODE_PATH="./externals/node16/bin/node" fi \${NODE_PATH} ./bin/RunnerService.js \$* & PID=\$! wait \$PID trap - TERM INT wait \$PID EOF cat << 'EOF' > ./bin/RunnerService.js {{.RunnerServiceJS}} EOF #--------------------------------------- # Configure run commands #--------------------------------------- # Configure job management hooks if script files exist if [ -e "/myshoes-actions-runner-hook-job-started.sh" ]; then export ACTIONS_RUNNER_HOOK_JOB_STARTED="/myshoes-actions-runner-hook-job-started.sh" fi if [ -e "/myshoes-actions-runner-hook-job-completed.sh" ]; then export ACTIONS_RUNNER_HOOK_JOB_COMPLETED="/myshoes-actions-runner-hook-job-completed.sh" fi #--------------------------------------- # run! #--------------------------------------- # GitHub-hosted runner load /etc/environment in /opt/runner/provisioner/provisioner. # So, we need to load /etc/environment for job on self-hosted runner. {{ if eq .RunnerArg "--once" -}} echo 'bash -c "source /etc/environment; ./bin/runsvc.sh {{.RunnerArg}}"' ${sudo_prefix}bash -c "source /etc/environment; ./bin/runsvc.sh {{.RunnerArg}}" {{ else -}} echo 'bash -c "source /etc/environment; ./bin/runsvc.sh"' ${sudo_prefix}bash -c "source /etc/environment; ./bin/runsvc.sh" {{ end }}` ================================================ FILE: pkg/starter/starter.go ================================================ package starter import ( "context" "database/sql" "encoding/json" "errors" "fmt" "net/url" "sync" "sync/atomic" "time" "github.com/google/go-github/v80/github" "golang.org/x/sync/errgroup" "golang.org/x/sync/semaphore" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" uuid "github.com/satori/go.uuid" "github.com/whywaita/myshoes/internal/util" "github.com/whywaita/myshoes/pkg/config" "github.com/whywaita/myshoes/pkg/datastore" "github.com/whywaita/myshoes/pkg/gh" "github.com/whywaita/myshoes/pkg/logger" "github.com/whywaita/myshoes/pkg/runner" "github.com/whywaita/myshoes/pkg/shoes" "github.com/whywaita/myshoes/pkg/starter/safety" ) var ( // CountRunning is count of running semaphore CountRunning atomic.Int64 // CountWaiting is count of waiting job CountWaiting atomic.Int64 // CountRescued is count of rescued job per target CountRescued = sync.Map{} inProgress = sync.Map{} // AddInstanceRetryCount is count of retry to add instance AddInstanceRetryCount = sync.Map{} ) // Starter is dispatcher for running job type Starter struct { ds datastore.Datastore safety safety.Safety runnerVersion string notifyEnqueueCh <-chan struct{} } // New create starter instance func New(ds datastore.Datastore, s safety.Safety, runnerVersion string, notifyEnqueueCh <-chan struct{}) *Starter { return &Starter{ ds: ds, safety: s, runnerVersion: runnerVersion, notifyEnqueueCh: notifyEnqueueCh, } } // Loop is main loop for starter func (s *Starter) Loop(ctx context.Context) error { logger.Logf(false, "start starter loop") ch := make(chan datastore.Job) eg, ctx := errgroup.WithContext(ctx) eg.Go(func() error { ticker := time.NewTicker(10 * time.Second) defer ticker.Stop() for { select { case <-ticker.C: s.reRunWorkflow(ctx) case <-ctx.Done(): return nil } } }) eg.Go(func() error { if err := s.run(ctx, ch); err != nil { return fmt.Errorf("faied to start processor: %w", err) } return nil }) eg.Go(func() error { ticker := time.NewTicker(10 * time.Second) defer ticker.Stop() for { select { case <-ticker.C: if err := s.dispatcher(ctx, ch); err != nil { logger.Logf(false, "failed to starter: %+v", err) } case <-s.notifyEnqueueCh: ticker.Reset(10 * time.Second) if err := s.dispatcher(ctx, ch); err != nil { logger.Logf(false, "failed to starter: %+v", err) } case <-ctx.Done(): return nil } } }) if err := eg.Wait(); err != nil { return fmt.Errorf("failed to errgroup wait: %w", err) } return nil } func (s *Starter) dispatcher(ctx context.Context, ch chan datastore.Job) error { logger.Logf(true, "start to check starter") jobs, err := s.ds.ListJobs(ctx) if err != nil { return fmt.Errorf("failed to get jobs: %w", err) } for _, j := range jobs { // send to processor ch <- j } return nil } func (s *Starter) run(ctx context.Context, ch chan datastore.Job) error { sem := semaphore.NewWeighted(config.Config.MaxConnectionsToBackend) // Processor for { select { case job := <-ch: // receive job from dispatcher if _, ok := inProgress.Load(job.UUID); ok { // this job is in progress, skip continue } c, _ := AddInstanceRetryCount.LoadOrStore(job.UUID, 0) count, _ := c.(int) runID, jobID, err := extractWorkflowIDs(job) if err != nil { logger.Logf(true, "found new job: %s (repo: %s)", job.UUID, job.Repository) } else { logger.Logf(true, "found new job: %s (gh_run_id: %d, gh_job_id: %d, repo: %s)", job.UUID, runID, jobID, job.Repository) } CountWaiting.Add(1) if err := sem.Acquire(ctx, 1); err != nil { return fmt.Errorf("failed to Acquire: %w", err) } CountWaiting.Add(-1) CountRunning.Add(1) inProgress.Store(job.UUID, struct{}{}) sleep := util.CalcRetryTime(count) if count > 0 { AddInstanceRetryTotal.WithLabelValues(job.UUID.String()).Inc() AddInstanceBackoffDuration.WithLabelValues(job.UUID.String()).Observe(sleep.Seconds()) } go func(job datastore.Job, sleep time.Duration) { defer func() { sem.Release(1) inProgress.Delete(job.UUID) CountRunning.Add(-1) }() time.Sleep(sleep) if err := s.ProcessJob(ctx, job); err != nil { AddInstanceRetryCount.Store(job.UUID, count+1) logger.Logf(false, "failed to process job: %+v\n", err) } else { AddInstanceRetryCount.Delete(job.UUID) } }(job, sleep) case <-ctx.Done(): return nil } } } // extractWorkflowIDs extracts GitHub workflow run ID and job ID from a datastore.Job func extractWorkflowIDs(job datastore.Job) (runID int64, jobID int64, err error) { webhookEvent, err := github.ParseWebHook("workflow_job", []byte(job.CheckEventJSON)) if err != nil { return 0, 0, fmt.Errorf("failed to parse webhook: %w", err) } workflowJob, ok := webhookEvent.(*github.WorkflowJobEvent) if !ok { return 0, 0, fmt.Errorf("failed to cast to WorkflowJobEvent") } if workflowJob.GetWorkflowJob() == nil { return 0, 0, fmt.Errorf("workflow job is nil") } return workflowJob.GetWorkflowJob().GetRunID(), workflowJob.GetWorkflowJob().GetID(), nil } // ProcessJob is process job func (s *Starter) ProcessJob(ctx context.Context, job datastore.Job) error { runID, jobID, err := extractWorkflowIDs(job) if err != nil { logger.Logf(false, "start job (job id: %s, repo: %s)\n", job.UUID.String(), job.Repository) } else { logger.Logf(false, "start job (job id: %s, gh_run_id: %d, gh_job_id: %d, repo: %s)\n", job.UUID.String(), runID, jobID, job.Repository) } isOK, err := s.safety.Check(&job) if err != nil { return fmt.Errorf("failed to check safety: %w", err) } if !isOK { // is not ok, save job return nil } if err := datastore.UpdateTargetStatus(ctx, s.ds, job.TargetID, datastore.TargetStatusRunning, ""); err != nil { return fmt.Errorf("failed to update target status (target ID: %s, job ID: %s): %w", job.TargetID, job.UUID, err) } target, err := s.ds.GetTarget(ctx, job.TargetID) if err != nil { return fmt.Errorf("failed to retrieve relational target: (target ID: %s, job ID: %s): %w", job.TargetID, job.UUID, err) } cctx, cancel := context.WithTimeout(ctx, runner.MustRunningTime) defer cancel() cloudID, ipAddress, shoesType, resourceType, err := s.bung(cctx, job, *target) if err != nil { runID2, jobID2, extractErr := extractWorkflowIDs(job) if extractErr != nil { logger.Logf(false, "failed to bung (target ID: %s, job ID: %s): %+v", job.TargetID, job.UUID, err) } else { logger.Logf(false, "failed to bung (target ID: %s, job ID: %s, gh_run_id: %d, gh_job_id: %d): %+v", job.TargetID, job.UUID, runID2, jobID2, err) } if errors.Is(err, ErrInvalidLabel) { if extractErr != nil { logger.Logf(false, "invalid argument. so will delete (job ID: %s)", job.UUID) } else { logger.Logf(false, "invalid argument. so will delete (job ID: %s, gh_run_id: %d, gh_job_id: %d)", job.UUID, runID2, jobID2) } if err := s.ds.DeleteJob(ctx, job.UUID); err != nil { logger.Logf(false, "failed to delete job: %+v\n", err) if err := datastore.UpdateTargetStatus(ctx, s.ds, job.TargetID, datastore.TargetStatusErr, fmt.Sprintf("job id: %s", job.UUID)); err != nil { return fmt.Errorf("failed to update target status (target ID: %s, job ID: %s): %w", job.TargetID, job.UUID, err) } return fmt.Errorf("failed to delete job: %w", err) } if err := incrementDeleteJobMap(job); err != nil { return fmt.Errorf("failed to increment delete metrics: %w", err) } return nil } if err := datastore.UpdateTargetStatus(ctx, s.ds, job.TargetID, datastore.TargetStatusErr, fmt.Sprintf("failed to create an instance (job ID: %s)", job.UUID)); err != nil { return fmt.Errorf("failed to update target status (target ID: %s, job ID: %s): %w", job.TargetID, job.UUID, err) } return fmt.Errorf("failed to bung (target ID: %s, job ID: %s): %w", job.TargetID, job.UUID, err) } if resourceType == datastore.ResourceTypeUnknown { resourceType = target.ResourceType } runnerName := runner.ToName(job.UUID.String()) if config.Config.Strict { if err := s.checkRegisteredRunner(ctx, runnerName, *target); err != nil { logger.Logf(false, "failed to check to register runner (target ID: %s, job ID: %s): %+v\n", job.TargetID, job.UUID, err) if err := deleteInstance(ctx, cloudID, job.CheckEventJSON); err != nil { logger.Logf(false, "failed to delete an instance that not registered instance (target ID: %s, cloud ID: %s): %+v\n", job.TargetID, cloudID, err) // not return, need to update target status if err. } if err := datastore.UpdateTargetStatus(ctx, s.ds, job.TargetID, datastore.TargetStatusErr, fmt.Sprintf("cannot register runner to GitHub (job ID: %s)", job.UUID)); err != nil { return fmt.Errorf("failed to update target status (target ID: %s, job ID: %s): %w", job.TargetID, job.UUID, err) } return fmt.Errorf("failed to check to register runner (target ID: %s, job ID: %s): %w", job.TargetID, job.UUID, err) } } r := datastore.Runner{ UUID: job.UUID, ShoesType: shoesType, IPAddress: ipAddress, TargetID: job.TargetID, CloudID: cloudID, ResourceType: resourceType, RunnerUser: sql.NullString{ String: config.Config.RunnerUser, Valid: true, }, ProviderURL: target.ProviderURL, RepositoryURL: job.RepoURL(), RequestWebhook: job.CheckEventJSON, } if err := s.ds.CreateRunner(ctx, r); err != nil { logger.Logf(false, "failed to save runner to datastore (target ID: %s, job ID: %s): %+v\n", job.TargetID, job.UUID, err) if err := datastore.UpdateTargetStatus(ctx, s.ds, job.TargetID, datastore.TargetStatusErr, fmt.Sprintf("job id: %s", job.UUID)); err != nil { return fmt.Errorf("failed to update target status (target ID: %s, job ID: %s): %w", job.TargetID, job.UUID, err) } return fmt.Errorf("failed to save runner to datastore (target ID: %s, job ID: %s): %w", job.TargetID, job.UUID, err) } if err := s.ds.DeleteJob(ctx, job.UUID); err != nil { logger.Logf(false, "failed to delete job: %+v\n", err) if err := datastore.UpdateTargetStatus(ctx, s.ds, job.TargetID, datastore.TargetStatusErr, fmt.Sprintf("job id: %s", job.UUID)); err != nil { return fmt.Errorf("failed to update target status (target ID: %s, job ID: %s): %w", job.TargetID, job.UUID, err) } return fmt.Errorf("failed to delete job: %w", err) } return nil } // bung is start runner, like a pistol! :) func (s *Starter) bung(ctx context.Context, job datastore.Job, target datastore.Target) (string, string, string, datastore.ResourceType, error) { runID, jobID, extractErr := extractWorkflowIDs(job) if extractErr != nil { logger.Logf(false, "start create instance (job: %s)", job.UUID) } else { logger.Logf(false, "start create instance (job: %s, gh_run_id: %d, gh_job_id: %d)", job.UUID, runID, jobID) } runnerName := runner.ToName(job.UUID.String()) targetScope := getTargetScope(target, job) script, err := s.GetSetupScript(ctx, targetScope, runnerName) if err != nil { return "", "", "", datastore.ResourceTypeUnknown, fmt.Errorf("failed to get setup scripts: %w", err) } client, teardown, err := shoes.GetClient() if err != nil { return "", "", "", datastore.ResourceTypeUnknown, fmt.Errorf("failed to get plugin client: %w", err) } defer teardown() labels, err := gh.ExtractRunsOnLabels([]byte(job.CheckEventJSON)) if err != nil { return "", "", "", datastore.ResourceTypeUnknown, fmt.Errorf("failed to extract labels: %w", err) } cloudID, ipAddress, shoesType, resourceType, err := client.AddInstance(ctx, runnerName, script, target.ResourceType, labels) if err != nil { if stat, _ := status.FromError(err); stat.Code() == codes.InvalidArgument { return "", "", "", datastore.ResourceTypeUnknown, NewInvalidLabel(err) } return "", "", "", datastore.ResourceTypeUnknown, fmt.Errorf("failed to add instance: %w", err) } if extractErr != nil { logger.Logf(false, "instance create successfully! (job: %s, cloud ID: %s)", job.UUID, cloudID) } else { logger.Logf(false, "instance create successfully! (job: %s, cloud ID: %s, gh_run_id: %d, gh_job_id: %d)", job.UUID, cloudID, runID, jobID) } return cloudID, ipAddress, shoesType, resourceType, nil } // getTargetScope from target, but receive from job if datastore.target.Scope is empty // this function is for datastore that don't store target. func getTargetScope(target datastore.Target, job datastore.Job) string { if target.Scope == "" { return job.Repository } return target.Scope } func deleteInstance(ctx context.Context, cloudID, checkEventJSON string) error { client, teardown, err := shoes.GetClient() if err != nil { return fmt.Errorf("failed to get plugin client: %w", err) } defer teardown() labels, err := gh.ExtractRunsOnLabels([]byte(checkEventJSON)) if err != nil { return fmt.Errorf("failed to extract labels: %w", err) } if err := client.DeleteInstance(ctx, cloudID, labels); err != nil { return fmt.Errorf("failed to delete instance: %w", err) } logger.Logf(false, "successfully delete instance that not registered (cloud ID: %s)", cloudID) return nil } // checkRegisteredRunner check to register runner to GitHub func (s *Starter) checkRegisteredRunner(ctx context.Context, runnerName string, target datastore.Target) error { client, err := gh.NewClient(target.GitHubToken) if err != nil { return fmt.Errorf("failed to create github client: %w", err) } owner, repo := gh.DivideScope(target.Scope) cctx, cancel := context.WithTimeout(ctx, runner.MustRunningTime) defer cancel() ticker := time.NewTicker(1 * time.Second) defer ticker.Stop() count := 0 for { select { case <-cctx.Done(): // timeout return fmt.Errorf("faied to to check existing runner in GitHub: timeout in %s", runner.MustRunningTime) case <-ticker.C: if _, err := gh.ExistGitHubRunner(cctx, client, owner, repo, runnerName); err == nil { // success to register runner to GitHub return nil } else if !errors.Is(err, gh.ErrNotFound) { // not retryable error return fmt.Errorf("failed to check existing runner in GitHub: %w", err) } count++ logger.Logf(true, "%s is not found in GitHub, will retry... (second: %ds)", runnerName, count) } } } func (s *Starter) reRunWorkflow(ctx context.Context) { pendingRuns, err := datastore.GetPendingWorkflowRunByRecentRepositories(ctx, s.ds) if err != nil { logger.Logf(false, "failed to get pending workflow runs: %+v", err) return } for _, pendingRun := range pendingRuns { if err := reRunWorkflowByPendingRun(ctx, s.ds, pendingRun); err != nil { logger.Logf(false, "failed to re-run workflow: %+v", err) continue } } } func reRunWorkflowByPendingRun(ctx context.Context, ds datastore.Datastore, pendingRun datastore.PendingWorkflowRunWithTarget) error { if err := enqueueRescueRun(ctx, pendingRun, ds); err != nil { return fmt.Errorf("failed to enqueue rescue job: %w", err) } return nil } func enqueueRescueRun(ctx context.Context, pendingRun datastore.PendingWorkflowRunWithTarget, ds datastore.Datastore) error { fullName := pendingRun.WorkflowRun.GetRepository().GetFullName() client, target, err := datastore.NewClientInstallationByRepo(ctx, ds, fullName) if err != nil { return fmt.Errorf("failed to create a client of GitHub by repo (full_name: %s): %w", fullName, err) } jobs, err := gh.ListWorkflowJobByRunID( ctx, client, pendingRun.WorkflowRun.GetRepository().GetOwner().GetLogin(), pendingRun.WorkflowRun.GetRepository().GetName(), pendingRun.WorkflowRun.GetID(), ) if err != nil { return fmt.Errorf("failed to list workflow jobs: %w", err) } for _, job := range jobs { if job.GetStatus() != "queued" && job.GetStatus() != "pending" { continue } // Check if the job has appropriate labels for myshoes if !gh.IsRequestedMyshoesLabel(job.Labels) { logger.Logf(true, "skip rescue job because it doesn't have myshoes labels: (repo: %s, gh_run_id: %d, gh_job_id: %d, labels: %v)", fullName, pendingRun.WorkflowRun.GetID(), job.GetID(), job.Labels) continue } // Get installation ID from target scope installationID, err := gh.IsInstalledGitHubApp(ctx, target.Scope) if err != nil { return fmt.Errorf("failed to get installation ID: %w", err) } // Get full installation data from cache installation, err := gh.GetInstallationByID(ctx, installationID) if err != nil { logger.Logf(false, "failed to get installation from cache (installationID: %d), using minimal data: %+v", installationID, err) // Fallback to minimal installation data installation = &github.Installation{ ID: &installationID, } } owner := pendingRun.WorkflowRun.GetRepository().GetOwner() var org *github.Organization if owner != nil { org = &github.Organization{ ID: owner.ID, Login: owner.Login, Name: owner.Name, } } event := &github.WorkflowJobEvent{ WorkflowJob: job, Action: github.Ptr("queued"), Org: org, Repo: pendingRun.WorkflowRun.GetRepository(), Sender: pendingRun.WorkflowRun.GetActor(), Installation: installation, } if err := enqueueRescueJob(ctx, event, *target, ds); err != nil { return fmt.Errorf("failed to enqueue rescue job: %w", err) } } return nil } func enqueueRescueJob(ctx context.Context, workflowJob *github.WorkflowJobEvent, target datastore.Target, ds datastore.Datastore) error { jobJSON, err := json.Marshal(workflowJob) if err != nil { return fmt.Errorf("failed to marshal job: %w", err) } repository := workflowJob.GetRepo() if repository == nil { return fmt.Errorf("repository is nil") } fullName := repository.GetFullName() if fullName == "" { return fmt.Errorf("repository full name is empty") } htmlURL := repository.GetHTMLURL() if htmlURL == "" { return fmt.Errorf("repository html url is empty") } u, err := url.Parse(htmlURL) if err != nil { return fmt.Errorf("failed to parse repository url from event: %w", err) } gheDomain := "" if u.Host != "github.com" { gheDomain = fmt.Sprintf("%s://%s", u.Scheme, u.Host) } logger.Logf(false, "rescue pending job: (repo: %s, gh_run_id: %d, gh_job_id: %d)", *repository.HTMLURL, workflowJob.WorkflowJob.GetRunID(), workflowJob.WorkflowJob.GetID()) jobID := uuid.NewV4() job := datastore.Job{ UUID: jobID, GHEDomain: sql.NullString{ String: gheDomain, Valid: gheDomain != "", }, Repository: fullName, CheckEventJSON: string(jobJSON), TargetID: target.UUID, } if err := ds.EnqueueJob(ctx, job); err != nil { return fmt.Errorf("failed to enqueue job: %w", err) } // Increment rescued runs counter v, _ := CountRescued.LoadOrStore(target.Scope, &atomic.Int64{}) counter := v.(*atomic.Int64) counter.Add(1) return nil } ================================================ FILE: pkg/web/config.go ================================================ package web import ( "encoding/json" "net/http" "github.com/whywaita/myshoes/pkg/config" "github.com/whywaita/myshoes/pkg/logger" ) type inputConfigDebug struct { Debug bool `json:"debug"` } type inputConfigStrict struct { Strict bool `json:"strict"` } func handleConfigDebug(w http.ResponseWriter, r *http.Request) { i := inputConfigDebug{} if err := json.NewDecoder(r.Body).Decode(&i); err != nil { logger.Logf(false, "failed to decode request body: %+v", err) outputErrorMsg(w, http.StatusBadRequest, "json decode error") return } config.Config.Debug = i.Debug logger.Logf(false, "switch debug mode to %t", i.Debug) w.WriteHeader(http.StatusNoContent) } func handleConfigStrict(w http.ResponseWriter, r *http.Request) { i := inputConfigStrict{} if err := json.NewDecoder(r.Body).Decode(&i); err != nil { logger.Logf(false, "failed to decode request body: %+v", err) outputErrorMsg(w, http.StatusBadRequest, "json decode error") return } config.Config.Strict = i.Strict logger.Logf(false, "switch strict mode to %t", i.Strict) w.WriteHeader(http.StatusNoContent) } ================================================ FILE: pkg/web/http.go ================================================ package web import ( "context" "encoding/json" "fmt" "net/http" "time" "github.com/whywaita/myshoes/pkg/config" "github.com/whywaita/myshoes/pkg/datastore" "github.com/whywaita/myshoes/pkg/logger" goji "goji.io" "goji.io/pat" ) // NewMux create routed mux func NewMux(ds datastore.Datastore) *goji.Mux { mux := goji.NewMux() mux.HandleFunc(pat.Get("/healthz"), func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json;charset=utf-8") w.WriteHeader(http.StatusOK) h := struct { Health string `json:"health"` }{ Health: "ok", } json.NewEncoder(w).Encode(h) }) mux.HandleFunc(pat.Post("/github/events"), func(w http.ResponseWriter, r *http.Request) { apacheLogging(r) HandleGitHubEvent(w, r, ds) }) // REST API for targets mux.HandleFunc(pat.Post("/target"), func(w http.ResponseWriter, r *http.Request) { apacheLogging(r) handleTargetCreate(w, r, ds) }) mux.HandleFunc(pat.Get("/target"), func(w http.ResponseWriter, r *http.Request) { apacheLogging(r) handleTargetList(w, r, ds) }) mux.HandleFunc(pat.Get("/target/:id"), func(w http.ResponseWriter, r *http.Request) { apacheLogging(r) handleTargetRead(w, r, ds) }) mux.HandleFunc(pat.Post("/target/:id"), func(w http.ResponseWriter, r *http.Request) { apacheLogging(r) handleTargetUpdate(w, r, ds) }) mux.HandleFunc(pat.Delete("/target/:id"), func(w http.ResponseWriter, r *http.Request) { apacheLogging(r) handleTargetDelete(w, r, ds) }) // Config endpoints mux.HandleFunc(pat.Post("/config/debug"), func(w http.ResponseWriter, r *http.Request) { apacheLogging(r) handleConfigDebug(w, r) }) mux.HandleFunc(pat.Post("/config/strict"), func(w http.ResponseWriter, r *http.Request) { apacheLogging(r) handleConfigStrict(w, r) }) // metrics endpoint mux.HandleFunc(pat.Get("/metrics"), func(w http.ResponseWriter, r *http.Request) { apacheLogging(r) HandleMetrics(w, r, ds) }) return mux } // Serve start webhook receiver func Serve(ctx context.Context, ds datastore.Datastore) error { mux := NewMux(ds) listenAddress := fmt.Sprintf(":%d", config.Config.Port) s := &http.Server{ Addr: listenAddress, Handler: mux, } errCh := make(chan error) go func() { defer close(errCh) logger.Logf(false, "start webhook receiver, listen %s", listenAddress) if err := s.ListenAndServe(); err != nil { errCh <- fmt.Errorf("failed to listen and serve: %w", err) } }() select { case <-ctx.Done(): return s.Shutdown(ctx) case err := <-errCh: return fmt.Errorf("occurred error in web serve: %w", err) } } func apacheLogging(r *http.Request) { t := time.Now().UTC() logger.Logf(false, "HTTP - %s - - %s \"%s %s %s\"\n", r.RemoteAddr, t.Format("02/Jan/2006:15:04:05 -0700"), r.Method, r.URL.Path, r.Proto, //interceptor.HTTPStatus, //interceptor.ResponseSize, //r.UserAgent(), //time.Since(t), ) } ================================================ FILE: pkg/web/http_test.go ================================================ package web_test import ( "os" "testing" "github.com/whywaita/myshoes/internal/testutils" ) func TestMain(m *testing.M) { os.Exit(testutils.IntegrationTestRunner(m)) } ================================================ FILE: pkg/web/metrics.go ================================================ package web import ( "net/http" "github.com/whywaita/myshoes/pkg/datastore" "github.com/whywaita/myshoes/pkg/metric" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" ) // HandleMetrics handle metrics endpoint func HandleMetrics(w http.ResponseWriter, r *http.Request, ds datastore.Datastore) { ctx := r.Context() registry := prometheus.NewRegistry() registry.MustRegister(metric.NewCollector(ctx, ds)) gatherers := prometheus.Gatherers{ prometheus.DefaultGatherer, registry, } h := promhttp.HandlerFor(gatherers, promhttp.HandlerOpts{}) h.ServeHTTP(w, r) } ================================================ FILE: pkg/web/target.go ================================================ package web import ( "database/sql" "encoding/json" "fmt" "net/http" "sort" "strings" "time" "github.com/r3labs/diff/v2" uuid "github.com/satori/go.uuid" "github.com/whywaita/myshoes/pkg/datastore" "github.com/whywaita/myshoes/pkg/gh" "github.com/whywaita/myshoes/pkg/logger" "goji.io/pat" ) // TargetCreateParam is parameter for POST /target type TargetCreateParam struct { datastore.Target GHEDomain *string `json:"ghe_domain"` // ignore RunnerUser *string `json:"runner_user"` // nullable ProviderURL *string `json:"provider_url"` // nullable } // UserTarget is format for user type UserTarget struct { UUID uuid.UUID `json:"id"` Scope string `json:"scope"` TokenExpiredAt time.Time `json:"token_expired_at"` ResourceType string `json:"resource_type"` ProviderURL string `json:"provider_url"` Status datastore.TargetStatus `json:"status"` StatusDescription string `json:"status_description"` CreatedAt time.Time `json:"created_at"` UpdatedAt time.Time `json:"updated_at"` } func sortUserTarget(uts []UserTarget) []UserTarget { sort.SliceStable(uts, func(i, j int) bool { if uts[i].CreatedAt != uts[j].CreatedAt { return uts[i].CreatedAt.After(uts[j].CreatedAt) } iType := datastore.UnmarshalResourceTypeString(uts[i].ResourceType) jType := datastore.UnmarshalResourceTypeString(uts[j].ResourceType) return iType < jType }) return uts } // function pointer (for testing) var ( GHExistGitHubRepositoryFunc = gh.ExistGitHubRepository GHExistRunnerReleases = gh.ExistRunnerReleases GHListRunnersFunc = gh.ListRunners GHIsInstalledGitHubApp = gh.IsInstalledGitHubApp GHGenerateGitHubAppsToken = gh.GenerateGitHubAppsToken GHNewClientApps = gh.NewClientGitHubApps GHPurgeInstallationCache = gh.PurgeInstallationCache ) func handleTargetList(w http.ResponseWriter, r *http.Request, ds datastore.Datastore) { ctx := r.Context() ts, err := datastore.ListTargets(ctx, ds) if err != nil { logger.Logf(false, "failed to retrieve list of target: %+v", err) outputErrorMsg(w, http.StatusInternalServerError, "datastore read error") } fmt.Println(ts) var targets []UserTarget for _, t := range ts { ut := sanitizeTarget(t) targets = append(targets, ut) } targets = sortUserTarget(targets) w.Header().Set("Content-Type", "application/json;charset=utf-8") w.WriteHeader(http.StatusOK) json.NewEncoder(w).Encode(targets) } func handleTargetRead(w http.ResponseWriter, r *http.Request, ds datastore.Datastore) { ctx := r.Context() targetID, err := parseReqTargetID(r) if err != nil { logger.Logf(false, "failed to decode request body: %+v", err) outputErrorMsg(w, http.StatusBadRequest, "incorrect target id") return } target, err := ds.GetTarget(ctx, targetID) if err != nil { logger.Logf(false, "failed to retrieve target from datastore: %+v", err) outputErrorMsg(w, http.StatusInternalServerError, "datastore read error") return } ut := sanitizeTarget(*target) w.Header().Set("Content-Type", "application/json;charset=utf-8") w.WriteHeader(http.StatusOK) json.NewEncoder(w).Encode(ut) } func sanitizeTarget(t datastore.Target) UserTarget { ut := UserTarget{ UUID: t.UUID, Scope: t.Scope, TokenExpiredAt: t.TokenExpiredAt, ResourceType: t.ResourceType.String(), ProviderURL: t.ProviderURL.String, Status: t.Status, StatusDescription: t.StatusDescription.String, CreatedAt: t.CreatedAt, UpdatedAt: t.UpdatedAt, } return ut } func handleTargetUpdate(w http.ResponseWriter, r *http.Request, ds datastore.Datastore) { ctx := r.Context() targetID, err := parseReqTargetID(r) if err != nil { logger.Logf(false, "failed to decode request body: %+v", err) outputErrorMsg(w, http.StatusBadRequest, "incorrect target id") return } inputTarget := TargetCreateParam{} if err := json.NewDecoder(r.Body).Decode(&inputTarget); err != nil { logger.Logf(false, "failed to decode request body: %+v", err) outputErrorMsg(w, http.StatusBadRequest, "json decode error") return } newTarget := inputTarget.ToDS("", time.Time{}) oldTarget, err := ds.GetTarget(ctx, targetID) if err != nil { logger.Logf(false, "failed to get target: %+v", err) outputErrorMsg(w, http.StatusBadRequest, "incorrect target id (not found)") return } if err := validateUpdateTarget(*oldTarget, newTarget); err != nil { logger.Logf(false, "input error in validateUpdateTarget: %+v", err) outputErrorMsg(w, http.StatusBadRequest, err.Error()) return } resourceType, providerURL := getWillUpdateTargetVariable(getWillUpdateTargetVariableOld{ resourceType: oldTarget.ResourceType, providerURL: oldTarget.ProviderURL, }, getWillUpdateTargetVariableNew{ resourceType: inputTarget.ResourceType, providerURL: inputTarget.ProviderURL, }) if err := ds.UpdateTargetParam(ctx, targetID, resourceType, providerURL); err != nil { logger.Logf(false, "failed to ds.UpdateTargetParam: %+v", err) outputErrorMsg(w, http.StatusInternalServerError, "datastore update error") return } updatedTarget, err := ds.GetTarget(ctx, targetID) if err != nil { logger.Logf(false, "failed to get recently target in datastore: %+v", err) outputErrorMsg(w, http.StatusInternalServerError, "datastore get error") return } ut := sanitizeTarget(*updatedTarget) w.Header().Set("Content-Type", "application/json;charset=utf-8") w.WriteHeader(http.StatusOK) json.NewEncoder(w).Encode(ut) } func handleTargetDelete(w http.ResponseWriter, r *http.Request, ds datastore.Datastore) { ctx := r.Context() targetID, err := parseReqTargetID(r) if err != nil { logger.Logf(false, "failed to decode request body: %+v", err) outputErrorMsg(w, http.StatusBadRequest, "incorrect target id") return } target, err := ds.GetTarget(ctx, targetID) if err != nil { logger.Logf(false, "failed to get target: %+v", err) outputErrorMsg(w, http.StatusBadRequest, "incorrect target id (not found)") return } switch target.Status { case datastore.TargetStatusRunning: logger.Logf(true, "%s is running now", targetID) outputErrorMsg(w, http.StatusBadRequest, "target has running runner now, please stop all runner") return case datastore.TargetStatusDeleted: outputErrorMsg(w, http.StatusBadRequest, "target is already deleted") return } if err := ds.DeleteTarget(ctx, targetID); err != nil { logger.Logf(false, "failed to delete target in datastore: %+v", err) outputErrorMsg(w, http.StatusInternalServerError, "datastore delete error") return } w.Header().Set("Content-Type", "application/json;charset=utf-8") w.WriteHeader(http.StatusNoContent) } func parseReqTargetID(r *http.Request) (uuid.UUID, error) { targetIDStr := pat.Param(r, "id") targetID, err := uuid.FromString(targetIDStr) if err != nil { return uuid.UUID{}, fmt.Errorf("failed to parse target id: %w", err) } return targetID, nil } // ErrorResponse is error response type ErrorResponse struct { Error string `json:"error"` } func outputErrorMsg(w http.ResponseWriter, status int, msg string) { w.Header().Set("Content-Type", "application/json;charset=utf-8") w.WriteHeader(status) json.NewEncoder(w).Encode(ErrorResponse{Error: msg}) } // validateUpdateTarget check input target that can valid input in update. func validateUpdateTarget(old, new datastore.Target) error { oldv := old newv := new for _, t := range []*datastore.Target{&oldv, &newv} { t.UUID = uuid.UUID{} // can update variables t.ResourceType = datastore.ResourceTypeUnknown t.ProviderURL = sql.NullString{} // time t.TokenExpiredAt = time.Time{} t.CreatedAt = time.Time{} t.UpdatedAt = time.Time{} // generated t.Status = "" t.StatusDescription = sql.NullString{} t.GitHubToken = "" } changelog, err := diff.Diff(oldv, newv) if err != nil { logger.Logf(false, "failed to check diff: %+v", err) return fmt.Errorf("failed to check diff: %w", err) } if len(changelog) != 0 { logger.Logf(false, "invalid updatable parameter: %+v", changelog) var invalidFields []string for _, cl := range changelog { if len(cl.Path) == 2 && !strings.EqualFold(cl.Path[1], "String") { continue } fieldName := cl.Path[0] invalidFields = append(invalidFields, fieldName) } return fmt.Errorf("invalid input: can't updatable fields (%s)", strings.Join(invalidFields, ", ")) } return nil } func isValidTargetCreateParam(input TargetCreateParam) error { if input.Scope == "" || input.ResourceType == datastore.ResourceTypeUnknown { return fmt.Errorf("scope, resource_type must be set") } return nil } func toNullString(input *string) sql.NullString { if input == nil || strings.EqualFold(*input, "") { return sql.NullString{ Valid: false, } } return sql.NullString{ Valid: true, String: *input, } } // ToDS convert to datastore.Target func (t *TargetCreateParam) ToDS(appToken string, tokenExpired time.Time) datastore.Target { providerURL := toNullString(t.ProviderURL) return datastore.Target{ UUID: t.UUID, Scope: t.Scope, GitHubToken: appToken, TokenExpiredAt: tokenExpired, ResourceType: t.ResourceType, ProviderURL: providerURL, } } type getWillUpdateTargetVariableOld struct { resourceType datastore.ResourceType providerURL sql.NullString } type getWillUpdateTargetVariableNew struct { resourceType datastore.ResourceType providerURL *string } func getWillUpdateTargetVariable(oldParam getWillUpdateTargetVariableOld, newParam getWillUpdateTargetVariableNew) (datastore.ResourceType, sql.NullString) { rt := oldParam.resourceType if newParam.resourceType != datastore.ResourceTypeUnknown { rt = newParam.resourceType } providerURL := getWillUpdateTargetVariableString(oldParam.providerURL, newParam.providerURL) return rt, providerURL } func getWillUpdateTargetVariableString(old sql.NullString, new *string) sql.NullString { if new == nil { return old } return toNullString(new) } ================================================ FILE: pkg/web/target_create.go ================================================ package web import ( "context" "database/sql" "encoding/json" "errors" "fmt" "net/http" "time" uuid "github.com/satori/go.uuid" "github.com/whywaita/myshoes/pkg/config" "github.com/whywaita/myshoes/pkg/datastore" "github.com/whywaita/myshoes/pkg/gh" "github.com/whywaita/myshoes/pkg/logger" ) func handleTargetCreate(w http.ResponseWriter, r *http.Request, ds datastore.Datastore) { // input values: scope, gpt, resource_type ctx := r.Context() inputTarget := TargetCreateParam{} if err := json.NewDecoder(r.Body).Decode(&inputTarget); err != nil { logger.Logf(false, "failed to decode request body: %+v", err) outputErrorMsg(w, http.StatusBadRequest, "json decode error") return } if err := isValidTargetCreateParam(inputTarget); err != nil { logger.Logf(false, "failed to validate input: %+v", err) outputErrorMsg(w, http.StatusBadRequest, err.Error()) return } if err := GHPurgeInstallationCache(ctx); err != nil { logger.Logf(false, "failed to purge installation cache: %+v", err) outputErrorMsg(w, http.StatusInternalServerError, "failed to purge installation cache") return } installationID, err := GHIsInstalledGitHubApp(ctx, inputTarget.Scope) if err != nil { logger.Logf(false, "failed to check installed GitHub App: %+v", err) outputErrorMsg(w, http.StatusBadRequest, "failed to check to install GitHub Apps. Are you installed?") return } clientApps, err := GHNewClientApps() if err != nil { logger.Logf(false, "failed to client of GitHub Apps: %+v", err) outputErrorMsg(w, http.StatusInternalServerError, "failed to client GitHub Apps") return } token, expiredAt, err := GHGenerateGitHubAppsToken(ctx, clientApps, installationID, inputTarget.Scope) if err != nil { logger.Logf(false, "failed to generate GitHub Apps Token: %+v", err) outputErrorMsg(w, http.StatusInternalServerError, "failed to generate GitHub Apps token") return } t := inputTarget.ToDS(token, *expiredAt) if err := isValidScopeAndToken(ctx, t.Scope, token); err != nil { outputErrorMsg(w, http.StatusBadRequest, err.Error()) return } target, err := ds.GetTargetByScope(ctx, t.Scope) var targetUUID uuid.UUID switch { case errors.Is(err, datastore.ErrNotFound): // not created, will be creating u, err := createNewTarget(ctx, t, ds) if err != nil { outputErrorMsg(w, http.StatusInternalServerError, err.Error()) return } targetUUID = *u case err != nil: logger.Logf(false, "failed to get target by scope [ghe_domain: %s scope: %s]: %+v", config.Config.GitHubURL, t.Scope, err) outputErrorMsg(w, http.StatusInternalServerError, "datastore error") return case target.Status != datastore.TargetStatusDeleted: // already registered errMsg := fmt.Sprintf("%s is already registered, current status is %s.", t.Scope, target.Status) outputErrorMsg(w, http.StatusBadRequest, errMsg) return case target.Status == datastore.TargetStatusDeleted: // deleted, need to recreate //lint:ignore SA1019 ds.UpdateTargetStatus only use under. if err := ds.UpdateTargetStatus(ctx, target.UUID, datastore.TargetStatusActive, ""); err != nil { logger.Logf(false, "failed to recreate target: %+v", err) outputErrorMsg(w, http.StatusInternalServerError, "datastore recreate error") return } resourceType, providerURL := getWillUpdateTargetVariable(getWillUpdateTargetVariableOld{ resourceType: target.ResourceType, providerURL: target.ProviderURL, }, getWillUpdateTargetVariableNew{ resourceType: inputTarget.ResourceType, providerURL: inputTarget.ProviderURL, }) if err := ds.UpdateTargetParam(ctx, target.UUID, resourceType, providerURL); err != nil { logger.Logf(false, "failed to update resource type in recreating target: %+v", err) outputErrorMsg(w, http.StatusInternalServerError, "update resource type error") return } targetUUID = target.UUID } createdTarget, err := ds.GetTarget(ctx, targetUUID) if err != nil { logger.Logf(false, "failed to get recently target in datastore: %+v", err) outputErrorMsg(w, http.StatusInternalServerError, "datastore get error") return } ut := sanitizeTarget(*createdTarget) w.Header().Set("Content-Type", "application/json;charset=utf-8") w.WriteHeader(http.StatusCreated) json.NewEncoder(w).Encode(ut) } func isValidScopeAndToken(ctx context.Context, scope, githubPersonalToken string) error { if err := GHExistGitHubRepositoryFunc(scope, githubPersonalToken); err != nil { logger.Logf(false, "failed to found github repository: %+v", err) return fmt.Errorf("github scope is invalid (maybe, repository is not found)") } client, err := gh.NewClient(githubPersonalToken) if err != nil { logger.Logf(false, "failed to create GitHub client: %+v", err) return fmt.Errorf("invalid github token in input scope") } owner, repo := gh.DivideScope(scope) if _, err := GHListRunnersFunc(ctx, client, owner, repo); err != nil { logger.Logf(false, "failed to get list of registered runners: %+v", err) return fmt.Errorf("failed to get list of registered runners (maybe, invalid scope or token?)") } return nil } func createNewTarget(ctx context.Context, input datastore.Target, ds datastore.Datastore) (*uuid.UUID, error) { input.UUID = uuid.NewV4() now := time.Now().UTC() input.CreatedAt = now input.UpdatedAt = now input.GHEDomain = sql.NullString{} if config.Config.GitHubURL != "https://github.com" { input.GHEDomain = sql.NullString{ String: config.Config.GitHubURL, Valid: true, } } if err := ds.CreateTarget(ctx, input); err != nil { logger.Logf(false, "failed to create target in datastore: %+v", err) return nil, fmt.Errorf("datastore create error") } return &input.UUID, nil } ================================================ FILE: pkg/web/target_test.go ================================================ package web_test import ( "bytes" "context" "encoding/json" "fmt" "io" "net/http" "strings" "testing" "time" "github.com/google/go-cmp/cmp" "github.com/google/go-github/v80/github" uuid "github.com/satori/go.uuid" "github.com/whywaita/myshoes/internal/testutils" "github.com/whywaita/myshoes/pkg/datastore" "github.com/whywaita/myshoes/pkg/web" ) var testInstallationID = int64(100000000) var testGitHubAppToken = "secret-app-token" var testTime = time.Date(2037, 9, 3, 0, 0, 0, 0, time.UTC) func parseResponse(resp *http.Response) ([]byte, int) { defer resp.Body.Close() content, err := io.ReadAll(resp.Body) if err != nil { panic(err) } return content, resp.StatusCode } func setStubFunctions() { web.GHExistGitHubRepositoryFunc = func(scope string, githubPersonalToken string) error { return nil } web.GHExistRunnerReleases = func(runnerVersion string) error { return nil } web.GHListRunnersFunc = func(ctx context.Context, client *github.Client, owner, repo string) ([]*github.Runner, error) { return nil, nil } web.GHIsInstalledGitHubApp = func(ctx context.Context, inputScope string) (int64, error) { return testInstallationID, nil } web.GHGenerateGitHubAppsToken = func(ctx context.Context, clientInstallation *github.Client, installationID int64, scope string) (string, *time.Time, error) { return testGitHubAppToken, &testTime, nil } web.GHNewClientApps = func() (*github.Client, error) { return &github.Client{}, nil } web.GHPurgeInstallationCache = func(ctx context.Context) error { return nil } } func Test_handleTargetCreate(t *testing.T) { testURL := testutils.GetTestURL() _, teardown := testutils.GetTestDatastore() defer teardown() setStubFunctions() tests := []struct { input string inputGHEDomain string want *web.UserTarget err bool }{ { input: `{"scope": "octocat", "resource_type": "micro", "runner_user": "runner"}`, want: &web.UserTarget{ Scope: "octocat", TokenExpiredAt: testTime, ResourceType: datastore.ResourceTypeMicro.String(), Status: datastore.TargetStatusActive, }, err: false, }, { input: `{"scope": "whywaita/whywaita", "resource_type": "nano", "runner_user": "runner"}`, want: &web.UserTarget{ Scope: "whywaita/whywaita", TokenExpiredAt: testTime, ResourceType: datastore.ResourceTypeNano.String(), Status: datastore.TargetStatusActive, }, }, { // Confirm that no error occurs even if ghe_domain is specified input: `{"scope": "whywaita/whywaita2", "resource_type": "nano", "runner_user": "runner", "ghe_domain": "https://example.com"}`, want: &web.UserTarget{ Scope: "whywaita/whywaita2", TokenExpiredAt: testTime, ResourceType: datastore.ResourceTypeNano.String(), Status: datastore.TargetStatusActive, }, }, } for _, test := range tests { f := func() { resp, err := http.Post(testURL+"/target", "application/json", bytes.NewBufferString(test.input)) if !test.err && err != nil { t.Fatalf("failed to POST request: %+v", err) } content, code := parseResponse(resp) if code != http.StatusCreated { t.Fatalf("must be response statuscode is 201, but got %d", code) } var gotContent web.UserTarget if err := json.Unmarshal(content, &gotContent); err != nil { t.Fatalf("failed to unmarshal resoponse content: %+v", err) } gotContent.UUID = uuid.UUID{} gotContent.CreatedAt = time.Time{} gotContent.UpdatedAt = time.Time{} if diff := cmp.Diff(test.want, &gotContent); diff != "" { t.Errorf("mismatch (-want +got):\n%s", diff) } } f() } } func Test_handleTargetCreate_alreadyRegistered(t *testing.T) { testURL := testutils.GetTestURL() _, teardown := testutils.GetTestDatastore() defer teardown() setStubFunctions() input := `{"scope": "octocat", "resource_type": "micro", "runner_user": "runner", "ghe_domain": "https://example.com"}` // first create resp, err := http.Post(testURL+"/target", "application/json", bytes.NewBufferString(input)) if err != nil { t.Fatalf("failed to POST request: %+v", err) } _, code := parseResponse(resp) if code != http.StatusCreated { t.Fatalf("must be response statuscode is 201, but got %d", code) } // second create resp, err = http.Post(testURL+"/target", "application/json", bytes.NewBufferString(input)) if err != nil { t.Fatalf("failed to POST request: %+v", err) } _, code = parseResponse(resp) if code != http.StatusBadRequest { t.Fatalf("must be response statuscode is 400, but got %d", code) } } func Test_handleTargetCreate_recreated(t *testing.T) { testURL := testutils.GetTestURL() testDatastore, teardown := testutils.GetTestDatastore() defer teardown() setStubFunctions() input := `{"scope": "octocat", "resource_type": "micro", "runner_user": "runner", "ghe_domain": "https://example.com"}` // first create resp, err := http.Post(testURL+"/target", "application/json", bytes.NewBufferString(input)) if err != nil { t.Fatalf("failed to POST request: %+v", err) } content, code := parseResponse(resp) if code != http.StatusCreated { t.Fatalf("must be response statuscode is 201, but got %d: %+v", code, string(content)) } var gotContent web.UserTarget if err := json.Unmarshal(content, &gotContent); err != nil { t.Fatalf("failed to unmarshal resoponse content: %+v", err) } u := gotContent.UUID // first delete client := &http.Client{} req, err := http.NewRequest(http.MethodDelete, fmt.Sprintf("%s/target/%s", testURL, u.String()), nil) if err != nil { t.Fatalf("failed to create request: %+v", err) } resp, err = client.Do(req) if err != nil { t.Fatalf("failed to POST request: %+v", err) } _, code = parseResponse(resp) if code != http.StatusNoContent { t.Fatalf("must be response statuscode is 204, but got %d: %+v", code, string(content)) } // second create resp, err = http.Post(testURL+"/target", "application/json", bytes.NewBufferString(input)) if err != nil { t.Fatalf("failed to POST request: %+v", err) } content, code = parseResponse(resp) if code != http.StatusCreated { t.Fatalf("must be response statuscode is 201, but got %d: %+v", code, string(content)) } got, err := testDatastore.GetTarget(context.Background(), u) if err != nil { t.Fatalf("failed to get created target: %+v", err) } if got.Status != datastore.TargetStatusActive { t.Fatalf("must be status is active when recreated") } } func Test_handleTargetCreate_recreated_update(t *testing.T) { testURL := testutils.GetTestURL() testDatastore, teardown := testutils.GetTestDatastore() defer teardown() setStubFunctions() input := `{"scope": "octocat", "resource_type": "micro", "runner_user": "runner", "ghe_domain": "https://example.com"}` // first create resp, err := http.Post(testURL+"/target", "application/json", bytes.NewBufferString(input)) if err != nil { t.Fatalf("failed to POST request: %+v", err) } content, code := parseResponse(resp) if code != http.StatusCreated { t.Fatalf("must be response statuscode is 201, but got %d: %+v", code, string(content)) } var gotContent web.UserTarget if err := json.Unmarshal(content, &gotContent); err != nil { t.Fatalf("failed to unmarshal resoponse content: %+v", err) } u := gotContent.UUID // first delete client := &http.Client{} req, err := http.NewRequest(http.MethodDelete, fmt.Sprintf("%s/target/%s", testURL, u.String()), nil) if err != nil { t.Fatalf("failed to create request: %+v", err) } resp, err = client.Do(req) if err != nil { t.Fatalf("failed to POST request: %+v", err) } _, code = parseResponse(resp) if code != http.StatusNoContent { t.Fatalf("must be response statuscode is 204, but got %d: %+v", code, string(content)) } // second create secondInput := `{"scope": "octocat", "resource_type": "micro", "runner_user": "runner", "ghe_domain": "https://example.com"}` resp, err = http.Post(testURL+"/target", "application/json", bytes.NewBufferString(secondInput)) if err != nil { t.Fatalf("failed to POST request: %+v", err) } content, code = parseResponse(resp) if code != http.StatusCreated { t.Fatalf("must be response statuscode is 201, but got %d: %+v", code, string(content)) } got, err := testDatastore.GetTarget(context.Background(), u) if err != nil { t.Fatalf("failed to get created target: %+v", err) } if got.Status != datastore.TargetStatusActive { t.Fatalf("must be status is active when recreated") } } func Test_handleTargetList(t *testing.T) { testURL := testutils.GetTestURL() _, teardown := testutils.GetTestDatastore() defer teardown() setStubFunctions() for _, rt := range []string{"nano", "micro"} { target := fmt.Sprintf(`{"scope": "repo%s", "resource_type": "%s", "runner_user": "runner"}`, rt, rt) resp, err := http.Post(testURL+"/target", "application/json", bytes.NewBufferString(target)) if err != nil { t.Fatalf("failed to POST request: %+v", err) } if resp.StatusCode != http.StatusCreated { t.Fatalf("must be response statuscode is 201, but got %d", resp.StatusCode) } } tests := []struct { input interface{} want *[]web.UserTarget err bool }{ { input: nil, want: &[]web.UserTarget{ { Scope: "reponano", TokenExpiredAt: testTime, ResourceType: datastore.ResourceTypeNano.String(), Status: datastore.TargetStatusActive, }, { Scope: "repomicro", TokenExpiredAt: testTime, ResourceType: datastore.ResourceTypeMicro.String(), Status: datastore.TargetStatusActive, }, }, }, } for _, test := range tests { resp, err := http.Get(testURL + "/target") if !test.err && err != nil { t.Fatalf("failed to POST request: %+v", err) } content, code := parseResponse(resp) if code != http.StatusOK { t.Fatalf("must be response statuscode is 201, but got %d: %+v", code, string(content)) } var gotContents []web.UserTarget if err := json.Unmarshal(content, &gotContents); err != nil { t.Fatalf("failed to unmarshal resoponse content: %+v", err) } for i := range gotContents { gotContents[i].UUID = uuid.UUID{} gotContents[i].CreatedAt = time.Time{} gotContents[i].UpdatedAt = time.Time{} } if diff := cmp.Diff(test.want, &gotContents); diff != "" { t.Errorf("mismatch (-want +got):\n%s", diff) } } } func Test_handleTargetRead(t *testing.T) { testURL := testutils.GetTestURL() _, teardown := testutils.GetTestDatastore() defer teardown() setStubFunctions() target := `{"scope": "repo", "resource_type": "micro", "runner_user": "runner"}` resp, err := http.Post(testURL+"/target", "application/json", bytes.NewBufferString(target)) if err != nil { t.Fatalf("failed to POST request: %+v", err) } content, statusCode := parseResponse(resp) if statusCode != http.StatusCreated { t.Fatalf("must be response statuscode is 201, but got %d: %+v", resp.StatusCode, string(content)) } var respTarget web.UserTarget if err := json.Unmarshal(content, &respTarget); err != nil { t.Fatalf("failed to unmarshal response JSON: %+v", err) } targetUUID := respTarget.UUID tests := []struct { input uuid.UUID want *web.UserTarget err bool }{ { input: targetUUID, want: &web.UserTarget{ UUID: targetUUID, Scope: "repo", TokenExpiredAt: testTime, ResourceType: datastore.ResourceTypeMicro.String(), Status: datastore.TargetStatusActive, }, }, } for _, test := range tests { resp, err := http.Get(fmt.Sprintf("%s/target/%s", testURL, test.input)) if !test.err && err != nil { t.Fatalf("failed to POST request: %+v", err) } content, code := parseResponse(resp) if code != http.StatusOK { t.Fatalf("must be response statuscode is 201, but got %d: %+v", code, string(content)) } var got web.UserTarget if err := json.Unmarshal(content, &got); err != nil { t.Fatalf("failed to unmarshal resoponse content: %+v", err) } got.CreatedAt = time.Time{} got.UpdatedAt = time.Time{} if diff := cmp.Diff(test.want, &got); diff != "" { t.Errorf("mismatch (-want +got):\n%s", diff) } } } func Test_handleTargetUpdate(t *testing.T) { testURL := testutils.GetTestURL() _, teardown := testutils.GetTestDatastore() defer teardown() setStubFunctions() tests := []struct { input string want *web.UserTarget err bool }{ { // Update a few values input: `{"scope": "repo", "resource_type": "nano"}`, want: &web.UserTarget{ UUID: uuid.UUID{}, Scope: "repo", TokenExpiredAt: testTime, ResourceType: datastore.ResourceTypeNano.String(), ProviderURL: "https://example.com/default-shoes", Status: datastore.TargetStatusActive, }, }, { // Confirm that no error occurs even if ghe_domain is specified input: `{"scope": "repo", "resource_type": "nano", "ghe_domain": "https://example.com"}`, want: &web.UserTarget{ UUID: uuid.UUID{}, Scope: "repo", TokenExpiredAt: testTime, ResourceType: datastore.ResourceTypeNano.String(), ProviderURL: "https://example.com/default-shoes", Status: datastore.TargetStatusActive, }, }, { // Update all values input: `{"scope": "repo", "resource_type": "micro", "provider_url": "https://example.com/shoes-provider"}`, want: &web.UserTarget{ UUID: uuid.UUID{}, Scope: "repo", TokenExpiredAt: testTime, ResourceType: datastore.ResourceTypeMicro.String(), ProviderURL: "https://example.com/shoes-provider", Status: datastore.TargetStatusActive, }, }, { // Update value only one, other value is not update input: `{"scope": "repo", "resource_type": "nano"}`, want: &web.UserTarget{ UUID: uuid.UUID{}, Scope: "repo", TokenExpiredAt: testTime, ResourceType: datastore.ResourceTypeNano.String(), ProviderURL: "https://example.com/default-shoes", Status: datastore.TargetStatusActive, }, }, { // Remove provider_url, Set blank input: `{"scope": "repo", "resource_type": "nano" ,"provider_url": ""}`, want: &web.UserTarget{ UUID: uuid.UUID{}, Scope: "repo", TokenExpiredAt: testTime, ResourceType: datastore.ResourceTypeNano.String(), ProviderURL: "", Status: datastore.TargetStatusActive, }, }, } for _, test := range tests { target := `{"scope": "repo", "resource_type": "micro", "runner_user": "runner", "provider_url": "https://example.com/default-shoes"}` respCreate, err := http.Post(testURL+"/target", "application/json", bytes.NewBufferString(target)) if err != nil { t.Fatalf("failed to POST request: %+v", err) } contentCreate, statusCode := parseResponse(respCreate) if statusCode != http.StatusCreated { t.Fatalf("must be response statuscode is 201, but got %d: %+v", respCreate.StatusCode, string(contentCreate)) } var respTarget web.UserTarget if err := json.Unmarshal(contentCreate, &respTarget); err != nil { t.Fatalf("failed to unmarshal response JSON: %+v", err) } targetUUID := respTarget.UUID resp, err := http.Post(fmt.Sprintf("%s/target/%s", testURL, targetUUID.String()), "application/json", bytes.NewBufferString(test.input)) if !test.err && err != nil { t.Fatalf("failed to POST request: %+v", err) } content, code := parseResponse(resp) if code != http.StatusOK { t.Fatalf("must be response statuscode is 200, but got %d: %+v", code, string(content)) } var got web.UserTarget if err := json.Unmarshal(content, &got); err != nil { t.Fatalf("failed to unmarshal resoponse content: %+v", err) } got.UUID = uuid.UUID{} got.CreatedAt = time.Time{} got.UpdatedAt = time.Time{} if diff := cmp.Diff(test.want, &got); diff != "" { t.Errorf("mismatch (-want +got):\n%s", diff) } teardown() } } func Test_handleTargetUpdate_Error(t *testing.T) { testURL := testutils.GetTestURL() _, teardown := testutils.GetTestDatastore() defer teardown() setStubFunctions() tests := []struct { input string wantCode int want string }{ { // Invalid: must set scope input: `{"resource_type": "nano", "runner_user": "runner"}`, wantCode: http.StatusBadRequest, want: `{"error":"invalid input: can't updatable fields (Scope)"}`, }, } for _, test := range tests { target := `{"scope": "repo", "resource_type": "micro", "runner_user": "runner", "provider_url": "https://example.com/default-shoes"}` respCreate, err := http.Post(testURL+"/target", "application/json", bytes.NewBufferString(target)) if err != nil { t.Fatalf("failed to POST request: %+v", err) } contentCreate, statusCode := parseResponse(respCreate) if statusCode != http.StatusCreated { t.Fatalf("must be response statuscode is 201, but got %d: %+v", respCreate.StatusCode, string(contentCreate)) } var respTarget web.UserTarget if err := json.Unmarshal(contentCreate, &respTarget); err != nil { t.Fatalf("failed to unmarshal response JSON: %+v", err) } targetUUID := respTarget.UUID resp, err := http.Post(fmt.Sprintf("%s/target/%s", testURL, targetUUID.String()), "application/json", bytes.NewBufferString(test.input)) if err != nil { t.Fatalf("failed to POST request: %+v", err) } content, code := parseResponse(resp) got := string(content) if code != test.wantCode { t.Fatalf("must be response statuscode is %d, but got %d: %+v", test.wantCode, code, got) } if strings.EqualFold(test.want, got) { t.Fatalf("invalid error response: %+v", string(content)) } teardown() } } func Test_handleTargetDelete(t *testing.T) { testURL := testutils.GetTestURL() testDatastore, teardown := testutils.GetTestDatastore() defer teardown() setStubFunctions() target := `{"scope": "repo", "resource_type": "micro", "runner_user": "runner"}` resp, err := http.Post(testURL+"/target", "application/json", bytes.NewBufferString(target)) if err != nil { t.Fatalf("failed to POST request: %+v", err) } content, statusCode := parseResponse(resp) if statusCode != http.StatusCreated { t.Fatalf("must be response statuscode is 201, but got %d: %+v", resp.StatusCode, string(content)) } var respTarget web.UserTarget if err := json.Unmarshal(content, &respTarget); err != nil { t.Fatalf("failed to unmarshal response JSON: %+v", err) } targetUUID := respTarget.UUID tests := []struct { input uuid.UUID want *datastore.Target err bool }{ { input: targetUUID, want: &datastore.Target{ UUID: targetUUID, Scope: "repo", GitHubToken: testGitHubAppToken, TokenExpiredAt: testTime, ResourceType: datastore.ResourceTypeMicro, Status: datastore.TargetStatusDeleted, }, }, } for _, test := range tests { client := &http.Client{} req, err := http.NewRequest(http.MethodDelete, fmt.Sprintf("%s/target/%s", testURL, test.input), nil) if err != nil { t.Fatalf("failed to create request: %+v", err) } resp, err := client.Do(req) if !test.err && err != nil { t.Fatalf("failed to POST request: %+v", err) } content, code := parseResponse(resp) if code != http.StatusNoContent { t.Fatalf("must be response statuscode is 204, but got %d: %+v", code, string(content)) } got, err := testDatastore.GetTarget(context.Background(), test.input) if err != nil { t.Fatalf("failed to get target from datastore: %+v", err) } got.CreatedAt = time.Time{} got.UpdatedAt = time.Time{} if diff := cmp.Diff(test.want, got); diff != "" { t.Errorf("mismatch (-want +got):\n%s", diff) } } } ================================================ FILE: pkg/web/webhook.go ================================================ package web import ( "context" "database/sql" "encoding/json" "fmt" "net/http" "net/url" "strings" "time" "github.com/google/go-github/v80/github" uuid "github.com/satori/go.uuid" "github.com/whywaita/myshoes/pkg/config" "github.com/whywaita/myshoes/pkg/datastore" "github.com/whywaita/myshoes/pkg/gh" "github.com/whywaita/myshoes/pkg/logger" "github.com/whywaita/myshoes/pkg/metric" ) // HandleGitHubEvent handle GitHub webhook event func HandleGitHubEvent(w http.ResponseWriter, r *http.Request, ds datastore.Datastore) { ctx := r.Context() startTime := time.Now() eventType := github.WebHookType(r) payload, err := github.ValidatePayload(r, config.Config.GitHub.AppSecret) if err != nil { logger.Logf(false, "failed to validate webhook payload: %+v\n", err) w.WriteHeader(http.StatusBadRequest) metric.WebhookReceivedTotal.WithLabelValues(eventType, "invalid", "unknown").Inc() return } webhookEvent, err := github.ParseWebHook(eventType, payload) if err != nil { logger.Logf(false, "failed to parse webhook payload: %+v\n", err) w.WriteHeader(http.StatusBadRequest) metric.WebhookReceivedTotal.WithLabelValues(eventType, "parse_error", "unknown").Inc() return } // Extract runs-on labels runsOn := "unknown" if eventType == "workflow_job" { labels, err := gh.ExtractRunsOnLabels(payload) if err == nil && len(labels) > 0 { runsOn = strings.Join(labels, ",") } } switch event := webhookEvent.(type) { case *github.PingEvent: if err := receivePingWebhook(ctx, event); err != nil { logger.Logf(false, "failed to process ping event: %+v\n", err) w.WriteHeader(http.StatusInternalServerError) metric.WebhookReceivedTotal.WithLabelValues("ping", "error", "n/a").Inc() return } w.WriteHeader(http.StatusOK) metric.WebhookReceivedTotal.WithLabelValues("ping", "success", "n/a").Inc() metric.WebhookProcessingDuration.WithLabelValues("ping", "n/a").Observe(time.Since(startTime).Seconds()) return case *github.CheckRunEvent: if !config.Config.ModeWebhookType.Equal("check_run") { logger.Logf(false, "receive CheckRunEvent, but set %s. So ignore", config.Config.ModeWebhookType) return } if err := receiveCheckRunWebhook(ctx, event, ds); err != nil { logger.Logf(false, "failed to process check_run event: %+v\n", err) w.WriteHeader(http.StatusInternalServerError) metric.WebhookReceivedTotal.WithLabelValues("check_run", "error", "n/a").Inc() return } w.WriteHeader(http.StatusOK) metric.WebhookReceivedTotal.WithLabelValues("check_run", "success", "n/a").Inc() metric.WebhookProcessingDuration.WithLabelValues("check_run", "n/a").Observe(time.Since(startTime).Seconds()) return case *github.WorkflowJobEvent: if !config.Config.ModeWebhookType.Equal("workflow_job") { logger.Logf(false, "receive WorkflowJobEvent, but set %s. So ignore", config.Config.ModeWebhookType) return } if err := receiveWorkflowJobWebhook(ctx, event, ds); err != nil { logger.Logf(false, "failed to process workflow_job event: %+v\n", err) w.WriteHeader(http.StatusInternalServerError) metric.WebhookReceivedTotal.WithLabelValues("workflow_job", "error", runsOn).Inc() return } w.WriteHeader(http.StatusOK) metric.WebhookReceivedTotal.WithLabelValues("workflow_job", "success", runsOn).Inc() metric.WebhookProcessingDuration.WithLabelValues("workflow_job", runsOn).Observe(time.Since(startTime).Seconds()) return default: logger.Logf(false, "receive not register event(%+v), return NotFound", event) w.WriteHeader(http.StatusNotFound) metric.WebhookReceivedTotal.WithLabelValues(eventType, "not_found", "unknown").Inc() return } } func receivePingWebhook(_ context.Context, event *github.PingEvent) error { // do nothing return nil } func receiveCheckRunWebhook(ctx context.Context, event *github.CheckRunEvent, ds datastore.Datastore) error { action := event.GetAction() installationID := event.GetInstallation().GetID() repo := event.GetRepo() repoName := repo.GetFullName() repoURL := repo.GetHTMLURL() if action != "created" { logger.Logf(true, "check_action is not created, ignore (%s)", action) return nil } jb, err := json.Marshal(event) if err != nil { return fmt.Errorf("failed to json.Marshal: %w", err) } if err := processCheckRun(ctx, ds, repoName, repoURL, installationID, jb); err != nil { return err } // Record job enqueued metric metric.WebhookJobsEnqueued.WithLabelValues("check_run", repoName, "n/a").Inc() return nil } // processCheckRun process webhook event // repoName is :owner/:repo // repoURL is https://github.com/:owenr/:repo (in github.com) or https://github.example.com/:owner/:repo (in GitHub Enterprise) func processCheckRun(ctx context.Context, ds datastore.Datastore, repoName, repoURL string, installationID int64, requestJSON []byte) error { if err := gh.CheckSignature(installationID); err != nil { return fmt.Errorf("failed to create GitHub client: %w", err) } u, err := url.Parse(repoURL) if err != nil { return fmt.Errorf("failed to parse repository url from event: %w", err) } //var domain string gheDomain := "" if u.Host != "github.com" { gheDomain = fmt.Sprintf("%s://%s", u.Scheme, u.Host) } logger.Logf(false, "receive webhook repository: %s/%s", gheDomain, repoName) target, err := datastore.SearchRepo(ctx, ds, repoName) if err != nil { return fmt.Errorf("failed to search registered target: %w", err) } if !target.CanReceiveJob() { // do nothing if status is cannot receive logger.Logf(false, "%s/%s is %s now, do nothing", gheDomain, repoName, target.Status) return nil } jobID := uuid.NewV4() j := datastore.Job{ UUID: jobID, GHEDomain: sql.NullString{ String: gheDomain, Valid: gheDomain != "", }, Repository: repoName, CheckEventJSON: string(requestJSON), TargetID: target.UUID, } if err := ds.EnqueueJob(ctx, j); err != nil { return fmt.Errorf("failed to enqueue job: %w", err) } return nil } func receiveWorkflowJobWebhook(ctx context.Context, event *github.WorkflowJobEvent, ds datastore.Datastore) error { action := event.GetAction() installationID := event.GetInstallation().GetID() repo := event.GetRepo() repoName := repo.GetFullName() repoURL := repo.GetHTMLURL() labels := event.GetWorkflowJob().Labels if !gh.IsRequestedMyshoesLabel(labels) { // is not request myshoes, So will be ignored logger.Logf(true, "label \"myshoes\" is not found in labels, so ignore (labels: %s)", labels) return nil } if action != "queued" { logger.Logf(true, "workflow_job actions is not queued, ignore") return nil } jb, err := json.Marshal(event) if err != nil { return fmt.Errorf("failed to json.Marshal: %w", err) } if err := processCheckRun(ctx, ds, repoName, repoURL, installationID, jb); err != nil { return err } // Record job enqueued metric for workflow_job runsOn := strings.Join(labels, ",") metric.WebhookJobsEnqueued.WithLabelValues("workflow_job", repoName, runsOn).Inc() return nil }